Merge pull request #3283 from leseb/rgw-refactor
ceph: refactor rgw bootstrap
This commit is contained in:
21 files changed
+240
-169
No files matched your search
@@ -36,7 +36,6 @@ spec:
|
||||
port: 80
|
||||
securePort:
|
||||
instances: 1
|
||||
allNodes: false
|
||||
# A key/value list of annotations
|
||||
annotations:
|
||||
# key: value
|
||||
@@ -85,8 +84,7 @@ The gateway settings correspond to the RGW daemon settings.
|
||||
- `sslCertificateRef`: If the certificate is not specified, SSL will not be configured. If specified, this is the name of the Kubernetes secret that contains the SSL certificate to be used for secure connections to the object store. Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be in the format expected by the [RGW service](http://docs.ceph.com/docs/master/install/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in pem form."
|
||||
- `port`: The port on which the RGW pods and the RGW service will be listening (not encrypted).
|
||||
- `securePort`: The secure port on which RGW pods will be listening. An SSL certificate must be specified.
|
||||
- `instances`: The number of pods that will be started to load balance this object store. Ignored if `allNodes` is true.
|
||||
- `allNodes`: Whether RGW pods should be started on all nodes. If true, a daemonset is created. If false, `instances` must be set.
|
||||
- `instances`: The number of pods that will be started to load balance this object store.
|
||||
- `annotations`: Key value pair list of annotations to add.
|
||||
- `placement`: The Kubernetes placement settings to determine where the RGW pods should be started in the cluster.
|
||||
- `resources`: Set resource requests/limits for the Gateway Pod(s), see [Resource Requirements/Limits](ceph-cluster-crd.md#resource-requirementslimits).
|
||||
|
||||
@@ -42,7 +42,6 @@ spec:
|
||||
port: 80
|
||||
securePort:
|
||||
instances: 1
|
||||
allNodes: false
|
||||
```
|
||||
|
||||
When the object store is created the Rook operator will create all the pools and other resources necessary to start the service. This may take a minute to complete.
|
||||
|
||||
@@ -13,6 +13,7 @@ an example usage
|
||||
- Rgw pods have liveness probe enabled
|
||||
- Rgw is now configured with the Beast backend as of the Nautilus release
|
||||
- OSD: newly updated cluster from 0.9 to 1.0.3 and thus Ceph Nautilus will have their OSDs allowing new features for Nautilus
|
||||
- Rgw instances have their own key and thus are properly reflected in the Ceph status
|
||||
|
||||
## Breaking Changes
|
||||
|
||||
@@ -24,4 +25,11 @@ an example usage
|
||||
|
||||
## Deprecations
|
||||
|
||||
### Ceph
|
||||
|
||||
- For rgw, when deploying an object store with `object.yaml`, using `allNodes` is not supported anymore, a transition path has been implemented in the code though.
|
||||
So if you were using `allNodes: true`, Rook will replace each daemonset with a deployment (one for one replacement) gradually.
|
||||
This operation will be triggered on an update or when a new version of the operator is deployed.
|
||||
Once complete, it is expected that you edit your object CR with `kubectl -n rook-ceph edit cephobjectstore.ceph.rook.io/my-store` and set `allNodes: false` and `instances` with the current number of rgw instances.
|
||||
|
||||
### <Storage Provider>
|
||||
@@ -31,10 +31,8 @@ spec:
|
||||
port: 80
|
||||
# The port that RGW pods will listen on (https). An ssl certificate is required.
|
||||
securePort:
|
||||
# The number of pods in the rgw deployment (ignored if allNodes=true)
|
||||
# The number of pods in the rgw deployment
|
||||
instances: 1
|
||||
# Whether the rgw pods should be deployed on all nodes as a daemonset
|
||||
allNodes: false
|
||||
# The affinity rules to apply to the rgw deployment or daemonset.
|
||||
placement:
|
||||
# nodeAffinity:
|
||||
|
||||
@@ -30,10 +30,8 @@ spec:
|
||||
port: 8080
|
||||
# The port that RGW pods will listen on (https). An ssl certificate is required.
|
||||
securePort:
|
||||
# The number of pods in the rgw deployment (ignored if allNodes=true)
|
||||
# The number of pods in the rgw deployment
|
||||
instances: 1
|
||||
# Whether the rgw pods should be deployed on all nodes as a daemonset
|
||||
allNodes: false
|
||||
# The affinity rules to apply to the rgw deployment or daemonset.
|
||||
placement:
|
||||
# nodeAffinity:
|
||||
|
||||
@@ -20,4 +20,3 @@ spec:
|
||||
port: 80
|
||||
securePort:
|
||||
instances: 1
|
||||
allNodes: false
|
||||
@@ -30,10 +30,8 @@ spec:
|
||||
port: 80
|
||||
# The port that RGW pods will listen on (https). An ssl certificate is required.
|
||||
securePort:
|
||||
# The number of pods in the rgw deployment (ignored if allNodes=true)
|
||||
# The number of pods in the rgw deployment
|
||||
instances: 1
|
||||
# Whether the rgw pods should be deployed on all nodes as a daemonset
|
||||
allNodes: false
|
||||
# The affinity rules to apply to the rgw deployment or daemonset.
|
||||
placement:
|
||||
# nodeAffinity:
|
||||
|
||||
@@ -151,7 +151,6 @@ metadata:
|
||||
"port": 8080,
|
||||
"securePort": null,
|
||||
"instances": 1,
|
||||
"allNodes": false,
|
||||
"placement": null,
|
||||
"annotations": null,
|
||||
"resources": null
|
||||
|
||||
@@ -34,7 +34,6 @@ spec:
|
||||
port: 80
|
||||
securePort: 443
|
||||
instances: 3
|
||||
allNodes: false
|
||||
```
|
||||
|
||||
Now create the object store.
|
||||
@@ -82,8 +81,7 @@ The gateway settings correspond to the RGW service.
|
||||
- `sslCertificateRef`: If specified, this is the name of the Kubernetes secret that contains the SSL certificate to be used for secure connections to the object store. The secret must be in the same namespace as the Rook cluster. Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be in the format expected by the [RGW service](http://docs.ceph.com/docs/master/install/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in pem form." If the certificate is not specified, SSL will not be configured.
|
||||
- `port`: The service port where the RGW service will be listening (http)
|
||||
- `securePort`: The service port where the RGW service will be listening (https)
|
||||
- `instances`: The number of RGW pods that will be started for this object store (ignored if allNodes=true)
|
||||
- `allNodes`: Whether all nodes in the cluster should run RGW as a daemonset
|
||||
- `instances`: The number of RGW pods that will be started for this object store
|
||||
- `placement`: The rgw pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](/cluster/examples/kubernetes/ceph/cluster.yaml).
|
||||
|
||||
The RGW service can be configured to listen on both http and https by specifying both `port` and `securePort`.
|
||||
@@ -95,7 +93,6 @@ The RGW service can be configured to listen on both http and https by specifying
|
||||
port: 80
|
||||
securePort: 443
|
||||
instances: 1
|
||||
allNodes: false
|
||||
```
|
||||
|
||||
### Realms, Zone Groups, and Zones
|
||||
|
||||
@@ -62,7 +62,7 @@ func (c *Cluster) generateKeyring(m *mgrConfig) error {
|
||||
/* TODO: can we change this ownerref to be the deployment or service? */
|
||||
s := keyring.GetSecretStore(c.context, c.Namespace, &c.ownerRef)
|
||||
|
||||
key, err := s.GenerateKey(m.ResourceName, user, access)
|
||||
key, err := s.GenerateKey(user, access)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@ func (m *Mirroring) generateKeyring(daemonConfig *daemonConfig) error {
|
||||
access := []string{"mon", "profile rbd-mirror", "osd", "profile rbd"}
|
||||
s := keyring.GetSecretStore(m.context, m.Namespace, &m.ownerRef)
|
||||
|
||||
key, err := s.GenerateKey(daemonConfig.ResourceName, user, access)
|
||||
key, err := s.GenerateKey(user, access)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/api/core/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
@@ -65,7 +65,7 @@ func keyringSecretName(resourceName string) string {
|
||||
// GenerateKey generates a key for a Ceph user with the given access permissions. It returns the key
|
||||
// generated on success. Ceph will always return the most up-to-date key for a daemon, and the key
|
||||
// usually does not change.
|
||||
func (k *SecretStore) GenerateKey(resourceName, user string, access []string) (string, error) {
|
||||
func (k *SecretStore) GenerateKey(user string, access []string) (string, error) {
|
||||
// get-or-create-key for the user account
|
||||
key, err := client.AuthGetOrCreateKey(k.context, k.namespace, user, access)
|
||||
if err != nil {
|
||||
|
||||
@@ -52,20 +52,20 @@ func TestGenerateKey(t *testing.T) {
|
||||
|
||||
generateKey = "generatedsecretkey"
|
||||
failGenerateKey = false
|
||||
k, e := s.GenerateKey("testresource", "testuser", []string{"test", "access"})
|
||||
k, e := s.GenerateKey("testuser", []string{"test", "access"})
|
||||
assert.NoError(t, e)
|
||||
assert.Equal(t, "generatedsecretkey", k)
|
||||
|
||||
generateKey = "differentsecretkey"
|
||||
failGenerateKey = false
|
||||
k, e = s.GenerateKey("testresource", "testuser", []string{"test", "access"})
|
||||
k, e = s.GenerateKey("testuser", []string{"test", "access"})
|
||||
assert.NoError(t, e)
|
||||
assert.Equal(t, "differentsecretkey", k)
|
||||
|
||||
// make sure error on fail
|
||||
generateKey = "failgeneratekey"
|
||||
failGenerateKey = true
|
||||
_, e = s.GenerateKey("newresource", "newuser", []string{"new", "access"})
|
||||
_, e = s.GenerateKey("newuser", []string{"new", "access"})
|
||||
assert.Error(t, e)
|
||||
}
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ func (c *Cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error {
|
||||
}
|
||||
s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef)
|
||||
|
||||
key, err := s.GenerateKey(m.ResourceName, user, access)
|
||||
key, err := s.GenerateKey(user, access)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -20,17 +20,17 @@ import (
|
||||
"fmt"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
keyringTemplate = `
|
||||
[client.radosgw.gateway]
|
||||
[%s]
|
||||
key = %s
|
||||
caps mon = "allow rw"
|
||||
caps osd = "allow rwx"
|
||||
@@ -87,27 +87,22 @@ func (c *clusterConfig) portString() string {
|
||||
return portString
|
||||
}
|
||||
|
||||
func generateCephXUser(name string) string {
|
||||
user := strings.TrimPrefix(name, AppName)
|
||||
return "client" + strings.Replace(user, "-", ".", -1)
|
||||
}
|
||||
|
||||
func (c *clusterConfig) generateKeyring(replicationControllerOwnerRef *metav1.OwnerReference) error {
|
||||
user := "client.radosgw.gateway"
|
||||
user := generateCephXUser(replicationControllerOwnerRef.Name)
|
||||
/* TODO: this says `osd allow rwx` while template says `osd allow *`; which is correct? */
|
||||
access := []string{"osd", "allow rwx", "mon", "allow rw"}
|
||||
s := keyring.GetSecretStore(c.context, c.store.Namespace, replicationControllerOwnerRef)
|
||||
|
||||
key, err := s.GenerateKey(c.instanceName(), user, access)
|
||||
key, err := s.GenerateKey(user, access)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x
|
||||
err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Delete(c.instanceName(), &metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
logger.Debugf("legacy rgw key %s is already removed", c.instanceName())
|
||||
} else {
|
||||
logger.Warningf("legacy rgw key %s could not be removed. %+v", c.instanceName(), err)
|
||||
}
|
||||
}
|
||||
|
||||
keyring := fmt.Sprintf(keyringTemplate, key)
|
||||
return s.CreateOrUpdate(c.instanceName(), keyring)
|
||||
keyring := fmt.Sprintf(keyringTemplate, user, key)
|
||||
return s.CreateOrUpdate(replicationControllerOwnerRef.Name, keyring)
|
||||
}
|
||||
@@ -86,3 +86,8 @@ func TestFrontend(t *testing.T) {
|
||||
result = rgwFrontend(cfg.clusterInfo.CephVersion)
|
||||
assert.Equal(t, "beast", result)
|
||||
}
|
||||
|
||||
func TestGenerateCephXUser(t *testing.T) {
|
||||
fakeUser := generateCephXUser("rook-ceph-rgw-fake-store-fake-user")
|
||||
assert.Equal(t, "client.fake.store.fake.user", fakeUser)
|
||||
}
|
||||
+159
-51
@@ -19,16 +19,17 @@ package object
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/pool"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
)
|
||||
|
||||
type clusterConfig struct {
|
||||
@@ -42,6 +43,15 @@ type clusterConfig struct {
|
||||
DataPathMap *config.DataPathMap
|
||||
}
|
||||
|
||||
type rgwConfig struct {
|
||||
ResourceName string
|
||||
DaemonID string
|
||||
}
|
||||
|
||||
const (
|
||||
oldRgwKeyName = "client.radosgw.gateway"
|
||||
)
|
||||
|
||||
// Start the rgw manager
|
||||
func (c *clusterConfig) createStore() error {
|
||||
return c.createOrUpdate(false)
|
||||
@@ -91,64 +101,145 @@ func (c *clusterConfig) createOrUpdate(update bool) error {
|
||||
}
|
||||
|
||||
func (c *clusterConfig) startRGWPods(update bool) error {
|
||||
|
||||
// if intended to update, remove the old pods so they can be created with the new spec settings
|
||||
if update {
|
||||
err := k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, c.instanceName())
|
||||
if err != nil {
|
||||
logger.Warning(err.Error())
|
||||
}
|
||||
err = k8sutil.DeleteDaemonset(c.context.Clientset, c.store.Namespace, c.instanceName())
|
||||
if err != nil {
|
||||
logger.Warning(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// start the deployment or daemonset
|
||||
var uid types.UID
|
||||
var controllerType string
|
||||
// backward compatibility, triggered during updates
|
||||
if c.store.Spec.Gateway.AllNodes {
|
||||
daemonSet, err := c.startDaemonset()
|
||||
// log we don't support that anymore
|
||||
logger.Warningf(
|
||||
"setting 'AllNodes' to %t is not supported anymore, please use 'instances' instead, removing old DaemonSets if any and replace them with Deployments in object store %s",
|
||||
c.store.Spec.Gateway.AllNodes, c.store.Name)
|
||||
}
|
||||
if c.store.Spec.Gateway.Instances < 1 {
|
||||
// Set the minimum of at least one instance
|
||||
logger.Warningf("spec.gateway.instances must be set to at least 1")
|
||||
c.store.Spec.Gateway.Instances = 1
|
||||
}
|
||||
|
||||
// start a new deployment and scale up
|
||||
desiredRgwInstances := int(c.store.Spec.Gateway.Instances)
|
||||
for i := 0; i < desiredRgwInstances; i++ {
|
||||
daemonLetterID := k8sutil.IndexToName(i)
|
||||
// Each rgw is id'ed by <store_name>-<letterID>
|
||||
daemonName := fmt.Sprintf("%s-%s", c.store.Name, daemonLetterID)
|
||||
// resource name is rook-ceph-rgw-<store_name>-<daemon_name>
|
||||
resourceName := fmt.Sprintf("%s-%s-%s", AppName, c.store.Name, daemonLetterID)
|
||||
|
||||
rgwConfig := &rgwConfig{
|
||||
ResourceName: resourceName,
|
||||
DaemonID: daemonName,
|
||||
}
|
||||
|
||||
deployment, err := c.startDeployment(rgwConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
uid = daemonSet.UID
|
||||
controllerType = "daemonset"
|
||||
} else {
|
||||
deployment, err := c.startDeployment()
|
||||
|
||||
resourceControllerOwnerRef := &metav1.OwnerReference{
|
||||
UID: deployment.UID,
|
||||
APIVersion: "v1",
|
||||
Kind: "deployment",
|
||||
Name: rgwConfig.ResourceName,
|
||||
}
|
||||
|
||||
// Generate the keyring after starting the replication controller so that the keyring may use
|
||||
// the controller as its owner reference; the keyring is deleted with the controller
|
||||
err = c.generateKeyring(resourceControllerOwnerRef)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to create rgw keyring. %+v", err)
|
||||
}
|
||||
|
||||
// Generate the mime.types file after the rep. controller as well for the same reason as keyring
|
||||
if err := c.generateMimeTypes(resourceControllerOwnerRef); err != nil {
|
||||
return fmt.Errorf("failed to generate the rgw mime.types config. %+v", err)
|
||||
}
|
||||
uid = deployment.UID
|
||||
controllerType = "deployment"
|
||||
}
|
||||
|
||||
resourceControllerOwnerRef := &metav1.OwnerReference{
|
||||
UID: uid,
|
||||
APIVersion: "v1",
|
||||
Kind: controllerType,
|
||||
Name: c.instanceName(),
|
||||
}
|
||||
|
||||
// Generate the keyring after starting the replication controller so that the keyring may use
|
||||
// the controller as its owner reference; the keyring is deleted with the controller
|
||||
err := c.generateKeyring(resourceControllerOwnerRef)
|
||||
// scale down scenario
|
||||
deps, err := k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create rgw keyring. %+v", err)
|
||||
logger.Warning("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Name, c.storeLabelSelector(), err)
|
||||
}
|
||||
|
||||
// Generate the mime.types file after the rep. controller as well for the same reason as keyring
|
||||
if err := c.generateMimeTypes(resourceControllerOwnerRef); err != nil {
|
||||
return fmt.Errorf("failed to generate the rgw mime.types config. %+v", err)
|
||||
currentRgwInstances := int(len(deps.Items))
|
||||
if currentRgwInstances > desiredRgwInstances {
|
||||
logger.Infof("found more rgw deployments %d than desired %d in object store %s, scaling down", currentRgwInstances, c.store.Spec.Gateway.Instances, c.store.Name)
|
||||
diffCount := currentRgwInstances - desiredRgwInstances
|
||||
for i := 0; i < diffCount; {
|
||||
depIDToRemove := currentRgwInstances - 1
|
||||
depNameToRemove := fmt.Sprintf("%s-%s-%s", AppName, c.store.Name, k8sutil.IndexToName(depIDToRemove))
|
||||
if err := k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, depNameToRemove); err != nil {
|
||||
logger.Warning("error during deletion of deployment %s resource: %+v", depNameToRemove, err)
|
||||
}
|
||||
currentRgwInstances = currentRgwInstances - 1
|
||||
i++
|
||||
|
||||
// Delete the auth key
|
||||
err = client.AuthDelete(c.context, c.store.Namespace, generateCephXUser(depNameToRemove))
|
||||
if err != nil {
|
||||
logger.Infof("failed to delete rgw key %s. %+v", depNameToRemove, err)
|
||||
}
|
||||
}
|
||||
// verify scale down was successful
|
||||
deps, err = k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector())
|
||||
if err != nil {
|
||||
logger.Warning("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Name, c.storeLabelSelector(), err)
|
||||
}
|
||||
currentRgwInstances = len(deps.Items)
|
||||
if currentRgwInstances == desiredRgwInstances {
|
||||
logger.Infof("successfully scaled down rgw deployments to %d in object store %s", desiredRgwInstances, c.store.Name)
|
||||
}
|
||||
}
|
||||
|
||||
c.deleteLegacyDaemons()
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteLegacyDaemons removes legacy rgw components that might have existed in Rook v1.0
|
||||
func (c *clusterConfig) deleteLegacyDaemons() {
|
||||
// Make a best effort to delete the rgw pods daemonsets
|
||||
daemons, err := k8sutil.GetDaemonsets(c.context.Clientset, c.store.Namespace, c.storeLabelSelector())
|
||||
if err != nil {
|
||||
logger.Warningf("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Name, c.storeLabelSelector(), err)
|
||||
}
|
||||
daemonsetNum := len(daemons.Items)
|
||||
if daemonsetNum > 0 {
|
||||
for _, d := range daemons.Items {
|
||||
// Delete any existing daemonset
|
||||
if err := k8sutil.DeleteDaemonset(c.context.Clientset, c.store.Namespace, d.Name); err != nil {
|
||||
logger.Errorf("error during deletion of daemonset %s resource: %+v", d.Name, err)
|
||||
}
|
||||
}
|
||||
// Delete legacy rgw key
|
||||
err = client.AuthDelete(c.context, c.store.Namespace, oldRgwKeyName)
|
||||
if err != nil {
|
||||
logger.Infof("failed to delete legacy rgw key %s. %+v", oldRgwKeyName, err)
|
||||
}
|
||||
}
|
||||
|
||||
// legacy deployment detection
|
||||
logger.Debugf("looking for legacy deployment in object store %s", c.store.Name)
|
||||
deps, err := k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector())
|
||||
if err != nil {
|
||||
logger.Warning("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Name, c.storeLabelSelector(), err)
|
||||
}
|
||||
for _, d := range deps.Items {
|
||||
if d.Name == c.instanceName() {
|
||||
logger.Infof("legacy deployment in object store %s found %s", c.store.Name, d.Name)
|
||||
if err := k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, d.Name); err != nil {
|
||||
logger.Warning("error during deletion of deployment %s resource: %+v", d.Name, err)
|
||||
}
|
||||
// Delete legacy rgw key
|
||||
err = client.AuthDelete(c.context, c.store.Namespace, oldRgwKeyName)
|
||||
if err != nil {
|
||||
logger.Infof("failed to delete legacy rgw key %s. %+v", oldRgwKeyName, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the object store.
|
||||
// WARNING: This is a very destructive action that deletes all metadata and data pools.
|
||||
func (c *clusterConfig) deleteStore() error {
|
||||
// check if the object store exists
|
||||
// check if the object store exists
|
||||
exists, err := c.storeExists()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to detect if there is an object store to delete. %+v", err)
|
||||
@@ -170,22 +261,33 @@ func (c *clusterConfig) deleteStore() error {
|
||||
logger.Warningf("failed to delete rgw service. %+v", err)
|
||||
}
|
||||
|
||||
// Make a best effort to delete the rgw pods
|
||||
err = k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, c.instanceName())
|
||||
// Make a best effort to delete the rgw pods deployments
|
||||
deps, err := k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector())
|
||||
if err != nil {
|
||||
logger.Warning(err.Error())
|
||||
logger.Warning("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Namespace, c.storeLabelSelector(), err)
|
||||
}
|
||||
err = k8sutil.DeleteDaemonset(c.context.Clientset, c.store.Namespace, c.instanceName())
|
||||
if err != nil {
|
||||
logger.Warning(err.Error())
|
||||
for _, d := range deps.Items {
|
||||
if err := k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, d.Name); err != nil {
|
||||
logger.Warning("error during deletion of deployment %s resource: %+v", d.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the rgw keyring
|
||||
// Delete the rgw config map keyrings
|
||||
err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Delete(c.instanceName(), options)
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
logger.Warningf("failed to delete rgw secret. %+v", err)
|
||||
}
|
||||
|
||||
// Delete rgw CephX keys
|
||||
for i := 0; i < int(c.store.Spec.Gateway.Instances); i++ {
|
||||
daemonLetterID := k8sutil.IndexToName(i)
|
||||
keyName := fmt.Sprintf("client.%s.%s", strings.Replace(c.store.Name, "-", ".", -1), daemonLetterID)
|
||||
err = client.AuthDelete(c.context, c.store.Namespace, keyName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the realm and pools
|
||||
objContext := NewContext(c.context, c.store.Name, c.store.Namespace)
|
||||
err = deleteRealmAndPools(objContext)
|
||||
@@ -199,8 +301,9 @@ func (c *clusterConfig) deleteStore() error {
|
||||
|
||||
// Check if the object store exists depending on either the deployment or the daemonset
|
||||
func (c *clusterConfig) storeExists() (bool, error) {
|
||||
_, err := c.context.Clientset.AppsV1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
if err == nil {
|
||||
d, err := k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector())
|
||||
// k8sutil.GetDeployments can return an empty list!
|
||||
if err == nil && len(d.Items) != 0 {
|
||||
// the deployment was found
|
||||
return true, nil
|
||||
}
|
||||
@@ -208,8 +311,9 @@ func (c *clusterConfig) storeExists() (bool, error) {
|
||||
return false, err
|
||||
}
|
||||
|
||||
_, err = c.context.Clientset.AppsV1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
if err == nil {
|
||||
dd, err := k8sutil.GetDaemonsets(c.context.Clientset, c.store.Namespace, c.storeLabelSelector())
|
||||
// k8sutil.GetDaemonsets can return an empty list!
|
||||
if err == nil && len(dd.Items) != 0 {
|
||||
// the daemonset was found
|
||||
return true, nil
|
||||
}
|
||||
@@ -225,6 +329,10 @@ func (c *clusterConfig) instanceName() string {
|
||||
return fmt.Sprintf("%s-%s", AppName, c.store.Name)
|
||||
}
|
||||
|
||||
func (c *clusterConfig) storeLabelSelector() string {
|
||||
return fmt.Sprintf("rook_object_store=%s", c.store.Name)
|
||||
}
|
||||
|
||||
// Validate the object store arguments
|
||||
func validateStore(context *clusterd.Context, s cephv1.CephObjectStore) error {
|
||||
if s.Name == "" {
|
||||
|
||||
@@ -27,7 +27,6 @@ import (
|
||||
testop "github.com/rook/rook/pkg/operator/test"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
)
|
||||
@@ -48,6 +47,7 @@ func TestStartRGW(t *testing.T) {
|
||||
info := testop.CreateConfigDir(1)
|
||||
context := &clusterd.Context{Clientset: clientset, Executor: executor, ConfigDir: configDir}
|
||||
store := simpleStore()
|
||||
store.Spec.Gateway.Instances = 1
|
||||
version := "v1.1.0"
|
||||
data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "my-fs", "rook-ceph", "/var/lib/rook/")
|
||||
|
||||
@@ -56,32 +56,18 @@ func TestStartRGW(t *testing.T) {
|
||||
err := c.createStore()
|
||||
assert.Nil(t, err)
|
||||
|
||||
validateStart(t, c, clientset, false)
|
||||
validateStart(t, c, clientset)
|
||||
|
||||
// starting again should update the pods with the new settings
|
||||
c.store.Spec.Gateway.AllNodes = true
|
||||
err = c.updateStore()
|
||||
assert.Nil(t, err)
|
||||
|
||||
validateStart(t, c, clientset, true)
|
||||
}
|
||||
|
||||
func validateStart(t *testing.T, c *clusterConfig, clientset *fake.Clientset, allNodes bool) {
|
||||
if !allNodes {
|
||||
r, err := clientset.AppsV1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, c.instanceName(), r.Name)
|
||||
|
||||
_, err = clientset.AppsV1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
assert.True(t, errors.IsNotFound(err))
|
||||
} else {
|
||||
r, err := clientset.AppsV1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, c.instanceName(), r.Name)
|
||||
|
||||
_, err = clientset.AppsV1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
assert.True(t, errors.IsNotFound(err))
|
||||
}
|
||||
func validateStart(t *testing.T, c *clusterConfig, clientset *fake.Clientset) {
|
||||
rgwName := c.instanceName() + "-a"
|
||||
r, err := clientset.AppsV1().Deployments(c.store.Namespace).Get(rgwName, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, rgwName, r.Name)
|
||||
|
||||
s, err := clientset.CoreV1().Services(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
|
||||
@@ -29,10 +29,11 @@ import (
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
)
|
||||
|
||||
func (c *clusterConfig) startDeployment() (*apps.Deployment, error) {
|
||||
func (c *clusterConfig) startDeployment(rgwConfig *rgwConfig) (*apps.Deployment, error) {
|
||||
replicas := int32(1)
|
||||
d := &apps.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: c.instanceName(),
|
||||
Name: rgwConfig.ResourceName,
|
||||
Namespace: c.store.Namespace,
|
||||
Labels: c.getLabels(),
|
||||
},
|
||||
@@ -40,8 +41,8 @@ func (c *clusterConfig) startDeployment() (*apps.Deployment, error) {
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: c.getLabels(),
|
||||
},
|
||||
Template: c.makeRGWPodSpec(),
|
||||
Replicas: &c.store.Spec.Gateway.Instances,
|
||||
Template: c.makeRGWPodSpec(rgwConfig),
|
||||
Replicas: &replicas,
|
||||
Strategy: apps.DeploymentStrategy{
|
||||
Type: apps.RecreateDeploymentStrategyType,
|
||||
},
|
||||
@@ -79,55 +80,15 @@ func (c *clusterConfig) startDeployment() (*apps.Deployment, error) {
|
||||
return deployment, err
|
||||
}
|
||||
|
||||
func (c *clusterConfig) startDaemonset() (*apps.DaemonSet, error) {
|
||||
d := &apps.DaemonSet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: c.instanceName(),
|
||||
Namespace: c.store.Namespace,
|
||||
Labels: c.getLabels(),
|
||||
},
|
||||
Spec: apps.DaemonSetSpec{
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: c.getLabels(),
|
||||
},
|
||||
UpdateStrategy: apps.DaemonSetUpdateStrategy{
|
||||
Type: apps.RollingUpdateDaemonSetStrategyType,
|
||||
},
|
||||
Template: c.makeRGWPodSpec(),
|
||||
},
|
||||
}
|
||||
k8sutil.AddRookVersionLabelToDaemonSet(d)
|
||||
opspec.AddCephVersionLabelToDaemonSet(c.clusterInfo.CephVersion, d)
|
||||
k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &d.ObjectMeta, c.ownerRefs)
|
||||
|
||||
logger.Debugf("starting rgw daemonset: %+v", d)
|
||||
daemonSet, err := c.context.Clientset.AppsV1().DaemonSets(c.store.Namespace).Create(d)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return nil, fmt.Errorf("failed to create rgw daemonset %s: %+v", c.instanceName(), err)
|
||||
}
|
||||
logger.Infof("daemonset for rgw %s already exists. updating if needed", c.instanceName())
|
||||
// There may be a *lot* of rgws, and they are stateless, so don't bother waiting until the
|
||||
// entire daemonset is updated to move on.
|
||||
// TODO: is the above statement safe to assume?
|
||||
// TODO: Are there any steps for RGW that need to happen before the daemons upgrade?
|
||||
daemonSet, err = c.context.Clientset.AppsV1().DaemonSets(c.store.Namespace).Update(d)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to update rgw daemonset %s. %+v", c.instanceName(), err)
|
||||
}
|
||||
}
|
||||
return daemonSet, nil
|
||||
}
|
||||
|
||||
func (c *clusterConfig) makeRGWPodSpec() v1.PodTemplateSpec {
|
||||
func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) v1.PodTemplateSpec {
|
||||
podSpec := v1.PodSpec{
|
||||
InitContainers: []v1.Container{},
|
||||
Containers: []v1.Container{
|
||||
c.makeDaemonContainer(),
|
||||
c.makeDaemonContainer(rgwConfig),
|
||||
},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: append(
|
||||
opspec.DaemonVolumes(c.DataPathMap, c.instanceName()),
|
||||
opspec.DaemonVolumes(c.DataPathMap, rgwConfig.ResourceName),
|
||||
c.mimeTypesVolume(),
|
||||
),
|
||||
HostNetwork: c.hostNetwork,
|
||||
@@ -154,7 +115,7 @@ func (c *clusterConfig) makeRGWPodSpec() v1.PodTemplateSpec {
|
||||
|
||||
podTemplateSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: c.instanceName(),
|
||||
Name: rgwConfig.ResourceName,
|
||||
Labels: c.getLabels(),
|
||||
},
|
||||
Spec: podSpec,
|
||||
@@ -164,7 +125,7 @@ func (c *clusterConfig) makeRGWPodSpec() v1.PodTemplateSpec {
|
||||
return podTemplateSpec
|
||||
}
|
||||
|
||||
func (c *clusterConfig) makeDaemonContainer() v1.Container {
|
||||
func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container {
|
||||
|
||||
// start the rgw daemon in the foreground
|
||||
container := v1.Container{
|
||||
@@ -177,13 +138,13 @@ func (c *clusterConfig) makeDaemonContainer() v1.Container {
|
||||
append(
|
||||
opspec.DaemonFlags(c.clusterInfo, c.store.Name),
|
||||
"--foreground",
|
||||
"--name=client.radosgw.gateway",
|
||||
cephconfig.NewFlag("name", generateCephXUser(rgwConfig.ResourceName)),
|
||||
cephconfig.NewFlag("host", opspec.ContainerEnvVarReference("POD_NAME")),
|
||||
cephconfig.NewFlag("rgw-mime-types-file", mimeTypesMountPath()),
|
||||
), c.defaultSettings().GlobalFlags()..., // use default settings as flags until mon kv store supported
|
||||
),
|
||||
VolumeMounts: append(
|
||||
opspec.DaemonVolumeMounts(c.DataPathMap, c.instanceName()),
|
||||
opspec.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName),
|
||||
c.mimeTypesVolumeMount(),
|
||||
),
|
||||
Env: opspec.DaemonEnvVars(c.cephVersion.Image),
|
||||
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
package object
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
@@ -56,7 +57,12 @@ func TestPodSpecs(t *testing.T) {
|
||||
DataPathMap: data,
|
||||
}
|
||||
|
||||
s := c.makeRGWPodSpec()
|
||||
resourceName := fmt.Sprintf("%s-%s", AppName, c.store.Name)
|
||||
rgwConfig := &rgwConfig{
|
||||
ResourceName: resourceName,
|
||||
}
|
||||
|
||||
s := c.makeRGWPodSpec(rgwConfig)
|
||||
|
||||
podTemplate := cephtest.NewPodTemplateSpecTester(t, &s)
|
||||
podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion",
|
||||
@@ -91,7 +97,11 @@ func TestSSLPodSpec(t *testing.T) {
|
||||
}
|
||||
c.hostNetwork = true
|
||||
|
||||
s := c.makeRGWPodSpec()
|
||||
resourceName := fmt.Sprintf("%s-%s", AppName, c.store.Name)
|
||||
rgwConfig := &rgwConfig{
|
||||
ResourceName: resourceName,
|
||||
}
|
||||
s := c.makeRGWPodSpec(rgwConfig)
|
||||
|
||||
podTemplate := cephtest.NewPodTemplateSpecTester(t, &s)
|
||||
podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion",
|
||||
|
||||
@@ -19,8 +19,8 @@ package k8sutil
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"k8s.io/api/apps/v1"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/apps/v1"
|
||||
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
@@ -63,3 +63,15 @@ func AddRookVersionLabelToDaemonSet(d *v1.DaemonSet) {
|
||||
}
|
||||
addRookVersionLabel(d.Labels)
|
||||
}
|
||||
|
||||
// GetDaemonsets returns a list of daemonsets names labels matching a given selector
|
||||
// example of a label selector might be "app=rook-ceph-mon, mon!=b"
|
||||
// more: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
|
||||
func GetDaemonsets(clientset kubernetes.Interface, namespace, labelSelector string) (*apps.DaemonSetList, error) {
|
||||
listOptions := metav1.ListOptions{LabelSelector: labelSelector}
|
||||
daemonsets, err := clientset.AppsV1().DaemonSets(namespace).List(listOptions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list deployments with labelSelector %s: %v", labelSelector, err)
|
||||
}
|
||||
return daemonsets, nil
|
||||
}
|
||||
Reference in new issue
Block a user