Merge pull request #3283 from leseb/rgw-refactor

ceph: refactor rgw bootstrap
This commit is contained in:
Travis Nielsen authored and GitHub committed 2019-06-14 15:17:58 -06:00
commit 8da2221296
21 files changed
+240 -169

No files matched your search

+1 -3
View File
@@ -36,7 +36,6 @@ spec:
port: 80
securePort:
instances: 1
allNodes: false
# A key/value list of annotations
annotations:
# key: value
@@ -85,8 +84,7 @@ The gateway settings correspond to the RGW daemon settings.
- `sslCertificateRef`: If the certificate is not specified, SSL will not be configured. If specified, this is the name of the Kubernetes secret that contains the SSL certificate to be used for secure connections to the object store. Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be in the format expected by the [RGW service](http://docs.ceph.com/docs/master/install/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in pem form."
- `port`: The port on which the RGW pods and the RGW service will be listening (not encrypted).
- `securePort`: The secure port on which RGW pods will be listening. An SSL certificate must be specified.
- `instances`: The number of pods that will be started to load balance this object store. Ignored if `allNodes` is true.
- `allNodes`: Whether RGW pods should be started on all nodes. If true, a daemonset is created. If false, `instances` must be set.
- `instances`: The number of pods that will be started to load balance this object store.
- `annotations`: Key value pair list of annotations to add.
- `placement`: The Kubernetes placement settings to determine where the RGW pods should be started in the cluster.
- `resources`: Set resource requests/limits for the Gateway Pod(s), see [Resource Requirements/Limits](ceph-cluster-crd.md#resource-requirementslimits).
-1
View File
@@ -42,7 +42,6 @@ spec:
port: 80
securePort:
instances: 1
allNodes: false
```
When the object store is created the Rook operator will create all the pools and other resources necessary to start the service. This may take a minute to complete.
+8
View File
@@ -13,6 +13,7 @@ an example usage
- Rgw pods have liveness probe enabled
- Rgw is now configured with the Beast backend as of the Nautilus release
- OSD: newly updated cluster from 0.9 to 1.0.3 and thus Ceph Nautilus will have their OSDs allowing new features for Nautilus
- Rgw instances have their own key and thus are properly reflected in the Ceph status
## Breaking Changes
@@ -24,4 +25,11 @@ an example usage
## Deprecations
### Ceph
- For rgw, when deploying an object store with `object.yaml`, using `allNodes` is not supported anymore, a transition path has been implemented in the code though.
So if you were using `allNodes: true`, Rook will replace each daemonset with a deployment (one for one replacement) gradually.
This operation will be triggered on an update or when a new version of the operator is deployed.
Once complete, it is expected that you edit your object CR with `kubectl -n rook-ceph edit cephobjectstore.ceph.rook.io/my-store` and set `allNodes: false` and `instances` with the current number of rgw instances.
### <Storage Provider>
@@ -31,10 +31,8 @@ spec:
port: 80
# The port that RGW pods will listen on (https). An ssl certificate is required.
securePort:
# The number of pods in the rgw deployment (ignored if allNodes=true)
# The number of pods in the rgw deployment
instances: 1
# Whether the rgw pods should be deployed on all nodes as a daemonset
allNodes: false
# The affinity rules to apply to the rgw deployment or daemonset.
placement:
# nodeAffinity:
@@ -30,10 +30,8 @@ spec:
port: 8080
# The port that RGW pods will listen on (https). An ssl certificate is required.
securePort:
# The number of pods in the rgw deployment (ignored if allNodes=true)
# The number of pods in the rgw deployment
instances: 1
# Whether the rgw pods should be deployed on all nodes as a daemonset
allNodes: false
# The affinity rules to apply to the rgw deployment or daemonset.
placement:
# nodeAffinity:
@@ -20,4 +20,3 @@ spec:
port: 80
securePort:
instances: 1
allNodes: false
+1 -3
View File
@@ -30,10 +30,8 @@ spec:
port: 80
# The port that RGW pods will listen on (https). An ssl certificate is required.
securePort:
# The number of pods in the rgw deployment (ignored if allNodes=true)
# The number of pods in the rgw deployment
instances: 1
# Whether the rgw pods should be deployed on all nodes as a daemonset
allNodes: false
# The affinity rules to apply to the rgw deployment or daemonset.
placement:
# nodeAffinity:
@@ -151,7 +151,6 @@ metadata:
"port": 8080,
"securePort": null,
"instances": 1,
"allNodes": false,
"placement": null,
"annotations": null,
"resources": null
+1 -4
View File
@@ -34,7 +34,6 @@ spec:
port: 80
securePort: 443
instances: 3
allNodes: false
```
Now create the object store.
@@ -82,8 +81,7 @@ The gateway settings correspond to the RGW service.
- `sslCertificateRef`: If specified, this is the name of the Kubernetes secret that contains the SSL certificate to be used for secure connections to the object store. The secret must be in the same namespace as the Rook cluster. Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be in the format expected by the [RGW service](http://docs.ceph.com/docs/master/install/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in pem form." If the certificate is not specified, SSL will not be configured.
- `port`: The service port where the RGW service will be listening (http)
- `securePort`: The service port where the RGW service will be listening (https)
- `instances`: The number of RGW pods that will be started for this object store (ignored if allNodes=true)
- `allNodes`: Whether all nodes in the cluster should run RGW as a daemonset
- `instances`: The number of RGW pods that will be started for this object store
- `placement`: The rgw pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](/cluster/examples/kubernetes/ceph/cluster.yaml).
The RGW service can be configured to listen on both http and https by specifying both `port` and `securePort`.
@@ -95,7 +93,6 @@ The RGW service can be configured to listen on both http and https by specifying
port: 80
securePort: 443
instances: 1
allNodes: false
```
### Realms, Zone Groups, and Zones
+1 -1
View File
@@ -62,7 +62,7 @@ func (c *Cluster) generateKeyring(m *mgrConfig) error {
/* TODO: can we change this ownerref to be the deployment or service? */
s := keyring.GetSecretStore(c.context, c.Namespace, &c.ownerRef)
key, err := s.GenerateKey(m.ResourceName, user, access)
key, err := s.GenerateKey(user, access)
if err != nil {
return err
}
+1 -1
View File
@@ -46,7 +46,7 @@ func (m *Mirroring) generateKeyring(daemonConfig *daemonConfig) error {
access := []string{"mon", "profile rbd-mirror", "osd", "profile rbd"}
s := keyring.GetSecretStore(m.context, m.Namespace, &m.ownerRef)
key, err := s.GenerateKey(daemonConfig.ResourceName, user, access)
key, err := s.GenerateKey(user, access)
if err != nil {
return err
}
+2 -2
View File
@@ -26,7 +26,7 @@ import (
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -65,7 +65,7 @@ func keyringSecretName(resourceName string) string {
// GenerateKey generates a key for a Ceph user with the given access permissions. It returns the key
// generated on success. Ceph will always return the most up-to-date key for a daemon, and the key
// usually does not change.
func (k *SecretStore) GenerateKey(resourceName, user string, access []string) (string, error) {
func (k *SecretStore) GenerateKey(user string, access []string) (string, error) {
// get-or-create-key for the user account
key, err := client.AuthGetOrCreateKey(k.context, k.namespace, user, access)
if err != nil {
@@ -52,20 +52,20 @@ func TestGenerateKey(t *testing.T) {
generateKey = "generatedsecretkey"
failGenerateKey = false
k, e := s.GenerateKey("testresource", "testuser", []string{"test", "access"})
k, e := s.GenerateKey("testuser", []string{"test", "access"})
assert.NoError(t, e)
assert.Equal(t, "generatedsecretkey", k)
generateKey = "differentsecretkey"
failGenerateKey = false
k, e = s.GenerateKey("testresource", "testuser", []string{"test", "access"})
k, e = s.GenerateKey("testuser", []string{"test", "access"})
assert.NoError(t, e)
assert.Equal(t, "differentsecretkey", k)
// make sure error on fail
generateKey = "failgeneratekey"
failGenerateKey = true
_, e = s.GenerateKey("newresource", "newuser", []string{"new", "access"})
_, e = s.GenerateKey("newuser", []string{"new", "access"})
assert.Error(t, e)
}
+1 -1
View File
@@ -46,7 +46,7 @@ func (c *Cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error {
}
s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef)
key, err := s.GenerateKey(m.ResourceName, user, access)
key, err := s.GenerateKey(user, access)
if err != nil {
return err
}
+11 -16
View File
@@ -20,17 +20,17 @@ import (
"fmt"
"path"
"strconv"
"strings"
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
keyringTemplate = `
[client.radosgw.gateway]
[%s]
key = %s
caps mon = "allow rw"
caps osd = "allow rwx"
@@ -87,27 +87,22 @@ func (c *clusterConfig) portString() string {
return portString
}
func generateCephXUser(name string) string {
user := strings.TrimPrefix(name, AppName)
return "client" + strings.Replace(user, "-", ".", -1)
}
func (c *clusterConfig) generateKeyring(replicationControllerOwnerRef *metav1.OwnerReference) error {
user := "client.radosgw.gateway"
user := generateCephXUser(replicationControllerOwnerRef.Name)
/* TODO: this says `osd allow rwx` while template says `osd allow *`; which is correct? */
access := []string{"osd", "allow rwx", "mon", "allow rw"}
s := keyring.GetSecretStore(c.context, c.store.Namespace, replicationControllerOwnerRef)
key, err := s.GenerateKey(c.instanceName(), user, access)
key, err := s.GenerateKey(user, access)
if err != nil {
return err
}
// Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x
err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Delete(c.instanceName(), &metav1.DeleteOptions{})
if err != nil {
if errors.IsNotFound(err) {
logger.Debugf("legacy rgw key %s is already removed", c.instanceName())
} else {
logger.Warningf("legacy rgw key %s could not be removed. %+v", c.instanceName(), err)
}
}
keyring := fmt.Sprintf(keyringTemplate, key)
return s.CreateOrUpdate(c.instanceName(), keyring)
keyring := fmt.Sprintf(keyringTemplate, user, key)
return s.CreateOrUpdate(replicationControllerOwnerRef.Name, keyring)
}
+5
View File
@@ -86,3 +86,8 @@ func TestFrontend(t *testing.T) {
result = rgwFrontend(cfg.clusterInfo.CephVersion)
assert.Equal(t, "beast", result)
}
func TestGenerateCephXUser(t *testing.T) {
fakeUser := generateCephXUser("rook-ceph-rgw-fake-store-fake-user")
assert.Equal(t, "client.fake.store.fake.user", fakeUser)
}
+159 -51
View File
@@ -19,16 +19,17 @@ package object
import (
"fmt"
"strings"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/pool"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
type clusterConfig struct {
@@ -42,6 +43,15 @@ type clusterConfig struct {
DataPathMap *config.DataPathMap
}
type rgwConfig struct {
ResourceName string
DaemonID string
}
const (
oldRgwKeyName = "client.radosgw.gateway"
)
// Start the rgw manager
func (c *clusterConfig) createStore() error {
return c.createOrUpdate(false)
@@ -91,64 +101,145 @@ func (c *clusterConfig) createOrUpdate(update bool) error {
}
func (c *clusterConfig) startRGWPods(update bool) error {
// if intended to update, remove the old pods so they can be created with the new spec settings
if update {
err := k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, c.instanceName())
if err != nil {
logger.Warning(err.Error())
}
err = k8sutil.DeleteDaemonset(c.context.Clientset, c.store.Namespace, c.instanceName())
if err != nil {
logger.Warning(err.Error())
}
}
// start the deployment or daemonset
var uid types.UID
var controllerType string
// backward compatibility, triggered during updates
if c.store.Spec.Gateway.AllNodes {
daemonSet, err := c.startDaemonset()
// log we don't support that anymore
logger.Warningf(
"setting 'AllNodes' to %t is not supported anymore, please use 'instances' instead, removing old DaemonSets if any and replace them with Deployments in object store %s",
c.store.Spec.Gateway.AllNodes, c.store.Name)
}
if c.store.Spec.Gateway.Instances < 1 {
// Set the minimum of at least one instance
logger.Warningf("spec.gateway.instances must be set to at least 1")
c.store.Spec.Gateway.Instances = 1
}
// start a new deployment and scale up
desiredRgwInstances := int(c.store.Spec.Gateway.Instances)
for i := 0; i < desiredRgwInstances; i++ {
daemonLetterID := k8sutil.IndexToName(i)
// Each rgw is id'ed by <store_name>-<letterID>
daemonName := fmt.Sprintf("%s-%s", c.store.Name, daemonLetterID)
// resource name is rook-ceph-rgw-<store_name>-<daemon_name>
resourceName := fmt.Sprintf("%s-%s-%s", AppName, c.store.Name, daemonLetterID)
rgwConfig := &rgwConfig{
ResourceName: resourceName,
DaemonID: daemonName,
}
deployment, err := c.startDeployment(rgwConfig)
if err != nil {
return err
}
uid = daemonSet.UID
controllerType = "daemonset"
} else {
deployment, err := c.startDeployment()
resourceControllerOwnerRef := &metav1.OwnerReference{
UID: deployment.UID,
APIVersion: "v1",
Kind: "deployment",
Name: rgwConfig.ResourceName,
}
// Generate the keyring after starting the replication controller so that the keyring may use
// the controller as its owner reference; the keyring is deleted with the controller
err = c.generateKeyring(resourceControllerOwnerRef)
if err != nil {
return err
return fmt.Errorf("failed to create rgw keyring. %+v", err)
}
// Generate the mime.types file after the rep. controller as well for the same reason as keyring
if err := c.generateMimeTypes(resourceControllerOwnerRef); err != nil {
return fmt.Errorf("failed to generate the rgw mime.types config. %+v", err)
}
uid = deployment.UID
controllerType = "deployment"
}
resourceControllerOwnerRef := &metav1.OwnerReference{
UID: uid,
APIVersion: "v1",
Kind: controllerType,
Name: c.instanceName(),
}
// Generate the keyring after starting the replication controller so that the keyring may use
// the controller as its owner reference; the keyring is deleted with the controller
err := c.generateKeyring(resourceControllerOwnerRef)
// scale down scenario
deps, err := k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector())
if err != nil {
return fmt.Errorf("failed to create rgw keyring. %+v", err)
logger.Warning("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Name, c.storeLabelSelector(), err)
}
// Generate the mime.types file after the rep. controller as well for the same reason as keyring
if err := c.generateMimeTypes(resourceControllerOwnerRef); err != nil {
return fmt.Errorf("failed to generate the rgw mime.types config. %+v", err)
currentRgwInstances := int(len(deps.Items))
if currentRgwInstances > desiredRgwInstances {
logger.Infof("found more rgw deployments %d than desired %d in object store %s, scaling down", currentRgwInstances, c.store.Spec.Gateway.Instances, c.store.Name)
diffCount := currentRgwInstances - desiredRgwInstances
for i := 0; i < diffCount; {
depIDToRemove := currentRgwInstances - 1
depNameToRemove := fmt.Sprintf("%s-%s-%s", AppName, c.store.Name, k8sutil.IndexToName(depIDToRemove))
if err := k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, depNameToRemove); err != nil {
logger.Warning("error during deletion of deployment %s resource: %+v", depNameToRemove, err)
}
currentRgwInstances = currentRgwInstances - 1
i++
// Delete the auth key
err = client.AuthDelete(c.context, c.store.Namespace, generateCephXUser(depNameToRemove))
if err != nil {
logger.Infof("failed to delete rgw key %s. %+v", depNameToRemove, err)
}
}
// verify scale down was successful
deps, err = k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector())
if err != nil {
logger.Warning("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Name, c.storeLabelSelector(), err)
}
currentRgwInstances = len(deps.Items)
if currentRgwInstances == desiredRgwInstances {
logger.Infof("successfully scaled down rgw deployments to %d in object store %s", desiredRgwInstances, c.store.Name)
}
}
c.deleteLegacyDaemons()
return nil
}
// deleteLegacyDaemons removes legacy rgw components that might have existed in Rook v1.0
func (c *clusterConfig) deleteLegacyDaemons() {
// Make a best effort to delete the rgw pods daemonsets
daemons, err := k8sutil.GetDaemonsets(c.context.Clientset, c.store.Namespace, c.storeLabelSelector())
if err != nil {
logger.Warningf("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Name, c.storeLabelSelector(), err)
}
daemonsetNum := len(daemons.Items)
if daemonsetNum > 0 {
for _, d := range daemons.Items {
// Delete any existing daemonset
if err := k8sutil.DeleteDaemonset(c.context.Clientset, c.store.Namespace, d.Name); err != nil {
logger.Errorf("error during deletion of daemonset %s resource: %+v", d.Name, err)
}
}
// Delete legacy rgw key
err = client.AuthDelete(c.context, c.store.Namespace, oldRgwKeyName)
if err != nil {
logger.Infof("failed to delete legacy rgw key %s. %+v", oldRgwKeyName, err)
}
}
// legacy deployment detection
logger.Debugf("looking for legacy deployment in object store %s", c.store.Name)
deps, err := k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector())
if err != nil {
logger.Warning("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Name, c.storeLabelSelector(), err)
}
for _, d := range deps.Items {
if d.Name == c.instanceName() {
logger.Infof("legacy deployment in object store %s found %s", c.store.Name, d.Name)
if err := k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, d.Name); err != nil {
logger.Warning("error during deletion of deployment %s resource: %+v", d.Name, err)
}
// Delete legacy rgw key
err = client.AuthDelete(c.context, c.store.Namespace, oldRgwKeyName)
if err != nil {
logger.Infof("failed to delete legacy rgw key %s. %+v", oldRgwKeyName, err)
}
}
}
}
// Delete the object store.
// WARNING: This is a very destructive action that deletes all metadata and data pools.
func (c *clusterConfig) deleteStore() error {
// check if the object store exists
// check if the object store exists
exists, err := c.storeExists()
if err != nil {
return fmt.Errorf("failed to detect if there is an object store to delete. %+v", err)
@@ -170,22 +261,33 @@ func (c *clusterConfig) deleteStore() error {
logger.Warningf("failed to delete rgw service. %+v", err)
}
// Make a best effort to delete the rgw pods
err = k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, c.instanceName())
// Make a best effort to delete the rgw pods deployments
deps, err := k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector())
if err != nil {
logger.Warning(err.Error())
logger.Warning("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Namespace, c.storeLabelSelector(), err)
}
err = k8sutil.DeleteDaemonset(c.context.Clientset, c.store.Namespace, c.instanceName())
if err != nil {
logger.Warning(err.Error())
for _, d := range deps.Items {
if err := k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, d.Name); err != nil {
logger.Warning("error during deletion of deployment %s resource: %+v", d.Name, err)
}
}
// Delete the rgw keyring
// Delete the rgw config map keyrings
err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Delete(c.instanceName(), options)
if err != nil && !errors.IsNotFound(err) {
logger.Warningf("failed to delete rgw secret. %+v", err)
}
// Delete rgw CephX keys
for i := 0; i < int(c.store.Spec.Gateway.Instances); i++ {
daemonLetterID := k8sutil.IndexToName(i)
keyName := fmt.Sprintf("client.%s.%s", strings.Replace(c.store.Name, "-", ".", -1), daemonLetterID)
err = client.AuthDelete(c.context, c.store.Namespace, keyName)
if err != nil {
return err
}
}
// Delete the realm and pools
objContext := NewContext(c.context, c.store.Name, c.store.Namespace)
err = deleteRealmAndPools(objContext)
@@ -199,8 +301,9 @@ func (c *clusterConfig) deleteStore() error {
// Check if the object store exists depending on either the deployment or the daemonset
func (c *clusterConfig) storeExists() (bool, error) {
_, err := c.context.Clientset.AppsV1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
if err == nil {
d, err := k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector())
// k8sutil.GetDeployments can return an empty list!
if err == nil && len(d.Items) != 0 {
// the deployment was found
return true, nil
}
@@ -208,8 +311,9 @@ func (c *clusterConfig) storeExists() (bool, error) {
return false, err
}
_, err = c.context.Clientset.AppsV1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
if err == nil {
dd, err := k8sutil.GetDaemonsets(c.context.Clientset, c.store.Namespace, c.storeLabelSelector())
// k8sutil.GetDaemonsets can return an empty list!
if err == nil && len(dd.Items) != 0 {
// the daemonset was found
return true, nil
}
@@ -225,6 +329,10 @@ func (c *clusterConfig) instanceName() string {
return fmt.Sprintf("%s-%s", AppName, c.store.Name)
}
func (c *clusterConfig) storeLabelSelector() string {
return fmt.Sprintf("rook_object_store=%s", c.store.Name)
}
// Validate the object store arguments
func validateStore(context *clusterd.Context, s cephv1.CephObjectStore) error {
if s.Name == "" {
+7 -21
View File
@@ -27,7 +27,6 @@ import (
testop "github.com/rook/rook/pkg/operator/test"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)
@@ -48,6 +47,7 @@ func TestStartRGW(t *testing.T) {
info := testop.CreateConfigDir(1)
context := &clusterd.Context{Clientset: clientset, Executor: executor, ConfigDir: configDir}
store := simpleStore()
store.Spec.Gateway.Instances = 1
version := "v1.1.0"
data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "my-fs", "rook-ceph", "/var/lib/rook/")
@@ -56,32 +56,18 @@ func TestStartRGW(t *testing.T) {
err := c.createStore()
assert.Nil(t, err)
validateStart(t, c, clientset, false)
validateStart(t, c, clientset)
// starting again should update the pods with the new settings
c.store.Spec.Gateway.AllNodes = true
err = c.updateStore()
assert.Nil(t, err)
validateStart(t, c, clientset, true)
}
func validateStart(t *testing.T, c *clusterConfig, clientset *fake.Clientset, allNodes bool) {
if !allNodes {
r, err := clientset.AppsV1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, c.instanceName(), r.Name)
_, err = clientset.AppsV1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
assert.True(t, errors.IsNotFound(err))
} else {
r, err := clientset.AppsV1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, c.instanceName(), r.Name)
_, err = clientset.AppsV1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
assert.True(t, errors.IsNotFound(err))
}
func validateStart(t *testing.T, c *clusterConfig, clientset *fake.Clientset) {
rgwName := c.instanceName() + "-a"
r, err := clientset.AppsV1().Deployments(c.store.Namespace).Get(rgwName, metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, rgwName, r.Name)
s, err := clientset.CoreV1().Services(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
assert.Nil(t, err)
+12 -51
View File
@@ -29,10 +29,11 @@ import (
"k8s.io/apimachinery/pkg/util/intstr"
)
func (c *clusterConfig) startDeployment() (*apps.Deployment, error) {
func (c *clusterConfig) startDeployment(rgwConfig *rgwConfig) (*apps.Deployment, error) {
replicas := int32(1)
d := &apps.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: c.instanceName(),
Name: rgwConfig.ResourceName,
Namespace: c.store.Namespace,
Labels: c.getLabels(),
},
@@ -40,8 +41,8 @@ func (c *clusterConfig) startDeployment() (*apps.Deployment, error) {
Selector: &metav1.LabelSelector{
MatchLabels: c.getLabels(),
},
Template: c.makeRGWPodSpec(),
Replicas: &c.store.Spec.Gateway.Instances,
Template: c.makeRGWPodSpec(rgwConfig),
Replicas: &replicas,
Strategy: apps.DeploymentStrategy{
Type: apps.RecreateDeploymentStrategyType,
},
@@ -79,55 +80,15 @@ func (c *clusterConfig) startDeployment() (*apps.Deployment, error) {
return deployment, err
}
func (c *clusterConfig) startDaemonset() (*apps.DaemonSet, error) {
d := &apps.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Name: c.instanceName(),
Namespace: c.store.Namespace,
Labels: c.getLabels(),
},
Spec: apps.DaemonSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: c.getLabels(),
},
UpdateStrategy: apps.DaemonSetUpdateStrategy{
Type: apps.RollingUpdateDaemonSetStrategyType,
},
Template: c.makeRGWPodSpec(),
},
}
k8sutil.AddRookVersionLabelToDaemonSet(d)
opspec.AddCephVersionLabelToDaemonSet(c.clusterInfo.CephVersion, d)
k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &d.ObjectMeta, c.ownerRefs)
logger.Debugf("starting rgw daemonset: %+v", d)
daemonSet, err := c.context.Clientset.AppsV1().DaemonSets(c.store.Namespace).Create(d)
if err != nil {
if !errors.IsAlreadyExists(err) {
return nil, fmt.Errorf("failed to create rgw daemonset %s: %+v", c.instanceName(), err)
}
logger.Infof("daemonset for rgw %s already exists. updating if needed", c.instanceName())
// There may be a *lot* of rgws, and they are stateless, so don't bother waiting until the
// entire daemonset is updated to move on.
// TODO: is the above statement safe to assume?
// TODO: Are there any steps for RGW that need to happen before the daemons upgrade?
daemonSet, err = c.context.Clientset.AppsV1().DaemonSets(c.store.Namespace).Update(d)
if err != nil {
return nil, fmt.Errorf("failed to update rgw daemonset %s. %+v", c.instanceName(), err)
}
}
return daemonSet, nil
}
func (c *clusterConfig) makeRGWPodSpec() v1.PodTemplateSpec {
func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) v1.PodTemplateSpec {
podSpec := v1.PodSpec{
InitContainers: []v1.Container{},
Containers: []v1.Container{
c.makeDaemonContainer(),
c.makeDaemonContainer(rgwConfig),
},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: append(
opspec.DaemonVolumes(c.DataPathMap, c.instanceName()),
opspec.DaemonVolumes(c.DataPathMap, rgwConfig.ResourceName),
c.mimeTypesVolume(),
),
HostNetwork: c.hostNetwork,
@@ -154,7 +115,7 @@ func (c *clusterConfig) makeRGWPodSpec() v1.PodTemplateSpec {
podTemplateSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: c.instanceName(),
Name: rgwConfig.ResourceName,
Labels: c.getLabels(),
},
Spec: podSpec,
@@ -164,7 +125,7 @@ func (c *clusterConfig) makeRGWPodSpec() v1.PodTemplateSpec {
return podTemplateSpec
}
func (c *clusterConfig) makeDaemonContainer() v1.Container {
func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container {
// start the rgw daemon in the foreground
container := v1.Container{
@@ -177,13 +138,13 @@ func (c *clusterConfig) makeDaemonContainer() v1.Container {
append(
opspec.DaemonFlags(c.clusterInfo, c.store.Name),
"--foreground",
"--name=client.radosgw.gateway",
cephconfig.NewFlag("name", generateCephXUser(rgwConfig.ResourceName)),
cephconfig.NewFlag("host", opspec.ContainerEnvVarReference("POD_NAME")),
cephconfig.NewFlag("rgw-mime-types-file", mimeTypesMountPath()),
), c.defaultSettings().GlobalFlags()..., // use default settings as flags until mon kv store supported
),
VolumeMounts: append(
opspec.DaemonVolumeMounts(c.DataPathMap, c.instanceName()),
opspec.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName),
c.mimeTypesVolumeMount(),
),
Env: opspec.DaemonEnvVars(c.cephVersion.Image),
+12 -2
View File
@@ -17,6 +17,7 @@ limitations under the License.
package object
import (
"fmt"
"testing"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -56,7 +57,12 @@ func TestPodSpecs(t *testing.T) {
DataPathMap: data,
}
s := c.makeRGWPodSpec()
resourceName := fmt.Sprintf("%s-%s", AppName, c.store.Name)
rgwConfig := &rgwConfig{
ResourceName: resourceName,
}
s := c.makeRGWPodSpec(rgwConfig)
podTemplate := cephtest.NewPodTemplateSpecTester(t, &s)
podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion",
@@ -91,7 +97,11 @@ func TestSSLPodSpec(t *testing.T) {
}
c.hostNetwork = true
s := c.makeRGWPodSpec()
resourceName := fmt.Sprintf("%s-%s", AppName, c.store.Name)
rgwConfig := &rgwConfig{
ResourceName: resourceName,
}
s := c.makeRGWPodSpec(rgwConfig)
podTemplate := cephtest.NewPodTemplateSpecTester(t, &s)
podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion",
+13 -1
View File
@@ -19,8 +19,8 @@ package k8sutil
import (
"fmt"
"k8s.io/api/apps/v1"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/apps/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
@@ -63,3 +63,15 @@ func AddRookVersionLabelToDaemonSet(d *v1.DaemonSet) {
}
addRookVersionLabel(d.Labels)
}
// GetDaemonsets returns a list of daemonsets names labels matching a given selector
// example of a label selector might be "app=rook-ceph-mon, mon!=b"
// more: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
func GetDaemonsets(clientset kubernetes.Interface, namespace, labelSelector string) (*apps.DaemonSetList, error) {
listOptions := metav1.ListOptions{LabelSelector: labelSelector}
daemonsets, err := clientset.AppsV1().DaemonSets(namespace).List(listOptions)
if err != nil {
return nil, fmt.Errorf("failed to list deployments with labelSelector %s: %v", labelSelector, err)
}
return daemonsets, nil
}