Merge pull request #7331 from thotz/healthcheckwhensslenabled

ceph: fix healthcheck in case ssl enabled for rgw
This commit is contained in:
Travis Nielsen authored and GitHub committed 2021-04-20 08:32:17 -06:00
commit 56e7af3ee0
6 files changed
+52 -30

No files matched your search

+4
View File
@@ -30,6 +30,10 @@ func (s *ObjectStoreSpec) IsMultisite() bool {
return s.Zone.Name != ""
}
func (s *ObjectStoreSpec) IsTLSEnabled() bool {
return s.Gateway.SecurePort != 0 && s.Gateway.SSLCertificateRef != ""
}
func (s *ObjectStoreSpec) IsExternal() bool {
return len(s.Gateway.ExternalRgwEndpoints) != 0
}
@@ -73,7 +73,7 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje
return nil, errors.Wrap(err, "Provision: can't create ceph user")
}
s3svc, err := cephObject.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), true)
s3svc, err := cephObject.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), true, nil)
if err != nil {
p.deleteOBCResourceLogError("")
return nil, err
@@ -147,7 +147,7 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu
return nil, errors.Wrapf(err, "could not get user (user: %s)", stats.Owner)
}
s3svc, err := cephObject.NewS3Agent(*objectUser.AccessKey, *objectUser.SecretKey, p.getObjectStoreEndpoint(), true)
s3svc, err := cephObject.NewS3Agent(*objectUser.AccessKey, *objectUser.SecretKey, p.getObjectStoreEndpoint(), true, nil)
if err != nil {
p.deleteOBCResourceLogError("")
return nil, err
@@ -242,7 +242,7 @@ func (p Provisioner) Revoke(ob *bktv1alpha1.ObjectBucket) error {
return nil
}
s3svc, err := cephObject.NewS3Agent(*user.AccessKey, *user.SecretKey, p.getObjectStoreEndpoint(), true)
s3svc, err := cephObject.NewS3Agent(*user.AccessKey, *user.SecretKey, p.getObjectStoreEndpoint(), true, nil)
if err != nil {
return err
}
+10 -11
View File
@@ -20,7 +20,6 @@ import (
"context"
"fmt"
"reflect"
"strconv"
"strings"
"syscall"
"time"
@@ -339,7 +338,6 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
objContext := NewContext(r.context, r.clusterInfo, cephObjectStore.Name)
objContext.UID = string(cephObjectStore.UID)
var serviceIP string
var err error
if cephObjectStore.Spec.IsExternal() {
@@ -347,7 +345,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
// RECONCILE SERVICE
logger.Info("reconciling object store service")
serviceIP, err = cfg.reconcileService(cephObjectStore)
_, err = cfg.reconcileService(cephObjectStore)
if err != nil {
return r.setFailedStatus(namespacedName, "failed to reconcile service", err)
}
@@ -384,7 +382,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
// RECONCILE SERVICE
logger.Debug("reconciling object store service")
serviceIP, err = cfg.reconcileService(cephObjectStore)
serviceIP, err := cfg.reconcileService(cephObjectStore)
if err != nil {
return r.setFailedStatus(namespacedName, "failed to reconcile service", err)
}
@@ -414,7 +412,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
// Start monitoring
if !cephObjectStore.Spec.HealthCheck.Bucket.Disabled {
r.startMonitoring(cephObjectStore, objContext, serviceIP, namespacedName)
r.startMonitoring(cephObjectStore, objContext, namespacedName)
}
return reconcile.Result{}, nil
@@ -502,7 +500,7 @@ func (r *ReconcileCephObjectStore) verifyObjectBucketCleanup(objectstore *cephv1
return opcontroller.WaitForRequeueIfFinalizerBlocked, false
}
func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjectStore, objContext *Context, serviceIP string, namespacedName types.NamespacedName) {
func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjectStore, objContext *Context, namespacedName types.NamespacedName) {
// Start monitoring object store
if r.objectStoreChannels[objectstore.Name].monitoringRunning {
logger.Debug("external rgw endpoint monitoring go routine already running!")
@@ -512,18 +510,19 @@ func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjec
// Set the monitoring flag so we don't start more than one go routine
r.objectStoreChannels[objectstore.Name].monitoringRunning = true
var port string
var port int32
if objectstore.Spec.Gateway.SecurePort != 0 && objectstore.Spec.Gateway.SSLCertificateRef != "" {
port = strconv.Itoa(int(objectstore.Spec.Gateway.SecurePort))
if objectstore.Spec.IsTLSEnabled() {
port = objectstore.Spec.Gateway.SecurePort
} else if objectstore.Spec.Gateway.Port != 0 {
port = strconv.Itoa(int(objectstore.Spec.Gateway.Port))
port = objectstore.Spec.Gateway.Port
} else {
logger.Error("At least one of Port or SecurePort should be non-zero")
return
}
rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck)
rgwChecker := newBucketChecker(r.context, objContext, port, r.client, namespacedName, &objectstore.Spec)
logger.Info("starting rgw healthcheck")
go rgwChecker.checkObjectStore(r.objectStoreChannels[objectstore.Name].stopChan)
}
+15 -10
View File
@@ -17,6 +17,7 @@ limitations under the License.
package object
import (
"context"
"fmt"
"time"
@@ -24,6 +25,7 @@ import (
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -45,28 +47,26 @@ type bucketChecker struct {
context *clusterd.Context
objContext *Context
interval *time.Duration
serviceIP string
port string
port int32
client client.Client
namespacedName types.NamespacedName
healthCheckSpec *cephv1.BucketHealthCheckSpec
objectStoreSpec *cephv1.ObjectStoreSpec
}
// newbucketChecker creates a new HealthChecker object
func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, port string, client client.Client, namespacedName types.NamespacedName, healthCheckSpec *cephv1.BucketHealthCheckSpec) *bucketChecker {
func newBucketChecker(context *clusterd.Context, objContext *Context, port int32, client client.Client, namespacedName types.NamespacedName, objectStoreSpec *cephv1.ObjectStoreSpec) *bucketChecker {
c := &bucketChecker{
context: context,
objContext: objContext,
interval: &defaultHealthCheckInterval,
serviceIP: serviceIP,
port: port,
namespacedName: namespacedName,
client: client,
healthCheckSpec: healthCheckSpec,
objectStoreSpec: objectStoreSpec,
}
// allow overriding the check interval
checkInterval := healthCheckSpec.Bucket.Interval
checkInterval := objectStoreSpec.HealthCheck.Bucket.Interval
if checkInterval != nil {
logger.Infof("ceph rgw status check interval for object store %q is %q", namespacedName.Name, checkInterval.Duration.String())
c.interval = &checkInterval.Duration
@@ -121,10 +121,11 @@ func (c *bucketChecker) checkObjectStoreHealth() error {
Always keep the bucket and the user for the health check, just do PUT and GET because bucket creation is expensive
*/
ctx := context.TODO()
var s3AccessKey string
var s3SecretKey string
s3endpoint := fmt.Sprintf("%s:%s", BuildDomainName(c.objContext.Name, c.namespacedName.Namespace), c.port)
var sslCert []byte
s3endpoint := buildDNSEndpoint(BuildDomainName(c.objContext.Name, c.namespacedName.Namespace), c.port, c.objectStoreSpec.IsTLSEnabled())
// Generate unique user and bucket name
bucketName := genUniqueBucketName(c.objContext.UID)
@@ -147,9 +148,13 @@ func (c *bucketChecker) checkObjectStoreHealth() error {
s3AccessKey = *user.AccessKey
s3SecretKey = *user.SecretKey
if c.objectStoreSpec.IsTLSEnabled() {
sslSecretCert, _ := c.context.Clientset.CoreV1().Secrets(c.namespacedName.Namespace).Get(ctx, c.objectStoreSpec.Gateway.SSLCertificateRef, metav1.GetOptions{})
sslCert = sslSecretCert.Data[certKeyName]
}
// Initiate s3 agent
logger.Debugf("initializing s3 connection for object store %q", c.namespacedName.Name)
s3client, err := NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false)
s3client, err := NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false, sslCert)
if err != nil {
return errors.Wrap(err, "failed to initialize s3 connection")
}
+19 -5
View File
@@ -18,6 +18,8 @@ package object
import (
"bytes"
"crypto/tls"
"crypto/x509"
"net/http"
"strings"
"time"
@@ -35,13 +37,27 @@ type S3Agent struct {
Client *s3.S3
}
func NewS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, error) {
func NewS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byte) (*S3Agent, error) {
const cephRegion = "us-east-1"
logLevel := aws.LogOff
if debug {
logLevel = aws.LogDebug
}
client := http.Client{
Timeout: time.Second * 15,
}
tlsEnabled := false
if len(tlsCert) > 0 {
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(tlsCert)
tr := &http.Transport{
TLSClientConfig: &tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12},
}
client.Transport = tr
tlsEnabled = true
}
sess, err := session.NewSession(
aws.NewConfig().
WithRegion(cephRegion).
@@ -49,10 +65,8 @@ func NewS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, er
WithEndpoint(endpoint).
WithS3ForcePathStyle(true).
WithMaxRetries(5).
WithDisableSSL(true).
WithHTTPClient(&http.Client{
Timeout: time.Second * 15,
}).
WithDisableSSL(!tlsEnabled).
WithHTTPClient(&client).
WithLogLevel(logLevel),
)
if err != nil {
+1 -1
View File
@@ -147,7 +147,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil)
assert.Nil(s.T(), err)
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)