ceph: fix healthcheck incase ssl enabled for rgw
In case ssl enabled for RGW, bucket healthcheck won't work since access is denied from the server. In that case configure S3 agent using "InsecureSkipVerify: true" option. Fixes: 7288 Signed-off-by: Jiffin Tony Thottan <thottanjiffin@gmail.com>
This commit is contained in:
1 parent
76902a9775
commit
bcbe707bcb
6 files changed
+48
-23
No files matched your search
@@ -30,6 +30,10 @@ func (s *ObjectStoreSpec) IsMultisite() bool {
|
||||
return s.Zone.Name != ""
|
||||
}
|
||||
|
||||
func (s *ObjectStoreSpec) IsTLSEnabled() bool {
|
||||
return s.Gateway.SecurePort != 0 && s.Gateway.SSLCertificateRef != ""
|
||||
}
|
||||
|
||||
func (s *ObjectStoreSpec) IsExternal() bool {
|
||||
return len(s.Gateway.ExternalRgwEndpoints) != 0
|
||||
}
|
||||
|
||||
@@ -73,7 +73,7 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje
|
||||
return nil, errors.Wrap(err, "Provision: can't create ceph user")
|
||||
}
|
||||
|
||||
s3svc, err := cephObject.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), true)
|
||||
s3svc, err := cephObject.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), true, nil)
|
||||
if err != nil {
|
||||
p.deleteOBCResourceLogError("")
|
||||
return nil, err
|
||||
@@ -147,7 +147,7 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu
|
||||
return nil, errors.Wrapf(err, "could not get user (user: %s)", stats.Owner)
|
||||
}
|
||||
|
||||
s3svc, err := cephObject.NewS3Agent(*objectUser.AccessKey, *objectUser.SecretKey, p.getObjectStoreEndpoint(), true)
|
||||
s3svc, err := cephObject.NewS3Agent(*objectUser.AccessKey, *objectUser.SecretKey, p.getObjectStoreEndpoint(), true, nil)
|
||||
if err != nil {
|
||||
p.deleteOBCResourceLogError("")
|
||||
return nil, err
|
||||
@@ -242,7 +242,7 @@ func (p Provisioner) Revoke(ob *bktv1alpha1.ObjectBucket) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
s3svc, err := cephObject.NewS3Agent(*user.AccessKey, *user.SecretKey, p.getObjectStoreEndpoint(), true)
|
||||
s3svc, err := cephObject.NewS3Agent(*user.AccessKey, *user.SecretKey, p.getObjectStoreEndpoint(), true, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -512,18 +511,19 @@ func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjec
|
||||
// Set the monitoring flag so we don't start more than one go routine
|
||||
r.objectStoreChannels[objectstore.Name].monitoringRunning = true
|
||||
|
||||
var port string
|
||||
var port int32
|
||||
|
||||
if objectstore.Spec.Gateway.SecurePort != 0 && objectstore.Spec.Gateway.SSLCertificateRef != "" {
|
||||
port = strconv.Itoa(int(objectstore.Spec.Gateway.SecurePort))
|
||||
if objectstore.Spec.IsTLSEnabled() {
|
||||
port = objectstore.Spec.Gateway.SecurePort
|
||||
} else if objectstore.Spec.Gateway.Port != 0 {
|
||||
port = strconv.Itoa(int(objectstore.Spec.Gateway.Port))
|
||||
port = objectstore.Spec.Gateway.Port
|
||||
|
||||
} else {
|
||||
logger.Error("At least one of Port or SecurePort should be non-zero")
|
||||
return
|
||||
}
|
||||
|
||||
rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck)
|
||||
rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec)
|
||||
logger.Info("starting rgw healthcheck")
|
||||
go rgwChecker.checkObjectStore(r.objectStoreChannels[objectstore.Name].stopChan)
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
package object
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
@@ -24,6 +25,7 @@ import (
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
)
|
||||
@@ -46,14 +48,14 @@ type bucketChecker struct {
|
||||
objContext *Context
|
||||
interval *time.Duration
|
||||
serviceIP string
|
||||
port string
|
||||
port int32
|
||||
client client.Client
|
||||
namespacedName types.NamespacedName
|
||||
healthCheckSpec *cephv1.BucketHealthCheckSpec
|
||||
objectStoreSpec *cephv1.ObjectStoreSpec
|
||||
}
|
||||
|
||||
// newbucketChecker creates a new HealthChecker object
|
||||
func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, port string, client client.Client, namespacedName types.NamespacedName, healthCheckSpec *cephv1.BucketHealthCheckSpec) *bucketChecker {
|
||||
func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP string, port int32, client client.Client, namespacedName types.NamespacedName, objectStoreSpec *cephv1.ObjectStoreSpec) *bucketChecker {
|
||||
c := &bucketChecker{
|
||||
context: context,
|
||||
objContext: objContext,
|
||||
@@ -62,11 +64,11 @@ func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP,
|
||||
port: port,
|
||||
namespacedName: namespacedName,
|
||||
client: client,
|
||||
healthCheckSpec: healthCheckSpec,
|
||||
objectStoreSpec: objectStoreSpec,
|
||||
}
|
||||
|
||||
// allow overriding the check interval
|
||||
checkInterval := healthCheckSpec.Bucket.Interval
|
||||
checkInterval := objectStoreSpec.HealthCheck.Bucket.Interval
|
||||
if checkInterval != nil {
|
||||
logger.Infof("ceph rgw status check interval for object store %q is %q", namespacedName.Name, checkInterval.Duration.String())
|
||||
c.interval = &checkInterval.Duration
|
||||
@@ -121,10 +123,11 @@ func (c *bucketChecker) checkObjectStoreHealth() error {
|
||||
|
||||
Always keep the bucket and the user for the health check, just do PUT and GET because bucket creation is expensive
|
||||
*/
|
||||
|
||||
ctx := context.TODO()
|
||||
var s3AccessKey string
|
||||
var s3SecretKey string
|
||||
s3endpoint := fmt.Sprintf("%s:%s", BuildDomainName(c.objContext.Name, c.namespacedName.Namespace), c.port)
|
||||
var sslCert []byte
|
||||
s3endpoint := buildDNSEndpoint(BuildDomainName(c.objContext.Name, c.namespacedName.Namespace), c.port, c.objectStoreSpec.IsTLSEnabled())
|
||||
|
||||
// Generate unique user and bucket name
|
||||
bucketName := genUniqueBucketName(c.objContext.UID)
|
||||
@@ -147,9 +150,13 @@ func (c *bucketChecker) checkObjectStoreHealth() error {
|
||||
s3AccessKey = *user.AccessKey
|
||||
s3SecretKey = *user.SecretKey
|
||||
|
||||
if c.objectStoreSpec.IsTLSEnabled() {
|
||||
sslSecretCert, _ := c.context.Clientset.CoreV1().Secrets(c.namespacedName.Namespace).Get(ctx, c.objectStoreSpec.Gateway.SSLCertificateRef, metav1.GetOptions{})
|
||||
sslCert = sslSecretCert.Data[certKeyName]
|
||||
}
|
||||
// Initiate s3 agent
|
||||
logger.Debugf("initializing s3 connection for object store %q", c.namespacedName.Name)
|
||||
s3client, err := NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false)
|
||||
s3client, err := NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false, sslCert)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to initialize s3 connection")
|
||||
}
|
||||
|
||||
@@ -18,6 +18,8 @@ package object
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -35,13 +37,27 @@ type S3Agent struct {
|
||||
Client *s3.S3
|
||||
}
|
||||
|
||||
func NewS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, error) {
|
||||
func NewS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byte) (*S3Agent, error) {
|
||||
const cephRegion = "us-east-1"
|
||||
|
||||
logLevel := aws.LogOff
|
||||
if debug {
|
||||
logLevel = aws.LogDebug
|
||||
}
|
||||
client := http.Client{
|
||||
Timeout: time.Second * 15,
|
||||
}
|
||||
tlsEnabled := false
|
||||
if len(tlsCert) > 0 {
|
||||
caCertPool := x509.NewCertPool()
|
||||
caCertPool.AppendCertsFromPEM(tlsCert)
|
||||
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12},
|
||||
}
|
||||
client.Transport = tr
|
||||
tlsEnabled = true
|
||||
}
|
||||
sess, err := session.NewSession(
|
||||
aws.NewConfig().
|
||||
WithRegion(cephRegion).
|
||||
@@ -49,10 +65,8 @@ func NewS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, er
|
||||
WithEndpoint(endpoint).
|
||||
WithS3ForcePathStyle(true).
|
||||
WithMaxRetries(5).
|
||||
WithDisableSSL(true).
|
||||
WithHTTPClient(&http.Client{
|
||||
Timeout: time.Second * 15,
|
||||
}).
|
||||
WithDisableSSL(!tlsEnabled).
|
||||
WithHTTPClient(&client).
|
||||
WithLogLevel(logLevel),
|
||||
)
|
||||
if err != nil {
|
||||
|
||||
@@ -147,7 +147,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
|
||||
s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
|
||||
s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
|
||||
s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
|
||||
s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
|
||||
s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil)
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
|
||||
|
||||
|
||||
Reference in new issue
Block a user