feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

object: add httpClient param to object.NewS3Agent()

Browse Source 
To allow the caller to pass in their own transport when testing.

Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
(cherry picked from commit 57b7eeec80)
This commit is contained in:
Joshua Hoblitt authored and Mergify committed 2024-12-12 21:37:28 +00:00
1 parent 7ac36bcef6
commit 40e5026b26
7 files changed
+50 -59
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pkg/operator/ceph/object/bucket/provisioner.go
+2 -11
View File
@@ -735,15 +735,6 @@ func (p *Provisioner) setAdminOpsAPIClient() error {
func (p *Provisioner) setS3Agent() error {
var err error
if p.insecureTLS {
p.s3Agent, err = object.NewInsecureS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), logger.LevelAt(capnslog.DEBUG))
} else {
p.s3Agent, err = object.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), logger.LevelAt(capnslog.DEBUG), p.tlsCert)
}
if err != nil {
return err
}
return nil
p.s3Agent, err = object.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), logger.LevelAt(capnslog.DEBUG), p.tlsCert, p.insecureTLS, nil)
return err
}
pkg/operator/ceph/object/notification/provisioner.go
+1 -4
View File
@@ -91,10 +91,7 @@ func newS3Agent(p provisioner) (*object.S3Agent, error) {
}
}
if insecureTLS {
return object.NewInsecureS3Agent(accessKey, secretKey, objContext.Endpoint, logger.LevelAt(capnslog.DEBUG))
}
return object.NewS3Agent(accessKey, secretKey, objContext.Endpoint, logger.LevelAt(capnslog.DEBUG), tlsCert)
return object.NewS3Agent(accessKey, secretKey, objContext.Endpoint, logger.LevelAt(capnslog.DEBUG), tlsCert, insecureTLS, nil)
}
// TODO: convert all rules without restrictions once the AWS SDK supports that
pkg/operator/ceph/object/s3-handlers.go
+11 -14
View File
@@ -39,27 +39,24 @@ type S3Agent struct {
Client *s3.S3
}
func NewS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byte) (*S3Agent, error) {
return newS3Agent(accessKey, secretKey, endpoint, debug, tlsCert, false)
}
func NewInsecureS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, error) {
return newS3Agent(accessKey, secretKey, endpoint, debug, nil, true)
}
func newS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byte, insecure bool) (*S3Agent, error) {
func NewS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byte, insecure bool, httpClient *http.Client) (*S3Agent, error) {
logLevel := aws.LogOff
if debug {
logLevel = aws.LogDebug
}
client := http.Client{
Timeout: HttpTimeOut,
}
tlsEnabled := false
if len(tlsCert) > 0 || insecure {
tlsEnabled = true
client.Transport = BuildTransportTLS(tlsCert, insecure)
}
if httpClient == nil {
httpClient = &http.Client{
Timeout: HttpTimeOut,
}
if tlsEnabled {
httpClient.Transport = BuildTransportTLS(tlsCert, insecure)
}
}
session, err := awssession.NewSession(
aws.NewConfig().
WithRegion(CephRegion).
@@ -68,7 +65,7 @@ func newS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byt
WithS3ForcePathStyle(true).
WithMaxRetries(5).
WithDisableSSL(!tlsEnabled).
WithHTTPClient(&client).
WithHTTPClient(httpClient).
WithLogLevel(logLevel),
)
if err != nil {
pkg/operator/ceph/object/s3-handlers_test.go
+27 -5
View File
@@ -32,7 +32,7 @@ func TestNewS3Agent(t *testing.T) {
t.Run("test without tls/debug", func(t *testing.T) {
debug := false
insecure := false
s3Agent, err := newS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure)
s3Agent, err := NewS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure, nil)
assert.NoError(t, err)
assert.NotEqual(t, aws.LogDebug, s3Agent.Client.Config.LogLevel)
assert.Equal(t, nil, s3Agent.Client.Config.HTTPClient.Transport)
@@ -42,7 +42,7 @@ func TestNewS3Agent(t *testing.T) {
debug := true
logLevel := aws.LogDebug
insecure := false
s3Agent, err := newS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure)
s3Agent, err := NewS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure, nil)
assert.NoError(t, err)
assert.Equal(t, &logLevel, s3Agent.Client.Config.LogLevel)
assert.Nil(t, s3Agent.Client.Config.HTTPClient.Transport)
@@ -51,7 +51,7 @@ func TestNewS3Agent(t *testing.T) {
t.Run("test without tls client cert but insecure tls", func(t *testing.T) {
debug := true
insecure := true
s3Agent, err := newS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure)
s3Agent, err := NewS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure, nil)
assert.NoError(t, err)
assert.NotNil(t, s3Agent.Client.Config.HTTPClient.Transport.(*http.Transport).TLSClientConfig.RootCAs) // still includes sys certs
assert.True(t, s3Agent.Client.Config.HTTPClient.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify)
@@ -61,7 +61,7 @@ func TestNewS3Agent(t *testing.T) {
debug := true
insecure := false
tlsCert := []byte("tlsCert")
s3Agent, err := newS3Agent(accessKey, secretKey, endpoint, debug, tlsCert, insecure)
s3Agent, err := NewS3Agent(accessKey, secretKey, endpoint, debug, tlsCert, insecure, nil)
assert.NoError(t, err)
assert.NotNil(t, s3Agent.Client.Config.HTTPClient.Transport.(*http.Transport).TLSClientConfig.RootCAs)
assert.False(t, *s3Agent.Client.Config.DisableSSL)
@@ -70,10 +70,32 @@ func TestNewS3Agent(t *testing.T) {
debug := true
insecure := true
tlsCert := []byte("tlsCert")
s3Agent, err := newS3Agent(accessKey, secretKey, endpoint, debug, tlsCert, insecure)
s3Agent, err := NewS3Agent(accessKey, secretKey, endpoint, debug, tlsCert, insecure, nil)
assert.NoError(t, err)
assert.NotNil(t, s3Agent.Client.Config.HTTPClient.Transport)
assert.True(t, s3Agent.Client.Config.HTTPClient.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify)
assert.False(t, *s3Agent.Client.Config.DisableSSL)
})
t.Run("test with custom http.Client", func(t *testing.T) {
debug := true
logLevel := aws.LogDebug
insecure := false
httpClient := &http.Client{
Transport: &http.Transport{
// set some values to check if they are passed through
MaxIdleConns: 7,
MaxIdleConnsPerHost: 13,
MaxConnsPerHost: 17,
},
}
s3Agent, err := NewS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure, httpClient)
assert.NoError(t, err)
assert.Equal(t, &logLevel, s3Agent.Client.Config.LogLevel)
assert.NotNil(t, s3Agent.Client.Config.HTTPClient.Transport)
assert.True(t, *s3Agent.Client.Config.DisableSSL)
transport := s3Agent.Client.Config.HTTPClient.Transport
assert.Equal(t, 7, transport.(*http.Transport).MaxIdleConns)
assert.Equal(t, 13, transport.(*http.Transport).MaxIdleConnsPerHost)
assert.Equal(t, 17, transport.(*http.Transport).MaxConnsPerHost)
})
}
tests/framework/clients/bucket.go
+1 -5
View File
@@ -157,11 +157,7 @@ func (b *BucketOperation) CheckBucketNotificationSetonRGW(namespace, storeName,
s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
if tlsEnabled {
s3client, err = rgw.NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
} else {
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil)
}
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, tlsEnabled, nil)
if err != nil {
logger.Infof("failed to s3client due to %v", err)
return false
tests/integration/ceph_bucket_notification_test.go
+4 -10
View File
@@ -123,11 +123,8 @@ func testBucketNotifications(s *suite.Suite, helper *clients.TestClient, k8sh *u
s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
if objectStore.Spec.IsTLSEnabled() {
s3client, err = rgw.NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
} else {
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil)
}
insecure := objectStore.Spec.IsTLSEnabled()
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil)
assert.Nil(t, err)
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
@@ -227,11 +224,8 @@ func testBucketNotifications(s *suite.Suite, helper *clients.TestClient, k8sh *u
s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
if objectStore.Spec.IsTLSEnabled() {
s3client, err = rgw.NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
} else {
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil)
}
insecure := objectStore.Spec.IsTLSEnabled()
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil)
assert.Nil(t, err)
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
tests/integration/ceph_object_test.go
+4 -10
View File
@@ -218,11 +218,8 @@ func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh
s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
if objectStore.Spec.IsTLSEnabled() {
s3client, err = rgw.NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
} else {
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil)
}
insecure := objectStore.Spec.IsTLSEnabled()
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil)
assert.Nil(t, err)
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
@@ -334,11 +331,8 @@ func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh
s3AccessKey := string(secret.Data["AWS_ACCESS_KEY_ID"])
s3SecretKey := string(secret.Data["AWS_SECRET_ACCESS_KEY"])
if objectStore.Spec.IsTLSEnabled() {
s3client, err = rgw.NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
} else {
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil)
}
insecure := objectStore.Spec.IsTLSEnabled()
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil)
assert.Nil(t, err)
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
})