diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md index 1e3df9890..66de3d99a 100644 --- a/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md @@ -54,6 +54,27 @@ spec: } ] } + bucketLifecycle: | + { + "Rules": [ + { + "ID": "AbortIncompleteMultipartUploads", + "Status": "Enabled", + "Prefix": "", + "AbortIncompleteMultipartUpload": { + "DaysAfterInitiation": 1 + } + }, + { + "ID": "ExpireAfter30Days", + "Status": "Enabled", + "Prefix": "", + "Expiration": { + "Days": 30 + } + } + ] + } ``` 1. `name` of the `ObjectBucketClaim`. This name becomes the name of the Secret and ConfigMap. @@ -72,6 +93,7 @@ If both `bucketName` and `generateBucketName` are blank or omitted then the stor * `bucketMaxObjects`: The maximum number of objects in the bucket as an individual bucket quota. This is useful when the bucket is shared among multiple users. * `bucketMaxSize`: The maximum size of the bucket as an individual bucket quota. * `bucketPolicy`: A raw JSON format string that defines an AWS S3 format the bucket policy. If set, the policy string will override any existing policy set on the bucket and any default bucket policy that the bucket provisioner potentially would have automatically generated. + * `bucketLifecycle`: A raw JSON format string that defines an AWS S3 format bucket lifecycle configuration. Note that the rules must be sorted by `ID` in order to be idempotent. ### OBC Custom Resource after Bucket Provisioning diff --git a/pkg/operator/ceph/object/bucket/provisioner.go b/pkg/operator/ceph/object/bucket/provisioner.go index d2d7cb9cc..9bd271b17 100644 --- a/pkg/operator/ceph/object/bucket/provisioner.go +++ b/pkg/operator/ceph/object/bucket/provisioner.go @@ -17,6 +17,7 @@ limitations under the License. package bucket import ( + "encoding/json" "fmt" "net/http" "strings" @@ -64,6 +65,7 @@ type additionalConfigSpec struct { bucketMaxObjects *int64 bucketMaxSize *int64 bucketPolicy *string + bucketLifecycle *string } var _ apibkt.Provisioner = &Provisioner{} @@ -587,6 +589,11 @@ func (p *Provisioner) setAdditionalSettings(additionalConfig *additionalConfigSp return errors.Wrap(err, "failed to set bucket policy") } + err = p.setBucketLifecycle(additionalConfig) + if err != nil { + return errors.Wrap(err, "failed to set bucket lifecycle") + } + return nil } @@ -741,6 +748,77 @@ func (p *Provisioner) setBucketPolicy(additionalConfig *additionalConfigSpec) er return nil } +func (p *Provisioner) setBucketLifecycle(additionalConfig *additionalConfigSpec) error { + svc := p.s3Agent.Client + var liveLc *s3.GetBucketLifecycleConfigurationOutput + + liveLc, err := svc.GetBucketLifecycleConfiguration(&s3.GetBucketLifecycleConfigurationInput{ + Bucket: &p.bucketName, + }) + if err != nil { + // when no lifecycle configuration is set, an err with a "code" of + // NoSuchLifecycleConfiguration is returned + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NoSuchLifecycleConfiguration" { + logger.Debugf("no lifecycle configuration set for bucket %q", p.bucketName) + } else { + return errors.Wrapf(err, "failed to fetch lifecycle configuration for bucket %q", p.bucketName) + } + } + + // parse the conf supplied lifecycle configuration json as aws-sdk-go will not operate on a + // json string + confLc := &s3.BucketLifecycleConfiguration{} + // don't try to parse the conf json if it is nil + if additionalConfig.bucketLifecycle != nil { + err = json.Unmarshal([]byte(*additionalConfig.bucketLifecycle), confLc) + if err != nil { + return errors.Wrapf(err, "failed to unmarshal lifecycle configuration for bucket %q", p.bucketName) + } + } + + // Comparing the conf json with a json serialization of the live rules is + // unreliable as aws-sdk-go does not tag fields with "ommitempty". The + // GetBucketLifecycleConfigurationOutput type has a String() method that + // returns a json like format which isn't valid json. The options are to + // write a custom json unmarshaler or compare go structs directly. + + // convert the api returned GetBucketLifecycleConfigurationOutput to a + // BucketLifecycleConfiguration for apples to apples comparison to determine + // sync state + diffLiveLc := &s3.BucketLifecycleConfiguration{Rules: liveLc.Rules} + + // note that the xml serialization of the rules returned by rgw sorts the + // rules by "ID". The rule ordering from the + // PutBucketLifecycleConfiguration() call is not preserved. + diff := cmp.Diff(diffLiveLc, confLc) + if diff == "" { + // policy is in sync + return nil + } + + logger.Debugf("Lifecycle configuration for bucket %q has changed. diff:%s", p.bucketName, diff) + if additionalConfig.bucketLifecycle == nil { + // if policy is out of sync and the new policy is nil, delete the live policy + _, err = svc.DeleteBucketLifecycle(&s3.DeleteBucketLifecycleInput{ + Bucket: &p.bucketName, + }) + if err != nil { + return errors.Wrapf(err, "failed to delete lifecycle configuration for bucket %q", p.bucketName) + } + } else { + // set the new policy + _, err = svc.PutBucketLifecycleConfiguration(&s3.PutBucketLifecycleConfigurationInput{ + Bucket: &p.bucketName, + LifecycleConfiguration: confLc, + }) + if err != nil { + return errors.Wrapf(err, "failed to set lifecycle configuration for bucket %q", p.bucketName) + } + } + + return nil +} + func (p *Provisioner) setTlsCaCert() error { objStore, err := p.getObjectStore() if err != nil { diff --git a/pkg/operator/ceph/object/bucket/provisioner_test.go b/pkg/operator/ceph/object/bucket/provisioner_test.go index e5a6ea478..23812f1ab 100644 --- a/pkg/operator/ceph/object/bucket/provisioner_test.go +++ b/pkg/operator/ceph/object/bucket/provisioner_test.go @@ -524,6 +524,18 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { assert.NoError(t, err) assert.Equal(t, additionalConfigSpec{bucketMaxSize: &(&struct{ i int64 }{5}).i}, *spec) }) + + t.Run("bucketPolicy field should be set", func(t *testing.T) { + spec, err := additionalConfigSpecFromMap(map[string]string{"bucketPolicy": "foo"}) + assert.NoError(t, err) + assert.Equal(t, additionalConfigSpec{bucketPolicy: &(&struct{ s string }{"foo"}).s}, *spec) + }) + + t.Run("bucketLifecycle field should be set", func(t *testing.T) { + spec, err := additionalConfigSpecFromMap(map[string]string{"bucketLifecycle": "foo"}) + assert.NoError(t, err) + assert.Equal(t, additionalConfigSpec{bucketLifecycle: &(&struct{ s string }{"foo"}).s}, *spec) + }) } func numberOfCallsWithValue(substr string, strs []string) int { diff --git a/pkg/operator/ceph/object/bucket/util.go b/pkg/operator/ceph/object/bucket/util.go index 22a0e04db..28a268e02 100644 --- a/pkg/operator/ceph/object/bucket/util.go +++ b/pkg/operator/ceph/object/bucket/util.go @@ -125,6 +125,11 @@ func additionalConfigSpecFromMap(config map[string]string) (*additionalConfigSpe spec.bucketPolicy = &policy } + if _, ok := config["bucketLifecycle"]; ok { + lifecycle := config["bucketLifecycle"] + spec.bucketLifecycle = &lifecycle + } + return &spec, nil }