object: add obc bucketLifecycle

Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
(cherry picked from commit a55fdb3fbe)

Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md

@@ -54,6 +54,27 @@ spec:
               }
           ]
       }
+    bucketLifecycle: |
+      {
+        "Rules": [
+          {
+            "ID": "AbortIncompleteMultipartUploads",
+            "Status": "Enabled",
+            "Prefix": "",
+            "AbortIncompleteMultipartUpload": {
+              "DaysAfterInitiation": 1
+            }
+          },
+          {
+            "ID": "ExpireAfter30Days",
+            "Status": "Enabled",
+            "Prefix": "",
+            "Expiration": {
+              "Days": 30
+            }
+          }
+        ]
+      }
 ```
 
 1. `name` of the `ObjectBucketClaim`. This name becomes the name of the Secret and ConfigMap.
@@ -72,6 +93,7 @@ If both `bucketName` and `generateBucketName` are blank or omitted then the stor
     * `bucketMaxObjects`: The maximum number of objects in the bucket as an individual bucket quota. This is useful when the bucket is shared among multiple users.
     * `bucketMaxSize`: The maximum size of the bucket as an individual bucket quota.
     * `bucketPolicy`: A raw JSON format string that defines an AWS S3 format the bucket policy. If set, the policy string will override any existing policy set on the bucket and any default bucket policy that the bucket provisioner potentially would have automatically generated.
+    * `bucketLifecycle`: A raw JSON format string that defines an AWS S3 format bucket lifecycle configuration. Note that the rules must be sorted by `ID` in order to be idempotent.
 
 ### OBC Custom Resource after Bucket Provisioning
 

pkg/operator/ceph/object/bucket/provisioner.go

@@ -17,6 +17,7 @@ limitations under the License.
 package bucket
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"strings"
@@ -64,6 +65,7 @@ type additionalConfigSpec struct {
 	bucketMaxObjects *int64
 	bucketMaxSize    *int64
 	bucketPolicy     *string
+	bucketLifecycle  *string
 }
 
 var _ apibkt.Provisioner = &Provisioner{}
@@ -587,6 +589,11 @@ func (p *Provisioner) setAdditionalSettings(additionalConfig *additionalConfigSp
 		return errors.Wrap(err, "failed to set bucket policy")
 	}
 
+	err = p.setBucketLifecycle(additionalConfig)
+	if err != nil {
+		return errors.Wrap(err, "failed to set bucket lifecycle")
+	}
+
 	return nil
 }
 
@@ -741,6 +748,77 @@ func (p *Provisioner) setBucketPolicy(additionalConfig *additionalConfigSpec) er
 	return nil
 }
 
+func (p *Provisioner) setBucketLifecycle(additionalConfig *additionalConfigSpec) error {
+	svc := p.s3Agent.Client
+	var liveLc *s3.GetBucketLifecycleConfigurationOutput
+
+	liveLc, err := svc.GetBucketLifecycleConfiguration(&s3.GetBucketLifecycleConfigurationInput{
+		Bucket: &p.bucketName,
+	})
+	if err != nil {
+		// when no lifecycle configuration is set, an err with a "code" of
+		// NoSuchLifecycleConfiguration is returned
+		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NoSuchLifecycleConfiguration" {
+			logger.Debugf("no lifecycle configuration set for bucket %q", p.bucketName)
+		} else {
+			return errors.Wrapf(err, "failed to fetch lifecycle configuration for bucket %q", p.bucketName)
+		}
+	}
+
+	// parse the conf supplied lifecycle configuration json as aws-sdk-go will not operate on a
+	// json string
+	confLc := &s3.BucketLifecycleConfiguration{}
+	// don't try to parse the conf json if it is nil
+	if additionalConfig.bucketLifecycle != nil {
+		err = json.Unmarshal([]byte(*additionalConfig.bucketLifecycle), confLc)
+		if err != nil {
+			return errors.Wrapf(err, "failed to unmarshal lifecycle configuration for bucket %q", p.bucketName)
+		}
+	}
+
+	// Comparing the conf json with a json serialization of the live rules is
+	// unreliable as aws-sdk-go does not tag fields with "ommitempty". The
+	// GetBucketLifecycleConfigurationOutput type has a String() method that
+	// returns a json like format which isn't valid json.  The options are to
+	// write a custom json unmarshaler or compare go structs directly.
+
+	// convert the api returned GetBucketLifecycleConfigurationOutput to a
+	// BucketLifecycleConfiguration for apples to apples comparison to determine
+	// sync state
+	diffLiveLc := &s3.BucketLifecycleConfiguration{Rules: liveLc.Rules}
+
+	// note that the xml serialization of the rules returned by rgw sorts the
+	// rules by "ID".  The rule ordering from the
+	// PutBucketLifecycleConfiguration() call is not preserved.
+	diff := cmp.Diff(diffLiveLc, confLc)
+	if diff == "" {
+		// policy is in sync
+		return nil
+	}
+
+	logger.Debugf("Lifecycle configuration for bucket %q has changed. diff:%s", p.bucketName, diff)
+	if additionalConfig.bucketLifecycle == nil {
+		// if policy is out of sync and the new policy is nil, delete the live policy
+		_, err = svc.DeleteBucketLifecycle(&s3.DeleteBucketLifecycleInput{
+			Bucket: &p.bucketName,
+		})
+		if err != nil {
+			return errors.Wrapf(err, "failed to delete lifecycle configuration for bucket %q", p.bucketName)
+		}
+	} else {
+		// set the new policy
+		_, err = svc.PutBucketLifecycleConfiguration(&s3.PutBucketLifecycleConfigurationInput{
+			Bucket:                 &p.bucketName,
+			LifecycleConfiguration: confLc,
+		})
+		if err != nil {
+			return errors.Wrapf(err, "failed to set lifecycle configuration for bucket %q", p.bucketName)
+		}
+	}
+
+	return nil
+}
+
 func (p *Provisioner) setTlsCaCert() error {
 	objStore, err := p.getObjectStore()
 	if err != nil {

pkg/operator/ceph/object/bucket/provisioner_test.go

@@ -524,6 +524,18 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) {
 		assert.NoError(t, err)
 		assert.Equal(t, additionalConfigSpec{bucketMaxSize: &(&struct{ i int64 }{5}).i}, *spec)
 	})
+
+	t.Run("bucketPolicy field should be set", func(t *testing.T) {
+		spec, err := additionalConfigSpecFromMap(map[string]string{"bucketPolicy": "foo"})
+		assert.NoError(t, err)
+		assert.Equal(t, additionalConfigSpec{bucketPolicy: &(&struct{ s string }{"foo"}).s}, *spec)
+	})
+
+	t.Run("bucketLifecycle field should be set", func(t *testing.T) {
+		spec, err := additionalConfigSpecFromMap(map[string]string{"bucketLifecycle": "foo"})
+		assert.NoError(t, err)
+		assert.Equal(t, additionalConfigSpec{bucketLifecycle: &(&struct{ s string }{"foo"}).s}, *spec)
+	})
 }
 
 func numberOfCallsWithValue(substr string, strs []string) int {

pkg/operator/ceph/object/bucket/util.go

@@ -125,6 +125,11 @@ func additionalConfigSpecFromMap(config map[string]string) (*additionalConfigSpe
 		spec.bucketPolicy = &policy
 	}
 
+	if _, ok := config["bucketLifecycle"]; ok {
+		lifecycle := config["bucketLifecycle"]
+		spec.bucketLifecycle = &lifecycle
+	}
+
 	return &spec, nil
 }