From a30966b6e4138474e1eca0d00a05c8de90abdb55 Mon Sep 17 00:00:00 2001
From: Dmitry Yusupov <dmitry.yusupov@nexenta.com>
Date: Wed, 20 Feb 2019 12:18:17 -0800
Subject: [PATCH] EdgeFS: support for S3 buckets as DNS subdomain style

Signed-off-by: Dmitry Yusupov <dmitry.yusupov@nexenta.com>
---
 Documentation/edgefs-iscsi-crd.md             | 24 +++++-----
 Documentation/edgefs-isgw-crd.md              | 32 +++++++-------
 Documentation/edgefs-nfs-crd.md               | 29 ++++++------
 Documentation/edgefs-s3-crd.md                | 39 +++++++++-------
 Documentation/edgefs-s3x-crd.md               | 24 +++++-----
 .../examples/kubernetes/edgefs/cluster.yaml   |  2 +-
 cluster/examples/kubernetes/edgefs/s3.yaml    |  3 ++
 pkg/apis/edgefs.rook.io/v1alpha1/types.go     | 10 +++--
 .../edgefs/cluster/target/config/config.go    |  2 +-
 pkg/operator/edgefs/cluster/target/pod.go     | 29 +++++++++++-
 .../edgefs/cluster/target/pod_test.go         |  2 +-
 pkg/operator/edgefs/s3/s3.go                  | 44 ++++++++++++++++---
 12 files changed, 155 insertions(+), 85 deletions(-)

diff --git a/Documentation/edgefs-iscsi-crd.md b/Documentation/edgefs-iscsi-crd.md
index 14bf536a4..ae2842389 100644
--- a/Documentation/edgefs-iscsi-crd.md
+++ b/Documentation/edgefs-iscsi-crd.md
@@ -72,54 +72,54 @@ Root object holds system information and table of namespaces registered to a loc
 
 To initialize system and prepare logical definitions, login to the toolbox as shown in this example:
 
-<pre>
+```
 kubectl get po --all-namespaces | grep edgefs-mgr
 kubectl exec -it -n rook-edgefs rook-edgefs-mgr-6cb9598469-czr7p -- env COLUMNS=$COLUMNS LINES=$LINES TERM=linux toolbox
-</pre>
+```
 
 Assumption at this point is that nodes are all configured and can be seen via the following command:
 
-<pre>
+```
 efscli system status
-</pre>
+```
 
 1. Initialize cluster
 
 Verify that HW (or better say emulated in this case) configuration look normal and accept it
 
-<pre>
+```
 efscli system init
-</pre>
+```
 
 At this point new dynamically discovered configuration checkpoint will be created at $NEDGE_HOME/var/run/flexhash-checkpoint.json
 This will also create system "root" object, holding Site's Namespace. Namespace may consist of more then single region.
 
 2. Create new local namespace (or we also call it "Region" or "Segment")
 
-<pre>
+```
 efscli cluster create Hawaii
-</pre>
+```
 
 3. Create logical tenants of cluster namespace "Hawaii", also buckets if needed
 
-<pre>
+```
 efscli tenant create Hawaii/Cola
 efscli bucket create Hawaii/Cola/bk1
 efscli tenant create Hawaii/Pepsi
 efscli bucket create Hawaii/Pepsi/bk1
-</pre>
+```
 
 Now cluster is setup, services can be now created.
 
 4. Create iSCSI Target services objects for tenants
 
-<pre>
+```
 efscli service create iscsi iscCola
 efscli service serve iscCola Hawaii/Cola/bk1/lun1 X-volsize=10G,ccow-chunkmap-chunk-size=16384
 efscli service serve iscCola Hawaii/Cola/bk1/lun2 X-volsize=20G,ccow-chunkmap-chunk-size=131072
 efscli service create iscsi iscPepsi
 efscli service serve iscPepsi Hawaii/Pepsi/bk1/lun1 X-volsize=20G
-</pre>
+```
 
 5. Create ISCSI CRDs
 
diff --git a/Documentation/edgefs-isgw-crd.md b/Documentation/edgefs-isgw-crd.md
index 16e35a42f..fada3d712 100644
--- a/Documentation/edgefs-isgw-crd.md
+++ b/Documentation/edgefs-isgw-crd.md
@@ -108,50 +108,50 @@ Root object holds system information and table of namespaces registered to a loc
 
 For each cluster, to initialize system and prepare logical definitions, login to the toolbox as shown in this example:
 
-<pre>
+```
 kubectl get po --all-namespaces | grep edgefs-mgr
 kubectl exec -it -n rook-edgefs rook-edgefs-mgr-6cb9598469-czr7p -- env COLUMNS=$COLUMNS LINES=$LINES TERM=linux toolbox
-</pre>
+```
 
 Assumption at this point is that nodes are all configured and can be seen via the following command:
 
-<pre>
+```
 efscli system status
-</pre>
+```
 
 1. Initialize cluster
 
 Verify that HW (or better say emulated in this case) configuration look normal and accept it
 
-<pre>
+```
 efscli system init
-</pre>
+```
 
 At this point new dynamically discovered configuration checkpoint will be created at $NEDGE_HOME/var/run/flexhash-checkpoint.json
 This will also create system "root" object, holding Site's Namespace. Namespace may consist of more then single region.
 
 2. Create new local namespace (or we also call it "Region" or "Segment")
 
-<pre>
+```
 efscli cluster create Hawaii
-</pre>
+```
 
 3. Create logical tenants of cluster namespace "Hawaii", also buckets if needed
 
-<pre>
+```
 efscli tenant create Hawaii/Cola
 efscli bucket create Hawaii/Cola/bk1
 efscli tenant create Hawaii/Pepsi
 efscli bucket create Hawaii/Pepsi/bk1
-</pre>
+```
 
 Now that Hawaii serving namespace cluster is setup, repeat steps on the secondary site:
 
-<pre>
+```
 efscli cluster create Hawaii
 efscli tenant create Hawaii/Cola -r 2
 efscli tenant create Hawaii/Pepsi -r 2
-</pre>
+```
 
 This steps also includes a possibility to override destination defaults on per-tenant (Cola and Pepsi) basis.
 In the example above we setting replication count to 2 on the secondary site.
@@ -163,11 +163,11 @@ Link status change will be noticed automatically and syncing can be delayed for
 
 On the primary site:
 
-<pre>
+```
 efscli service create isgw hawaii
 efscli service serve hawaii Hawaii/Cola/bk1
 efscli service serve hawaii Hawaii/Pepsi/bk1
-</pre>
+```
 
 Because we creating bi-directional link, on the secondary site we would need to create exactly same configuration except X-ISGW-Remote (remoteURL in CRD) where it should point back to primary. Synchronization is enabled on per-bucket level. Buckets which do not yet exist will be automatically created with tenant parameter overrides taken into an account.
 
@@ -205,6 +205,6 @@ EdgeFS UI Dashboard automates the process of CRD composing but also has a capabi
 
 At this point two ISGW link services should be operational. Verify its operation with `show` command:
 
-<pre>
+```
 efscli service show hawaii -s
-</pre>
+```
diff --git a/Documentation/edgefs-nfs-crd.md b/Documentation/edgefs-nfs-crd.md
index c86dadf8a..ba9e48572 100644
--- a/Documentation/edgefs-nfs-crd.md
+++ b/Documentation/edgefs-nfs-crd.md
@@ -66,53 +66,53 @@ Root object holds system information and table of namespaces registered to a loc
 
 To initialize system and prepare logical definitions, login to the toolbox as shown in this example:
 
-<pre>
+```
 kubectl get po --all-namespaces | grep edgefs-mgr
 kubectl exec -it -n rook-edgefs rook-edgefs-mgr-6cb9598469-czr7p -- env COLUMNS=$COLUMNS LINES=$LINES TERM=linux toolbox
-</pre>
+```
 
 Assumption at this point is that nodes are all configured and can be seen via the following command:
 
-<pre>
+```
 efscli system status
-</pre>
+```
 
 1. Initialize cluster
 
 Verify that HW (or better say emulated in this case) configuration look normal and accept it
 
-<pre>
+```
 efscli system init
-</pre>
+```
 
 At this point new dynamically discovered configuration checkpoint will be created at $NEDGE_HOME/var/run/flexhash-checkpoint.json
 This will also create system "root" object, holding Site's Namespace. Namespace may consist of more then single region.
 
 2. Create new local namespace (or we also call it "Region" or "Segment")
 
-<pre>
+```
 efscli cluster create Hawaii
-</pre>
+```
 
 3. Create logical tenants of cluster namespace "Hawaii", also buckets if needed
 
-<pre>
+```
 efscli tenant create Hawaii/Cola
 efscli bucket create Hawaii/Cola/bk1
 efscli tenant create Hawaii/Pepsi
 efscli bucket create Hawaii/Pepsi/bk1
-</pre>
+```
 
 Now cluster is setup, services can be now created and attached to CSI provisioner.
 
 4. Create NFS service objects for tenants
 
-<pre>
+```
 efscli service create nfs nfsCola
 efscli service serve nfsCola Hawaii/Cola/bk1
 efscli service create nfs nfsPepsi
 efscli service serve nfsPepsi Hawaii/Pepsi/bk1
-</pre>
+```
 
 5. Create NFS CRDs
 
@@ -138,8 +138,7 @@ spec:
 
 At this point two NFS services should be available. Verify that showmount command can see service (substitute CLUSTERIP with corresponding entry from `kubectl get svc` command):
 
-<pre>
+```
 kubectl get svc --all-namespaces
 showmount -e CLUSTERIP
-</pre>
-</pre>
+```
diff --git a/Documentation/edgefs-s3-crd.md b/Documentation/edgefs-s3-crd.md
index 28db46514..29f1e67a4 100644
--- a/Documentation/edgefs-s3-crd.md
+++ b/Documentation/edgefs-s3-crd.md
@@ -19,6 +19,7 @@ metadata:
   namespace: rook-edgefs
 spec:
   instances: 3
+  #s3type: s3s
   placement:
   #  nodeAffinity:
   #    requiredDuringSchedulingIgnoredDuringExecution:
@@ -46,9 +47,10 @@ spec:
 
 - `name`: The name of the S3 system to create, which must match existing EdgeFS service.
 - `namespace`: The namespace of the Rook cluster where the S3 service is created.
+- `s3type`: The type of S3 service to be created. It can be one of the following: `s3` (default, path style) or `s3s` (buckets as DNS style)
 - `sslCertificateRef`: If the certificate is not specified, SSL will use default crt and key files. If specified, this is the name of the Kubernetes secret that contains the SSL certificate to be used for secure connections. Please see [secret YAML file example](/cluster/examples/kubernetes/edgefs/sslKeyCertificate.yaml) on how to setup Kuberenetes secret. Notice that base64 encoding is required.
-- `port`: The port on which the S3 pods and the S3 service will be listening (not encrypted). Default port is 9982.
-- `securePort`: The secure port on which S3 pods will be listening. If not defined then default SSL certificates will be used. Default port is 8443.
+- `port`: The port on which the S3 pods and the S3 service will be listening (not encrypted). Default port is 9982 for `s3` and 9983 for `s3s`.
+- `securePort`: The secure port on which S3 pods will be listening. If not defined then default SSL certificates will be used. Default port is 8443 for `s3` and 8444 for `s3s`.
 - `instances`: The number of active S3 service instances. For load balancing we recommend to use nginx and the like solutions.
 - `placement`: The S3 pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](/cluster/examples/kubernetes/edgefs/cluster.yaml).
 - `resources`: Set resource requests/limits for the S3 pods, see [Resource Requirements/Limits](edgefs-cluster-crd.md#resource-requirementslimits).
@@ -69,55 +71,62 @@ Root object holds system information and table of namespaces registered to a loc
 
 To initialize system and prepare logical definitions, login to the toolbox as shown in this example:
 
-<pre>
+```
 kubectl get po --all-namespaces | grep edgefs-mgr
 kubectl exec -it -n rook-edgefs rook-edgefs-mgr-6cb9598469-czr7p -- env COLUMNS=$COLUMNS LINES=$LINES TERM=linux toolbox
-</pre>
+```
 
 Assumption at this point is that nodes are all configured and can be seen via the following command:
 
-<pre>
+```
 efscli system status
-</pre>
+```
 
 1. Initialize cluster
 
 Verify that HW (or better say emulated in this case) configuration look normal and accept it
 
-<pre>
+```
 efscli system init
-</pre>
+```
 
 At this point new dynamically discovered configuration checkpoint will be created at $NEDGE_HOME/var/run/flexhash-checkpoint.json
 This will also create system "root" object, holding Site's Namespace. Namespace may consist of more then single region.
 
 2. Create new local namespace (or we also call it "Region" or "Segment")
 
-<pre>
+```
 efscli cluster create Hawaii
-</pre>
+```
 
 3. Create logical tenants of cluster namespace "Hawaii", also buckets if needed
 
-<pre>
+```
 efscli tenant create Hawaii/Cola
 efscli bucket create Hawaii/Cola/bk1
 efscli tenant create Hawaii/Pepsi
 efscli bucket create Hawaii/Pepsi/bk1
-</pre>
+```
 
 Now cluster is setup, services can be now created.
 
 4. Create S3 services objects for tenants
 
-<pre>
+```
 efscli service create s3 s3Cola
 efscli service serve s3Cola Hawaii/Cola
 efscli service create s3 s3Pepsi
 efscli service serve s3Pepsi Hawaii/Pepsi/bk1
-</pre>
+```
 
-5. Create S3X CRDs
+In case of s3type set to `s3`, do not forget to configure default domain name:
+
+```
+efscli service config s3Cola X-Domain cola.com
+efscli service config s3Pepsi X-Domain pepsi.com
+```
+
+5. Create S3 CRDs
 
 ```yaml
 apiVersion: edgefs.rook.io/v1alpha1
diff --git a/Documentation/edgefs-s3x-crd.md b/Documentation/edgefs-s3x-crd.md
index ecc839d24..71f1fc0a2 100644
--- a/Documentation/edgefs-s3x-crd.md
+++ b/Documentation/edgefs-s3x-crd.md
@@ -73,53 +73,53 @@ Root object holds system information and table of namespaces registered to a loc
 
 To initialize system and prepare logical definitions, login to the toolbox as shown in this example:
 
-<pre>
+```
 kubectl get po --all-namespaces | grep edgefs-mgr
 kubectl exec -it -n rook-edgefs rook-edgefs-mgr-6cb9598469-czr7p -- env COLUMNS=$COLUMNS LINES=$LINES TERM=linux toolbox
-</pre>
+```
 
 Assumption at this point is that nodes are all configured and can be seen via the following command:
 
-<pre>
+```
 efscli system status
-</pre>
+```
 
 1. Initialize cluster
 
 Verify that HW (or better say emulated in this case) configuration look normal and accept it
 
-<pre>
+```
 efscli system init
-</pre>
+```
 
 At this point new dynamically discovered configuration checkpoint will be created at $NEDGE_HOME/var/run/flexhash-checkpoint.json
 This will also create system "root" object, holding Site's Namespace. Namespace may consist of more then single region.
 
 2. Create new local namespace (or we also call it "Region" or "Segment")
 
-<pre>
+```
 efscli cluster create Hawaii
-</pre>
+```
 
 3. Create logical tenants of cluster namespace "Hawaii", also buckets if needed
 
-<pre>
+```
 efscli tenant create Hawaii/Cola
 efscli bucket create Hawaii/Cola/bk1
 efscli tenant create Hawaii/Pepsi
 efscli bucket create Hawaii/Pepsi/bk1
-</pre>
+```
 
 Now cluster is setup, services can be now created.
 
 4. Create Edge-X S3 services objects for tenants
 
-<pre>
+```
 efscli service create s3x s3xCola
 efscli service serve s3xCola Hawaii/Cola
 efscli service create s3x s3xPepsi
 efscli service serve s3xPepsi Hawaii/Pepsi/bk1
-</pre>
+```
 
 5. Create S3X CRDs
 
diff --git a/cluster/examples/kubernetes/edgefs/cluster.yaml b/cluster/examples/kubernetes/edgefs/cluster.yaml
index c70f79eda..0a3f10177 100644
--- a/cluster/examples/kubernetes/edgefs/cluster.yaml
+++ b/cluster/examples/kubernetes/edgefs/cluster.yaml
@@ -55,7 +55,7 @@ subjects:
   name: rook-edgefs-cluster
   namespace: rook-edgefs
 ---  
-apiVersion: apps/v1
+apiVersion: policy/v1beta1
 kind: PodSecurityPolicy
 metadata:
   name: privileged
diff --git a/cluster/examples/kubernetes/edgefs/s3.yaml b/cluster/examples/kubernetes/edgefs/s3.yaml
index 93f03b5f3..797527cb7 100644
--- a/cluster/examples/kubernetes/edgefs/s3.yaml
+++ b/cluster/examples/kubernetes/edgefs/s3.yaml
@@ -5,6 +5,9 @@ metadata:
   namespace: rook-edgefs
 spec:
   instances: 1
+  #s3type: s3s                    # s3 (default) for path style, s3s for buckets as DNS style
+  #port: 4500                     # HTTP port (default is 9982 or 9983 for s3s)
+  #securePort: 4501               # HTTPS port (default is 8443 or 8444 for s3s)
   #sslCertificateRef: ssl-secret
   #placement:
   #  nodeAffinity:
diff --git a/pkg/apis/edgefs.rook.io/v1alpha1/types.go b/pkg/apis/edgefs.rook.io/v1alpha1/types.go
index 69aad2207..cdf3e7c82 100644
--- a/pkg/apis/edgefs.rook.io/v1alpha1/types.go
+++ b/pkg/apis/edgefs.rook.io/v1alpha1/types.go
@@ -142,12 +142,14 @@ type S3Spec struct {
 	Resources v1.ResourceRequirements `json:"resources,omitempty"`
 	// The number of pods in the S3 replicaset
 	Instances int32 `json:"instances"`
-	//S3X Http port (default value 3000)
+	//S3 Http port (default value 9982)
 	Port uint `json:"port,omitempty"`
-	//S3X Https port (default value 3001)
+	//S3 Https port (default value 9443)
 	SecurePort uint `json:"securePort,omitempty"`
 	// The name of the secret that stores the ssl certificate for secure s3 connections
-	SSLCertificateRef string `json:"sslCertificateRef,,omitempty"`
+	SSLCertificateRef string `json:"sslCertificateRef,omitempty"`
+	// S3 type: s3 (bucket as url, default), s3s (bucket as DNS subdomain), s3g (new, experimental)
+	S3Type string `json:"s3type,omitempty"`
 }
 
 // +genclient
@@ -181,7 +183,7 @@ type S3XSpec struct {
 	//S3X Https port (default value 3001)
 	SecurePort uint `json:"securePort,omitempty"`
 	// The name of the secret that stores the ssl certificate for secure s3x connections
-	SSLCertificateRef string `json:"sslCertificateRef,,omitempty"`
+	SSLCertificateRef string `json:"sslCertificateRef,omitempty"`
 }
 
 // +genclient
diff --git a/pkg/operator/edgefs/cluster/target/config/config.go b/pkg/operator/edgefs/cluster/target/config/config.go
index 8b1f86556..9b493e910 100644
--- a/pkg/operator/edgefs/cluster/target/config/config.go
+++ b/pkg/operator/edgefs/cluster/target/config/config.go
@@ -71,7 +71,7 @@ type StoreConfig struct {
 func DefaultStoreConfig() StoreConfig {
 	return StoreConfig{
 		RtVerifyChid:       1,
-		LmdbPageSize:       16348,
+		LmdbPageSize:       16384,
 		UseBCache:          false,
 		UseBCacheWB:        false,
 		UseMetadataMask:    "0xff",
diff --git a/pkg/operator/edgefs/cluster/target/pod.go b/pkg/operator/edgefs/cluster/target/pod.go
index 72591b6e7..13b9baa4c 100644
--- a/pkg/operator/edgefs/cluster/target/pod.go
+++ b/pkg/operator/edgefs/cluster/target/pod.go
@@ -169,6 +169,17 @@ func (c *Cluster) makeDaemonContainer(containerImage string, dro edgefsv1alpha1.
 		{
 			Name:      "devices",
 			MountPath: "/dev",
+			ReadOnly:  false,
+		},
+		{
+			Name:      "sys",
+			MountPath: "/sys",
+			ReadOnly:  true,
+		},
+		{
+			Name:      "udev",
+			MountPath: "/run/udev",
+			ReadOnly:  true,
 		},
 		{
 			Name:      dataVolumeName,
@@ -268,6 +279,22 @@ func (c *Cluster) createPodSpec(rookImage string, dro edgefsv1alpha1.DevicesResu
 				},
 			},
 		},
+		{
+			Name: "sys",
+			VolumeSource: v1.VolumeSource{
+				HostPath: &v1.HostPathVolumeSource{
+					Path: "/sys",
+				},
+			},
+		},
+		{
+			Name: "udev",
+			VolumeSource: v1.VolumeSource{
+				HostPath: &v1.HostPathVolumeSource{
+					Path: "/run/udev",
+				},
+			},
+		},
 	}
 
 	hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
@@ -323,9 +350,9 @@ func (c *Cluster) createPodSpec(rookImage string, dro edgefsv1alpha1.DevicesResu
 	volumes = append(volumes, c.configOverrideVolume())
 
 	containers = []v1.Container{
+		c.makeDaemonContainer(rookImage, dro, false),
 		c.makeCorosyncContainer(rookImage),
 		c.makeAuditdContainer(rookImage),
-		c.makeDaemonContainer(rookImage, dro, false),
 	}
 
 	return v1.PodSpec{
diff --git a/pkg/operator/edgefs/cluster/target/pod_test.go b/pkg/operator/edgefs/cluster/target/pod_test.go
index a981f45cf..3b92156e1 100644
--- a/pkg/operator/edgefs/cluster/target/pod_test.go
+++ b/pkg/operator/edgefs/cluster/target/pod_test.go
@@ -82,7 +82,7 @@ func TestStorageSpecConfig(t *testing.T) {
 
 	//check default config options
 	assert.Equal(t, storeConfig.RtVerifyChid, 1)
-	assert.Equal(t, storeConfig.LmdbPageSize, 16348)
+	assert.Equal(t, storeConfig.LmdbPageSize, 16384)
 	assert.Equal(t, storeConfig.UseMetadataMask, "0xff")
 	assert.Equal(t, storeConfig.RtrdPLevelOverride, 0)
 	assert.Equal(t, storeConfig.Sync, 1)
diff --git a/pkg/operator/edgefs/s3/s3.go b/pkg/operator/edgefs/s3/s3.go
index 5ffdb1d5f..20d950228 100644
--- a/pkg/operator/edgefs/s3/s3.go
+++ b/pkg/operator/edgefs/s3/s3.go
@@ -19,6 +19,7 @@ package s3
 
 import (
 	"fmt"
+	"strings"
 
 	edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
 	"github.com/rook/rook/pkg/clusterd"
@@ -33,6 +34,7 @@ const (
 	appName = "rook-edgefs-s3"
 
 	/* Volumes definitions */
+	defaultS3Image    = "edgefs/edgefs-restapi"
 	sslCertVolumeName = "ssl-cert-volume"
 	sslMountPath      = "/opt/nedge/etc/ssl/"
 	dataVolumeName    = "edgefs-datadir"
@@ -53,9 +55,8 @@ func (c *S3Controller) UpdateService(s edgefsv1alpha1.S3, ownerRefs []metav1.Own
 
 // Start the s3 instance
 func (c *S3Controller) CreateOrUpdate(s edgefsv1alpha1.S3, update bool, ownerRefs []metav1.OwnerReference) error {
-	logger.Infof("starting update=%v service=%s", update, s.Name)
+	logger.Debugf("starting update=%v service=%s", update, s.Name)
 
-	logger.Infof("S3 Base image is %s", c.rookImage)
 	// validate S3 service settings
 	if err := validateService(c.context, s); err != nil {
 		return fmt.Errorf("invalid S3 service %s arguments. %+v", s.Name, err)
@@ -70,6 +71,31 @@ func (c *S3Controller) CreateOrUpdate(s edgefsv1alpha1.S3, update bool, ownerRef
 		s.Spec.SecurePort = defaultSecurePort
 	}
 
+	imageArgs := "s3"
+	var rookImage string
+	if s.Spec.S3Type == "" {
+		rookImage = defaultS3Image
+	} else if s.Spec.S3Type == "s3" || s.Spec.S3Type == "s3s" {
+		// all rest APIs coming from edgefs-restapi image, that
+		// includes mgmt, s3, s3s and swift
+		rookImage = defaultS3Image
+		imageArgs = s.Spec.S3Type
+	} else if s.Spec.S3Type == "s3g" {
+		// built-in s3 version, limited and experimental coming
+		// as a part of core edgefs image
+		rookImage = c.rookImage
+	} else {
+		return fmt.Errorf("invalid S3 service type %s", s.Spec.S3Type)
+	}
+
+	var rookImageVer string
+	rookImageComponents := strings.Split(rookImage, ":")
+	if len(rookImageComponents) == 2 {
+		rookImageVer = rookImageComponents[1]
+	} else {
+		rookImageVer = "latest"
+	}
+
 	// check if S3 service already exists
 	exists, err := serviceExists(c.context, s)
 	if err == nil && exists {
@@ -81,7 +107,7 @@ func (c *S3Controller) CreateOrUpdate(s edgefsv1alpha1.S3, update bool, ownerRef
 	}
 
 	// start the deployment
-	deployment := c.makeDeployment(s.Name, s.Namespace, c.rookImage, s.Spec)
+	deployment := c.makeDeployment(s.Name, s.Namespace, rookImage+":"+rookImageVer, imageArgs, s.Spec)
 	if _, err := c.context.Clientset.Apps().Deployments(s.Namespace).Create(deployment); err != nil {
 		if !errors.IsAlreadyExists(err) {
 			return fmt.Errorf("failed to create %s deployment. %+v", appName, err)
@@ -128,7 +154,7 @@ func (c *S3Controller) makeS3Service(name, svcname, namespace string, s3Spec edg
 	return svc
 }
 
-func (c *S3Controller) makeDeployment(svcname, namespace, rookImage string, s3Spec edgefsv1alpha1.S3Spec) *apps.Deployment {
+func (c *S3Controller) makeDeployment(svcname, namespace, rookImage, imageArgs string, s3Spec edgefsv1alpha1.S3Spec) *apps.Deployment {
 
 	name := instanceName(svcname)
 	volumes := []v1.Volume{}
@@ -183,7 +209,7 @@ func (c *S3Controller) makeDeployment(svcname, namespace, rookImage string, s3Sp
 			Labels: getLabels(name, svcname, namespace),
 		},
 		Spec: v1.PodSpec{
-			Containers:    []v1.Container{c.s3Container(svcname, name, rookImage, s3Spec)},
+			Containers:    []v1.Container{c.s3Container(svcname, name, rookImage, imageArgs, s3Spec)},
 			RestartPolicy: v1.RestartPolicyAlways,
 			Volumes:       volumes,
 			HostIPC:       true,
@@ -215,7 +241,7 @@ func (c *S3Controller) makeDeployment(svcname, namespace, rookImage string, s3Sp
 	return d
 }
 
-func (c *S3Controller) s3Container(svcname, name, containerImage string, s3Spec edgefsv1alpha1.S3Spec) v1.Container {
+func (c *S3Controller) s3Container(svcname, name, containerImage, args string, s3Spec edgefsv1alpha1.S3Spec) v1.Container {
 
 	runAsUser := int64(0)
 	readOnlyRootFilesystem := false
@@ -248,7 +274,7 @@ func (c *S3Controller) s3Container(svcname, name, containerImage string, s3Spec
 		Name:            name,
 		Image:           containerImage,
 		ImagePullPolicy: v1.PullAlways,
-		Args:            []string{"s3"},
+		Args:            []string{args},
 		Env: []v1.EnvVar{
 			{
 				Name:  "CCOW_LOG_LEVEL",
@@ -258,6 +284,10 @@ func (c *S3Controller) s3Container(svcname, name, containerImage string, s3Spec
 				Name:  "CCOW_SVCNAME",
 				Value: svcname,
 			},
+			{
+				Name:  "DEBUG",
+				Value: "alert,error,info",
+			},
 			{
 				Name: "HOST_HOSTNAME",
 				ValueFrom: &v1.EnvVarSource{