From 9dd2e422bb4df5fe9fc8a207e41ff61216d9a4fa Mon Sep 17 00:00:00 2001 From: Dmitry Yusupov Date: Fri, 26 Apr 2019 19:01:23 -0700 Subject: [PATCH] address regression in ISGW dynamicFetch configuring Signed-off-by: Dmitry Yusupov --- Documentation/edgefs-isgw-crd.md | 6 ++--- pkg/apis/edgefs.rook.io/v1beta1/types.go | 4 ++-- pkg/operator/edgefs/isgw/isgw.go | 29 +++++++++++++++++------- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/Documentation/edgefs-isgw-crd.md b/Documentation/edgefs-isgw-crd.md index a411d4715..1f2b99862 100644 --- a/Documentation/edgefs-isgw-crd.md +++ b/Documentation/edgefs-isgw-crd.md @@ -50,8 +50,8 @@ spec: remoteURL: "ccow://192.168.1.43:14000" #replicationType: "initial" #metadataOnly: true - #dynamicFetchPort: 49600 - #localPort: 10000 + #dynamicFetchAddr: 0.0.0.0:49600 + #localAddr: 0.0.0.0:10000 #useEncryptedTunnel: true #chunkCacheSize: 1Gi #placement: @@ -89,7 +89,7 @@ spec: - `remoteURL`: I/O flow with destination pointing to the other ISGW endpoint service, locally mounted directory or other (non-EdgeFS) S3 compatible service. Format should be of form PROTOCOL://LOCATION, where PROTOCOL can be of "ccow", "file", "s3" and LOCATION depends on the PROTOCOL. In case of "ccow" LOCATION needs to specify a HOST:PORT of the other EdgeFS site, e.g. ccow://192.168.1.43:14000. In case of "file" LOCATION needs to be a path to a local POSIX directory, e.g. file://volumeDir. In cae of "s3" LOCATION can either point to an existing bucket, e.g. s3://mybucket.aws.com or to the other S3 compatible server IP:HOST, e.g. s3://214.4.34.22:4567. It is important to note that file:// and s3:// protocols only provided for easy integration with existing infrastructure. - `replicationType`: Defines how tenant's buckets needs to be replicated. Can be specified as `initial`, `continuous` or `initial+continuous` (default). In case of `initial` replication type, ISGW service will only sync served buckets once with ability to manually trigger re-sync again. In case of `continuous` replication type, ISGW service will enable asynchronous syncing from the moment a bucket added to a served list of buckets, so that only recent changes will be synchronized, continuously. In case of `initial+continuous` replication type, ISGW service will first initiate syncing of all existing served buckets contents and once all is copied, it will automatically enable `continuous` mode of operation, guaranteeing that none of the source objects are skipped. - `metadataOnly`: If true, then only immutable metadata changes will be transferred. This also enables on-demand data fetch and E-LRU cache capability. In case of metadata only transfers, dynamically fetched data will be cached at the accessing endpoints ISGW links locations. If an application would want to read an offset of a data which isn't found, global search request will be formed and distributed across all the connected ISGW bi-directional links serving source bucket. Missing data chunks will be found and fetched, delivered transparently to application. -- `dynamicFetchPort`: Defines local cluster pod service port. Used by EdgeFS gateways (S3, S3X, NFS or iSCSI) to distribute missing chunk fetch requests. Default is 49678. +- `dynamicFetchAddr`: Defines local cluster pod service host:port. Used by EdgeFS gateways (S3, S3X, NFS or iSCSI) to distribute missing chunk fetch requests. It needs to be set to other than 0.0.0.0 on access endpoints so to enable dynamic fetch feature. Default is 0.0.0.0:49678. - `localAddr`: Defines local cluster pod service address and port. Used by ISGW link endpoint. Kubernetes ExternalIP will be automatically constructed if address is provided. Default is 0.0.0.0:14000. - `useEncryptedTunnel`: If true, enables usage of encrypted SSH tunnel. By default, transferred metadata and data chunks are just compressed but unencrypted. Enable encryption if ISGW links crossing boundaries of single data center or multiple clouds. - `chunkCacheSize`: Limit amount of memory allocated for dynamic chunk cache. By default ISGW pod uses up to 75% of available memory as chunk caching area. This option can influence this allocation strategy. diff --git a/pkg/apis/edgefs.rook.io/v1beta1/types.go b/pkg/apis/edgefs.rook.io/v1beta1/types.go index 59dbfb860..2ac41625e 100644 --- a/pkg/apis/edgefs.rook.io/v1beta1/types.go +++ b/pkg/apis/edgefs.rook.io/v1beta1/types.go @@ -323,8 +323,8 @@ type ISGWSpec struct { ReplicationType string `json:"replicationType,omitempty"` // ISGW Metadata Only flag MetadataOnly bool `json:"metadataOnly,omitempty"` - // ISGW Dynamic Fetch Port (default 49678) - DynamicFetchPort uint `json:"dynamicFetchPort,omitempty"` + // ISGW Dynamic Fetch Addr (default is '-', means disabled) + DynamicFetchAddr string `json:"dynamicFetchAddr,omitempty"` // ISGW Endpoint local address (default value 0.0.0.0:14000) LocalAddr string `json:"localAddr,omitempty"` // ISGW Encrypted Tunnel flag diff --git a/pkg/operator/edgefs/isgw/isgw.go b/pkg/operator/edgefs/isgw/isgw.go index 6d19170c0..6fa2914b6 100644 --- a/pkg/operator/edgefs/isgw/isgw.go +++ b/pkg/operator/edgefs/isgw/isgw.go @@ -69,8 +69,8 @@ func (c *ISGWController) CreateOrUpdate(s edgefsv1beta1.ISGW, update bool, owner s.Spec.ReplicationType = defaultReplicationType } - if s.Spec.DynamicFetchPort == 0 { - s.Spec.DynamicFetchPort = defaultDynamicFetchPort + if s.Spec.DynamicFetchAddr == "" { + s.Spec.DynamicFetchAddr = "-" } if s.Spec.LocalAddr == "" { @@ -157,8 +157,15 @@ func (c *ISGWController) makeISGWService(name, svcname, namespace string, isgwSp svc.Spec.ExternalIPs = []string{laddr} } } - if direction == 1 || direction == 3 { - svc.Spec.Ports = append(svc.Spec.Ports, v1.ServicePort{Name: "dfport", Port: int32(isgwSpec.DynamicFetchPort), Protocol: v1.ProtocolTCP}) + if (direction == 1 || direction == 3) && isgwSpec.DynamicFetchAddr != "-" { + _, port, err := net.SplitHostPort(isgwSpec.DynamicFetchAddr) + if err != nil { + logger.Errorf("wrong dynamicFetchAddr format") + return svc + } + lport, _ := strconv.Atoi(port) + + svc.Spec.Ports = append(svc.Spec.Ports, v1.ServicePort{Name: "dfport", Port: int32(lport), Protocol: v1.ProtocolTCP}) } k8sutil.SetOwnerRef(c.context.Clientset, namespace, &svc.ObjectMeta, &c.ownerRef) @@ -309,8 +316,8 @@ func (c *ISGWController) isgwContainer(svcname, name, containerImage string, isg Value: isgwSpec.LocalAddr, }, { - Name: "EFSISGW_DYNAMIC_FETCH_PORT", - Value: strconv.Itoa(int(isgwSpec.DynamicFetchPort)), + Name: "EFSISGW_DYNAMIC_FETCH_ADDR", + Value: isgwSpec.DynamicFetchAddr, }, { Name: "EFSISGW_REPLICATION_TYPE", @@ -342,8 +349,14 @@ func (c *ISGWController) isgwContainer(svcname, name, containerImage string, isg lport, _ := strconv.Atoi(port) cont.Ports = append(cont.Ports, v1.ContainerPort{Name: "lport", ContainerPort: int32(lport), Protocol: v1.ProtocolTCP}) } - if direction == 1 || direction == 3 { - cont.Ports = append(cont.Ports, v1.ContainerPort{Name: "dfport", ContainerPort: int32(isgwSpec.DynamicFetchPort), Protocol: v1.ProtocolTCP}) + if (direction == 1 || direction == 3) && isgwSpec.DynamicFetchAddr != "-" { + _, port, err := net.SplitHostPort(isgwSpec.DynamicFetchAddr) + if err != nil { + logger.Errorf("wrong dynamicFetchAddr format") + return cont + } + lport, _ := strconv.Atoi(port) + cont.Ports = append(cont.Ports, v1.ContainerPort{Name: "dfport", ContainerPort: int32(lport), Protocol: v1.ProtocolTCP}) } cont.Env = append(cont.Env, edgefsv1beta1.GetInitiatorEnvArr("isgw",