diff --git a/Documentation/edgefs-iscsi-crd.md b/Documentation/edgefs-iscsi-crd.md index ae2842389..e7e4d237e 100644 --- a/Documentation/edgefs-iscsi-crd.md +++ b/Documentation/edgefs-iscsi-crd.md @@ -60,9 +60,7 @@ spec: For more detailed instructions please refer to [EdgeFS Wiki](https://github.com/Nexenta/edgefs/wiki). -Simple procedure to get things initialized and configured: - -## Setting up FlexHash and Site root object +Below is an exampmle procedure to get things initialized and configured. Before new local namespace (or local site) can be used, it has to be initialized with FlexHash and special purpose root object. @@ -114,11 +112,11 @@ Now cluster is setup, services can be now created. 4. Create iSCSI Target services objects for tenants ``` -efscli service create iscsi iscCola -efscli service serve iscCola Hawaii/Cola/bk1/lun1 X-volsize=10G,ccow-chunkmap-chunk-size=16384 -efscli service serve iscCola Hawaii/Cola/bk1/lun2 X-volsize=20G,ccow-chunkmap-chunk-size=131072 -efscli service create iscsi iscPepsi -efscli service serve iscPepsi Hawaii/Pepsi/bk1/lun1 X-volsize=20G +efscli service create iscsi isc-cola +efscli service serve isc-cola Hawaii/Cola/bk1/lun1 X-volsize=10G,ccow-chunkmap-chunk-size=16384 +efscli service serve isc-cola Hawaii/Cola/bk1/lun2 X-volsize=20G,ccow-chunkmap-chunk-size=131072 +efscli service create iscsi isc-pepsi +efscli service serve isc-pepsi Hawaii/Pepsi/bk1/lun1 X-volsize=20G ``` 5. Create ISCSI CRDs diff --git a/Documentation/edgefs-isgw-crd.md b/Documentation/edgefs-isgw-crd.md index 10354cba7..3a9f04d9a 100644 --- a/Documentation/edgefs-isgw-crd.md +++ b/Documentation/edgefs-isgw-crd.md @@ -96,9 +96,7 @@ This example will demonstrate creation of simple syncing namespace between two s For more detailed instructions please refer to [EdgeFS Wiki](https://github.com/Nexenta/edgefs/wiki). -Simple procedure to get things initialized and configured: - -## Setting up FlexHash and Site root object +Below is an exampmle procedure to get things initialized and configured. Before new local namespace (or local site) can be used, it has to be initialized with FlexHash and special purpose root object. diff --git a/Documentation/edgefs-nfs-crd.md b/Documentation/edgefs-nfs-crd.md index e72dfa824..fcfaa8de0 100644 --- a/Documentation/edgefs-nfs-crd.md +++ b/Documentation/edgefs-nfs-crd.md @@ -54,9 +54,7 @@ spec: For more detailed instructions please refer to [EdgeFS Wiki](https://github.com/Nexenta/edgefs/wiki). -Simple procedure to get things initialized and configured: - -## Setting up FlexHash and Site root object +Below is an exampmle procedure to get things initialized and configured. Before new local namespace (or local site) can be used, it has to be initialized with FlexHash and special purpose root object. diff --git a/Documentation/edgefs-s3-crd.md b/Documentation/edgefs-s3-crd.md index 49d67f2a4..229f2d318 100644 --- a/Documentation/edgefs-s3-crd.md +++ b/Documentation/edgefs-s3-crd.md @@ -59,9 +59,7 @@ spec: For more detailed instructions please refer to [EdgeFS Wiki](https://github.com/Nexenta/edgefs/wiki). -Simple procedure to get things initialized and configured: - -## Setting up FlexHash and Site root object +Below is an exampmle procedure to get things initialized and configured. Before new local namespace (or local site) can be used, it has to be initialized with FlexHash and special purpose root object. @@ -116,7 +114,7 @@ Now cluster is setup, services can be now created. efscli service create s3 s3-cola efscli service serve s3-cola Hawaii/Cola efscli service create s3 s3-pepsi -efscli service serve s3-pepsi Hawaii/Pepsi/bk1 +efscli service serve s3-pepsi Hawaii/Pepsi ``` In case of s3type set to `s3`, do not forget to configure default domain name: diff --git a/Documentation/edgefs-s3x-crd.md b/Documentation/edgefs-s3x-crd.md index e6e6c8e6b..7f7d99185 100644 --- a/Documentation/edgefs-s3x-crd.md +++ b/Documentation/edgefs-s3x-crd.md @@ -61,9 +61,7 @@ spec: For more detailed instructions please refer to [EdgeFS Wiki](https://github.com/Nexenta/edgefs/wiki). -Simple procedure to get things initialized and configured: - -## Setting up FlexHash and Site root object +Below is an exampmle procedure to get things initialized and configured. Before new local namespace (or local site) can be used, it has to be initialized with FlexHash and special purpose root object. @@ -118,7 +116,7 @@ Now cluster is setup, services can be now created. efscli service create s3x s3x-cola efscli service serve s3x-cola Hawaii/Cola efscli service create s3x s3x-pepsi -efscli service serve s3x-pepsi Hawaii/Pepsi/bk1 +efscli service serve s3x-pepsi Hawaii/Pepsi ``` 5. Create S3X CRDs diff --git a/Documentation/edgefs-swift-crd.md b/Documentation/edgefs-swift-crd.md new file mode 100644 index 000000000..3c8525d7e --- /dev/null +++ b/Documentation/edgefs-swift-crd.md @@ -0,0 +1,140 @@ +--- +title: EdgeFS OpenStack/SWIFT CRD +weight: 4550 +indent: true +--- + +# EdgeFS OpenStack/SWIFT CRD + +Rook allows creation and customization of OpenStack/SWIFT compatible services through the custom resource definitions (CRDs). +The following settings are available for customization of SWIFT services. + +## Sample + +```yaml +apiVersion: edgefs.rook.io/v1alpha1 +kind: SWIFT +metadata: + name: swift01 + namespace: rook-edgefs +spec: + instances: 3 + placement: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: role + # operator: In + # values: + # - swift-node + # tolerations: + # - key: swift-node + # operator: Exists + # podAffinity: + # podAntiAffinity: + resources: + # limits: + # cpu: "500m" + # memory: "1024Mi" + # requests: + # cpu: "500m" + # memory: "1024Mi" +``` + +### Metadata + +- `name`: The name of the SWIFT service to create, which must match existing EdgeFS service. +- `namespace`: The namespace of the Rook cluster where the SWIFT service is created. +- `sslCertificateRef`: If the certificate is not specified, SSL will use default crt and key files. If specified, this is the name of the Kubernetes secret that contains the SSL certificate to be used for secure connections. Please see [secret YAML file example](/cluster/examples/kubernetes/edgefs/sslKeyCertificate.yaml) on how to setup Kuberenetes secret. Notice that base64 encoding is required. +- `port`: The port on which the SWIFT pods and the SWIFT service will be listening (not encrypted). Default port is 9981. +- `securePort`: The secure port on which SWIFT pods will be listening. If not defined then default SSL certificates will be used. Default port is 443. +- `instances`: The number of active SWIFT service instances. For load balancing we recommend to use nginx and the like solutions. +- `placement`: The SWIFT pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](/cluster/examples/kubernetes/edgefs/cluster.yaml). +- `resources`: Set resource requests/limits for the SWIFT pods, see [Resource Requirements/Limits](edgefs-cluster-crd.md#resource-requirementslimits). + +### Setting up EdgeFS namespace and tenant + +For more detailed instructions please refer to [EdgeFS Wiki](https://github.com/Nexenta/edgefs/wiki). + +Below is an exampmle procedure to get things initialized and configured. + +Before new local namespace (or local site) can be used, it has to be initialized with FlexHash and special purpose root object. + +FlexHash consists of dynamically discovered configuration and checkpoint of accepted distribution table. FlexHash is responsible for I/O direction and plays important role in dynamic load balancing logic. It defines so-called Negotiating Groups (typically across zoned 8-24 disks) and final table distribution across all the participating components, e.g. data nodes, service gateways and tools. + +Root object holds system information and table of namespaces registered to a local site. Root object is always local and never shared between the sites. + +To initialize system and prepare logical definitions, login to the toolbox as shown in this example: + +``` +kubectl get po --all-namespaces | grep edgefs-mgr +kubectl exec -it -n rook-edgefs rook-edgefs-mgr-6cb9598469-czr7p -- env COLUMNS=$COLUMNS LINES=$LINES TERM=linux toolbox +``` + +Assumption at this point is that nodes are all configured and can be seen via the following command: + +``` +efscli system status +``` + +1. Initialize cluster + +Verify that HW (or better say emulated in this case) configuration look normal and accept it + +``` +efscli system init +``` + +At this point new dynamically discovered configuration checkpoint will be created at $NEDGE_HOME/var/run/flexhash-checkpoint.json +This will also create system "root" object, holding Site's Namespace. Namespace may consist of more then single region. + +2. Create new local namespace (or we also call it "Region" or "Segment") + +``` +efscli cluster create Hawaii +``` + +3. Create logical tenants of cluster namespace "Hawaii", also buckets if needed + +``` +efscli tenant create Hawaii/Cola +efscli bucket create Hawaii/Cola/bk1 +efscli tenant create Hawaii/Pepsi +efscli bucket create Hawaii/Pepsi/bk1 +``` + +Now cluster is setup, services can be now created. + +4. Create SWIFT services objects for tenants + +``` +efscli service create swift swift-cola +efscli service serve swift-cola Hawaii +efscli service create swift swiftPepsi +efscli service serve swift-pepsi Hawaii +``` + +5. Create SWIFT CRDs + +```yaml +apiVersion: edgefs.rook.io/v1alpha1 +kind: SWIFT +metadata: + name: swiftCola + namespace: rook-edgefs +spec: + instances: 1 +``` + +```yaml +apiVersion: edgefs.rook.io/v1alpha1 +kind: SWIFT +metadata: + name: swiftPepsi + namespace: rook-edgefs +spec: + instances: 1 +``` + +At this point two SWIFT services should be available and listening on default ports. diff --git a/cluster/examples/kubernetes/edgefs/operator.yaml b/cluster/examples/kubernetes/edgefs/operator.yaml index c22e0a5ea..946667e29 100644 --- a/cluster/examples/kubernetes/edgefs/operator.yaml +++ b/cluster/examples/kubernetes/edgefs/operator.yaml @@ -33,6 +33,20 @@ spec: --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition +metadata: + name: swifts.edgefs.rook.io +spec: + group: edgefs.rook.io + names: + kind: SWIFT + listKind: SWIFTList + plural: swifts + singular: swift + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition metadata: name: s3s.edgefs.rook.io spec: diff --git a/cluster/examples/kubernetes/edgefs/swift.yaml b/cluster/examples/kubernetes/edgefs/swift.yaml new file mode 100644 index 000000000..1c6670ec5 --- /dev/null +++ b/cluster/examples/kubernetes/edgefs/swift.yaml @@ -0,0 +1,31 @@ +apiVersion: edgefs.rook.io/v1alpha1 +kind: SWIFT +metadata: + name: swift01 + namespace: rook-edgefs +spec: + instances: 1 + #port: 4500 # HTTP port (default is 9981) + #securePort: 4501 # HTTPS port (default is 443) + #sslCertificateRef: ssl-secret + #placement: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: rook-edgefs-nodetype + # operator: In + # values: + # - gateway + # tolerations: + # - key: rook-edgefs-nodetype + # operator: Exists + # podAffinity: + # podAntiAffinity: + #resources: + # limits: + # cpu: "2" + # memory: "4Gi" + # requests: + # cpu: "2" + # memory: "4Gi" diff --git a/pkg/apis/edgefs.rook.io/v1alpha1/register.go b/pkg/apis/edgefs.rook.io/v1alpha1/register.go index 039ab3d4d..3fa05bb2d 100644 --- a/pkg/apis/edgefs.rook.io/v1alpha1/register.go +++ b/pkg/apis/edgefs.rook.io/v1alpha1/register.go @@ -59,6 +59,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &NFSList{}, &S3{}, &S3List{}, + &SWIFT{}, + &SWIFTList{}, &S3X{}, &S3XList{}, &ISCSI{}, diff --git a/pkg/apis/edgefs.rook.io/v1alpha1/types.go b/pkg/apis/edgefs.rook.io/v1alpha1/types.go index cdf3e7c82..08da936e5 100644 --- a/pkg/apis/edgefs.rook.io/v1alpha1/types.go +++ b/pkg/apis/edgefs.rook.io/v1alpha1/types.go @@ -156,6 +156,40 @@ type S3Spec struct { // +genclient:noStatus // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type SWIFT struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata"` + Spec SWIFTSpec `json:"spec"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type SWIFTList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + Items []S3 `json:"items"` +} + +// SWIFTSpec represent the spec of a swift service +type SWIFTSpec struct { + // The affinity to place the SWIFT pods (default is to place on any available nodes in EdgeFS running namespace) + Placement rook.Placement `json:"placement"` + // Resources set resource requests and limits + Resources v1.ResourceRequirements `json:"resources,omitempty"` + // The number of pods in the S3 replicaset + Instances int32 `json:"instances"` + //S3 Http port (default value 9982) + Port uint `json:"port,omitempty"` + //S3 Https port (default value 9443) + SecurePort uint `json:"securePort,omitempty"` + // The name of the secret that stores the ssl certificate for secure s3 connections + SSLCertificateRef string `json:"sslCertificateRef,omitempty"` +} + +// +genclient +// +genclient:noStatus +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + type S3X struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata"` diff --git a/pkg/apis/edgefs.rook.io/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/edgefs.rook.io/v1alpha1/zz_generated.deepcopy.go index f28f358a0..9917604ad 100644 --- a/pkg/apis/edgefs.rook.io/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/edgefs.rook.io/v1alpha1/zz_generated.deepcopy.go @@ -793,6 +793,84 @@ func (in *S3XSpec) DeepCopy() *S3XSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SWIFT) DeepCopyInto(out *SWIFT) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SWIFT. +func (in *SWIFT) DeepCopy() *SWIFT { + if in == nil { + return nil + } + out := new(SWIFT) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SWIFT) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SWIFTList) DeepCopyInto(out *SWIFTList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]S3, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SWIFTList. +func (in *SWIFTList) DeepCopy() *SWIFTList { + if in == nil { + return nil + } + out := new(SWIFTList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SWIFTList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SWIFTSpec) DeepCopyInto(out *SWIFTSpec) { + *out = *in + in.Placement.DeepCopyInto(&out.Placement) + in.Resources.DeepCopyInto(&out.Resources) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SWIFTSpec. +func (in *SWIFTSpec) DeepCopy() *SWIFTSpec { + if in == nil { + return nil + } + out := new(SWIFTSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SetupNode) DeepCopyInto(out *SetupNode) { *out = *in diff --git a/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/edgefs.rook.io_client.go b/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/edgefs.rook.io_client.go index 4a5cae1bc..87d810f23 100644 --- a/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/edgefs.rook.io_client.go +++ b/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/edgefs.rook.io_client.go @@ -33,6 +33,7 @@ type EdgefsV1alpha1Interface interface { NFSsGetter S3sGetter S3XsGetter + SWIFTsGetter } // EdgefsV1alpha1Client is used to interact with features provided by the edgefs.rook.io group. @@ -64,6 +65,10 @@ func (c *EdgefsV1alpha1Client) S3Xs(namespace string) S3XInterface { return newS3Xs(c, namespace) } +func (c *EdgefsV1alpha1Client) SWIFTs(namespace string) SWIFTInterface { + return newSWIFTs(c, namespace) +} + // NewForConfig creates a new EdgefsV1alpha1Client for the given config. func NewForConfig(c *rest.Config) (*EdgefsV1alpha1Client, error) { config := *c diff --git a/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/fake/fake_edgefs.rook.io_client.go b/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/fake/fake_edgefs.rook.io_client.go index c57b64d42..2f3cdde0a 100644 --- a/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/fake/fake_edgefs.rook.io_client.go +++ b/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/fake/fake_edgefs.rook.io_client.go @@ -52,6 +52,10 @@ func (c *FakeEdgefsV1alpha1) S3Xs(namespace string) v1alpha1.S3XInterface { return &FakeS3Xs{c, namespace} } +func (c *FakeEdgefsV1alpha1) SWIFTs(namespace string) v1alpha1.SWIFTInterface { + return &FakeSWIFTs{c, namespace} +} + // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. func (c *FakeEdgefsV1alpha1) RESTClient() rest.Interface { diff --git a/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/fake/fake_swift.go b/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/fake/fake_swift.go new file mode 100644 index 000000000..8d5295907 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/fake/fake_swift.go @@ -0,0 +1,128 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeSWIFTs implements SWIFTInterface +type FakeSWIFTs struct { + Fake *FakeEdgefsV1alpha1 + ns string +} + +var swiftsResource = schema.GroupVersionResource{Group: "edgefs.rook.io", Version: "v1alpha1", Resource: "swifts"} + +var swiftsKind = schema.GroupVersionKind{Group: "edgefs.rook.io", Version: "v1alpha1", Kind: "SWIFT"} + +// Get takes name of the sWIFT, and returns the corresponding sWIFT object, and an error if there is any. +func (c *FakeSWIFTs) Get(name string, options v1.GetOptions) (result *v1alpha1.SWIFT, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(swiftsResource, c.ns, name), &v1alpha1.SWIFT{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.SWIFT), err +} + +// List takes label and field selectors, and returns the list of SWIFTs that match those selectors. +func (c *FakeSWIFTs) List(opts v1.ListOptions) (result *v1alpha1.SWIFTList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(swiftsResource, swiftsKind, c.ns, opts), &v1alpha1.SWIFTList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.SWIFTList{ListMeta: obj.(*v1alpha1.SWIFTList).ListMeta} + for _, item := range obj.(*v1alpha1.SWIFTList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested sWIFTs. +func (c *FakeSWIFTs) Watch(opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(swiftsResource, c.ns, opts)) + +} + +// Create takes the representation of a sWIFT and creates it. Returns the server's representation of the sWIFT, and an error, if there is any. +func (c *FakeSWIFTs) Create(sWIFT *v1alpha1.SWIFT) (result *v1alpha1.SWIFT, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(swiftsResource, c.ns, sWIFT), &v1alpha1.SWIFT{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.SWIFT), err +} + +// Update takes the representation of a sWIFT and updates it. Returns the server's representation of the sWIFT, and an error, if there is any. +func (c *FakeSWIFTs) Update(sWIFT *v1alpha1.SWIFT) (result *v1alpha1.SWIFT, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(swiftsResource, c.ns, sWIFT), &v1alpha1.SWIFT{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.SWIFT), err +} + +// Delete takes name of the sWIFT and deletes it. Returns an error if one occurs. +func (c *FakeSWIFTs) Delete(name string, options *v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteAction(swiftsResource, c.ns, name), &v1alpha1.SWIFT{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeSWIFTs) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(swiftsResource, c.ns, listOptions) + + _, err := c.Fake.Invokes(action, &v1alpha1.SWIFTList{}) + return err +} + +// Patch applies the patch and returns the patched sWIFT. +func (c *FakeSWIFTs) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.SWIFT, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(swiftsResource, c.ns, name, data, subresources...), &v1alpha1.SWIFT{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.SWIFT), err +} diff --git a/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/generated_expansion.go b/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/generated_expansion.go index ac15682e6..584f798dd 100644 --- a/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/generated_expansion.go @@ -29,3 +29,5 @@ type NFSExpansion interface{} type S3Expansion interface{} type S3XExpansion interface{} + +type SWIFTExpansion interface{} diff --git a/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/swift.go b/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/swift.go new file mode 100644 index 000000000..6a9327ed6 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/edgefs.rook.io/v1alpha1/swift.go @@ -0,0 +1,157 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + scheme "github.com/rook/rook/pkg/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// SWIFTsGetter has a method to return a SWIFTInterface. +// A group's client should implement this interface. +type SWIFTsGetter interface { + SWIFTs(namespace string) SWIFTInterface +} + +// SWIFTInterface has methods to work with SWIFT resources. +type SWIFTInterface interface { + Create(*v1alpha1.SWIFT) (*v1alpha1.SWIFT, error) + Update(*v1alpha1.SWIFT) (*v1alpha1.SWIFT, error) + Delete(name string, options *v1.DeleteOptions) error + DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error + Get(name string, options v1.GetOptions) (*v1alpha1.SWIFT, error) + List(opts v1.ListOptions) (*v1alpha1.SWIFTList, error) + Watch(opts v1.ListOptions) (watch.Interface, error) + Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.SWIFT, err error) + SWIFTExpansion +} + +// sWIFTs implements SWIFTInterface +type sWIFTs struct { + client rest.Interface + ns string +} + +// newSWIFTs returns a SWIFTs +func newSWIFTs(c *EdgefsV1alpha1Client, namespace string) *sWIFTs { + return &sWIFTs{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the sWIFT, and returns the corresponding sWIFT object, and an error if there is any. +func (c *sWIFTs) Get(name string, options v1.GetOptions) (result *v1alpha1.SWIFT, err error) { + result = &v1alpha1.SWIFT{} + err = c.client.Get(). + Namespace(c.ns). + Resource("swifts"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of SWIFTs that match those selectors. +func (c *sWIFTs) List(opts v1.ListOptions) (result *v1alpha1.SWIFTList, err error) { + result = &v1alpha1.SWIFTList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("swifts"). + VersionedParams(&opts, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested sWIFTs. +func (c *sWIFTs) Watch(opts v1.ListOptions) (watch.Interface, error) { + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("swifts"). + VersionedParams(&opts, scheme.ParameterCodec). + Watch() +} + +// Create takes the representation of a sWIFT and creates it. Returns the server's representation of the sWIFT, and an error, if there is any. +func (c *sWIFTs) Create(sWIFT *v1alpha1.SWIFT) (result *v1alpha1.SWIFT, err error) { + result = &v1alpha1.SWIFT{} + err = c.client.Post(). + Namespace(c.ns). + Resource("swifts"). + Body(sWIFT). + Do(). + Into(result) + return +} + +// Update takes the representation of a sWIFT and updates it. Returns the server's representation of the sWIFT, and an error, if there is any. +func (c *sWIFTs) Update(sWIFT *v1alpha1.SWIFT) (result *v1alpha1.SWIFT, err error) { + result = &v1alpha1.SWIFT{} + err = c.client.Put(). + Namespace(c.ns). + Resource("swifts"). + Name(sWIFT.Name). + Body(sWIFT). + Do(). + Into(result) + return +} + +// Delete takes name of the sWIFT and deletes it. Returns an error if one occurs. +func (c *sWIFTs) Delete(name string, options *v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("swifts"). + Name(name). + Body(options). + Do(). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *sWIFTs) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("swifts"). + VersionedParams(&listOptions, scheme.ParameterCodec). + Body(options). + Do(). + Error() +} + +// Patch applies the patch and returns the patched sWIFT. +func (c *sWIFTs) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.SWIFT, err error) { + result = &v1alpha1.SWIFT{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("swifts"). + SubResource(subresources...). + Name(name). + Body(data). + Do(). + Into(result) + return +} diff --git a/pkg/client/informers/externalversions/edgefs.rook.io/v1alpha1/interface.go b/pkg/client/informers/externalversions/edgefs.rook.io/v1alpha1/interface.go index 79c5477c5..2b7f66db4 100644 --- a/pkg/client/informers/externalversions/edgefs.rook.io/v1alpha1/interface.go +++ b/pkg/client/informers/externalversions/edgefs.rook.io/v1alpha1/interface.go @@ -36,6 +36,8 @@ type Interface interface { S3s() S3Informer // S3Xs returns a S3XInformer. S3Xs() S3XInformer + // SWIFTs returns a SWIFTInformer. + SWIFTs() SWIFTInformer } type version struct { @@ -78,3 +80,8 @@ func (v *version) S3s() S3Informer { func (v *version) S3Xs() S3XInformer { return &s3XInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } + +// SWIFTs returns a SWIFTInformer. +func (v *version) SWIFTs() SWIFTInformer { + return &sWIFTInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/client/informers/externalversions/edgefs.rook.io/v1alpha1/swift.go b/pkg/client/informers/externalversions/edgefs.rook.io/v1alpha1/swift.go new file mode 100644 index 000000000..0e267141e --- /dev/null +++ b/pkg/client/informers/externalversions/edgefs.rook.io/v1alpha1/swift.go @@ -0,0 +1,89 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + time "time" + + edgefsrookiov1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + versioned "github.com/rook/rook/pkg/client/clientset/versioned" + internalinterfaces "github.com/rook/rook/pkg/client/informers/externalversions/internalinterfaces" + v1alpha1 "github.com/rook/rook/pkg/client/listers/edgefs.rook.io/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// SWIFTInformer provides access to a shared informer and lister for +// SWIFTs. +type SWIFTInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.SWIFTLister +} + +type sWIFTInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewSWIFTInformer constructs a new informer for SWIFT type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewSWIFTInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredSWIFTInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredSWIFTInformer constructs a new informer for SWIFT type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredSWIFTInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.EdgefsV1alpha1().SWIFTs(namespace).List(options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.EdgefsV1alpha1().SWIFTs(namespace).Watch(options) + }, + }, + &edgefsrookiov1alpha1.SWIFT{}, + resyncPeriod, + indexers, + ) +} + +func (f *sWIFTInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredSWIFTInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *sWIFTInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&edgefsrookiov1alpha1.SWIFT{}, f.defaultInformer) +} + +func (f *sWIFTInformer) Lister() v1alpha1.SWIFTLister { + return v1alpha1.NewSWIFTLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index fa90999d1..539b55904 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -106,6 +106,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Edgefs().V1alpha1().S3s().Informer()}, nil case edgefsrookiov1alpha1.SchemeGroupVersion.WithResource("s3xs"): return &genericInformer{resource: resource.GroupResource(), informer: f.Edgefs().V1alpha1().S3Xs().Informer()}, nil + case edgefsrookiov1alpha1.SchemeGroupVersion.WithResource("swifts"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Edgefs().V1alpha1().SWIFTs().Informer()}, nil // Group=minio.rook.io, Version=v1alpha1 case miniorookiov1alpha1.SchemeGroupVersion.WithResource("objectstores"): diff --git a/pkg/client/listers/edgefs.rook.io/v1alpha1/expansion_generated.go b/pkg/client/listers/edgefs.rook.io/v1alpha1/expansion_generated.go index 7d5ed44cc..cd3b56020 100644 --- a/pkg/client/listers/edgefs.rook.io/v1alpha1/expansion_generated.go +++ b/pkg/client/listers/edgefs.rook.io/v1alpha1/expansion_generated.go @@ -65,3 +65,11 @@ type S3XListerExpansion interface{} // S3XNamespaceListerExpansion allows custom methods to be added to // S3XNamespaceLister. type S3XNamespaceListerExpansion interface{} + +// SWIFTListerExpansion allows custom methods to be added to +// SWIFTLister. +type SWIFTListerExpansion interface{} + +// SWIFTNamespaceListerExpansion allows custom methods to be added to +// SWIFTNamespaceLister. +type SWIFTNamespaceListerExpansion interface{} diff --git a/pkg/client/listers/edgefs.rook.io/v1alpha1/swift.go b/pkg/client/listers/edgefs.rook.io/v1alpha1/swift.go new file mode 100644 index 000000000..5ad6883f0 --- /dev/null +++ b/pkg/client/listers/edgefs.rook.io/v1alpha1/swift.go @@ -0,0 +1,94 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// SWIFTLister helps list SWIFTs. +type SWIFTLister interface { + // List lists all SWIFTs in the indexer. + List(selector labels.Selector) (ret []*v1alpha1.SWIFT, err error) + // SWIFTs returns an object that can list and get SWIFTs. + SWIFTs(namespace string) SWIFTNamespaceLister + SWIFTListerExpansion +} + +// sWIFTLister implements the SWIFTLister interface. +type sWIFTLister struct { + indexer cache.Indexer +} + +// NewSWIFTLister returns a new SWIFTLister. +func NewSWIFTLister(indexer cache.Indexer) SWIFTLister { + return &sWIFTLister{indexer: indexer} +} + +// List lists all SWIFTs in the indexer. +func (s *sWIFTLister) List(selector labels.Selector) (ret []*v1alpha1.SWIFT, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.SWIFT)) + }) + return ret, err +} + +// SWIFTs returns an object that can list and get SWIFTs. +func (s *sWIFTLister) SWIFTs(namespace string) SWIFTNamespaceLister { + return sWIFTNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// SWIFTNamespaceLister helps list and get SWIFTs. +type SWIFTNamespaceLister interface { + // List lists all SWIFTs in the indexer for a given namespace. + List(selector labels.Selector) (ret []*v1alpha1.SWIFT, err error) + // Get retrieves the SWIFT from the indexer for a given namespace and name. + Get(name string) (*v1alpha1.SWIFT, error) + SWIFTNamespaceListerExpansion +} + +// sWIFTNamespaceLister implements the SWIFTNamespaceLister +// interface. +type sWIFTNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all SWIFTs in the indexer for a given namespace. +func (s sWIFTNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.SWIFT, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.SWIFT)) + }) + return ret, err +} + +// Get retrieves the SWIFT from the indexer for a given namespace and name. +func (s sWIFTNamespaceLister) Get(name string) (*v1alpha1.SWIFT, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("swift"), name) + } + return obj.(*v1alpha1.SWIFT), nil +} diff --git a/pkg/operator/edgefs/cluster/controller.go b/pkg/operator/edgefs/cluster/controller.go index 997836e7e..1dd5d19a4 100644 --- a/pkg/operator/edgefs/cluster/controller.go +++ b/pkg/operator/edgefs/cluster/controller.go @@ -33,6 +33,7 @@ import ( "github.com/rook/rook/pkg/operator/edgefs/nfs" "github.com/rook/rook/pkg/operator/edgefs/s3" "github.com/rook/rook/pkg/operator/edgefs/s3x" + "github.com/rook/rook/pkg/operator/edgefs/swift" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -194,6 +195,15 @@ func (c *ClusterController) onAdd(obj interface{}) { cluster.ownerRef) S3Controller.StartWatch(cluster.Namespace, cluster.stopCh) + // Start SWIFT service CRD watcher + SWIFTController := swift.NewSWIFTController(c.context, c.containerImage, + isHostNetworkDefined(cluster.Spec.Network), + cluster.Spec.DataDirHostPath, cluster.Spec.DataVolumeSize, + edgefsv1alpha1.GetTargetPlacement(cluster.Spec.Placement), + cluster.Spec.Resources, + cluster.ownerRef) + SWIFTController.StartWatch(cluster.Namespace, cluster.stopCh) + // Start S3X service CRD watcher S3XController := s3x.NewS3XController(c.context, c.containerImage, isHostNetworkDefined(cluster.Spec.Network), diff --git a/pkg/operator/edgefs/cluster/mgr/mgr.go b/pkg/operator/edgefs/cluster/mgr/mgr.go index 5d4848308..b1342bdcd 100644 --- a/pkg/operator/edgefs/cluster/mgr/mgr.go +++ b/pkg/operator/edgefs/cluster/mgr/mgr.go @@ -362,6 +362,10 @@ func (c *Cluster) uiContainer(name string, containerImage string) v1.Container { Name: "API_ENDPOINT", Value: "http://0.0.0.0:8080", }, + { + Name: "K8S_NAMESPACE", + Value: c.Namespace, + }, }, SecurityContext: securityContext, Resources: c.resources, @@ -419,6 +423,10 @@ func (c *Cluster) mgmtContainer(name string, containerImage string) v1.Container Name: "DEBUG", Value: "alert,error,info", }, + { + Name: "K8S_NAMESPACE", + Value: c.Namespace, + }, { Name: "HOST_HOSTNAME", ValueFrom: &v1.EnvVarSource{ @@ -486,6 +494,10 @@ func (c *Cluster) mgrContainer(name string, containerImage string) v1.Container Name: "CCOW_LOG_LEVEL", Value: "5", }, + { + Name: "K8S_NAMESPACE", + Value: c.Namespace, + }, { Name: "HOST_HOSTNAME", ValueFrom: &v1.EnvVarSource{ diff --git a/pkg/operator/edgefs/s3/s3.go b/pkg/operator/edgefs/s3/s3.go index 20d950228..973fe1152 100644 --- a/pkg/operator/edgefs/s3/s3.go +++ b/pkg/operator/edgefs/s3/s3.go @@ -41,7 +41,7 @@ const ( stateVolumeFolder = ".state" etcVolumeFolder = ".etc" defaultPort = 9982 - defaultSecurePort = 8443 + defaultSecurePort = 9443 ) // Start the S3 manager diff --git a/pkg/operator/edgefs/s3x/s3x.go b/pkg/operator/edgefs/s3x/s3x.go index 067339075..7999d0eef 100644 --- a/pkg/operator/edgefs/s3x/s3x.go +++ b/pkg/operator/edgefs/s3x/s3x.go @@ -38,8 +38,8 @@ const ( dataVolumeName = "edgefs-datadir" stateVolumeFolder = ".state" etcVolumeFolder = ".etc" - defaultPort = 3000 - defaultSecurePort = 3001 + defaultPort = 4000 + defaultSecurePort = 4443 ) // Start the rgw manager diff --git a/pkg/operator/edgefs/swift/controller.go b/pkg/operator/edgefs/swift/controller.go new file mode 100644 index 000000000..b02263551 --- /dev/null +++ b/pkg/operator/edgefs/swift/controller.go @@ -0,0 +1,171 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// scale-out, multi-cloud OpenStack/SWIFT services controller +package swift + +import ( + "fmt" + "reflect" + + "github.com/coreos/pkg/capnslog" + opkit "github.com/rook/operator-kit" + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/rook/rook/pkg/clusterd" + "k8s.io/api/core/v1" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" +) + +const ( + customResourceName = "swift" + customResourceNamePlural = "swifts" +) + +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-swift") + +// SWIFTResource represents the swift custom resource +var SWIFTResource = opkit.CustomResource{ + Name: customResourceName, + Plural: customResourceNamePlural, + Group: edgefsv1alpha1.CustomResourceGroup, + Version: edgefsv1alpha1.Version, + Scope: apiextensionsv1beta1.NamespaceScoped, + Kind: reflect.TypeOf(edgefsv1alpha1.SWIFT{}).Name(), +} + +// SWIFTController represents a controller object for swift custom resources +type SWIFTController struct { + context *clusterd.Context + rookImage string + hostNetwork bool + dataDirHostPath string + dataVolumeSize resource.Quantity + placement rookalpha.Placement + resources v1.ResourceRequirements + ownerRef metav1.OwnerReference +} + +// NewSWIFTController create controller for watching SWIFT custom resources created +func NewSWIFTController( + context *clusterd.Context, rookImage string, + hostNetwork bool, + dataDirHostPath string, + dataVolumeSize resource.Quantity, + placement rookalpha.Placement, + resources v1.ResourceRequirements, + ownerRef metav1.OwnerReference, +) *SWIFTController { + return &SWIFTController{ + context: context, + rookImage: rookImage, + hostNetwork: hostNetwork, + dataDirHostPath: dataDirHostPath, + dataVolumeSize: dataVolumeSize, + placement: placement, + resources: resources, + ownerRef: ownerRef, + } +} + +// StartWatch watches for instances of SWIFT custom resources and acts on them +func (c *SWIFTController) StartWatch(namespace string, stopCh chan struct{}) error { + + resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{ + AddFunc: c.onAdd, + UpdateFunc: c.onUpdate, + DeleteFunc: c.onDelete, + } + + logger.Infof("start watching swift resources in namespace %s", namespace) + watcher := opkit.NewWatcher(SWIFTResource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient()) + go watcher.Watch(&edgefsv1alpha1.SWIFT{}, stopCh) + + return nil +} + +func (c *SWIFTController) onAdd(obj interface{}) { + swift, err := getSWIFTObject(obj) + if err != nil { + logger.Errorf("failed to get swift object: %+v", err) + return + } + + if err = c.CreateService(*swift, c.serviceOwners(swift)); err != nil { + logger.Errorf("failed to create swift %s. %+v", swift.Name, err) + } +} + +func (c *SWIFTController) onUpdate(oldObj, newObj interface{}) { + oldService, err := getSWIFTObject(oldObj) + if err != nil { + logger.Errorf("failed to get old swift object: %+v", err) + return + } + newService, err := getSWIFTObject(newObj) + if err != nil { + logger.Errorf("failed to get new swift object: %+v", err) + return + } + + if !serviceChanged(oldService.Spec, newService.Spec) { + logger.Debugf("swift %s did not change", newService.Name) + return + } + + logger.Infof("applying swift %s changes", newService.Name) + if err = c.UpdateService(*newService, c.serviceOwners(newService)); err != nil { + logger.Errorf("failed to create (modify) swift %s. %+v", newService.Name, err) + } +} + +func (c *SWIFTController) onDelete(obj interface{}) { + swift, err := getSWIFTObject(obj) + if err != nil { + logger.Errorf("failed to get swift object: %+v", err) + return + } + + if err = c.DeleteService(*swift); err != nil { + logger.Errorf("failed to delete swift %s. %+v", swift.Name, err) + } +} + +func (c *SWIFTController) serviceOwners(service *edgefsv1alpha1.SWIFT) []metav1.OwnerReference { + // Only set the cluster crd as the owner of the SWIFT resources. + // If the SWIFT crd is deleted, the operator will explicitly remove the SWIFT resources. + // If the SWIFT crd still exists when the cluster crd is deleted, this will make sure the SWIFT + // resources are cleaned up. + return []metav1.OwnerReference{c.ownerRef} +} + +func serviceChanged(oldService, newService edgefsv1alpha1.SWIFTSpec) bool { + return false +} + +func getSWIFTObject(obj interface{}) (swift *edgefsv1alpha1.SWIFT, err error) { + var ok bool + swift, ok = obj.(*edgefsv1alpha1.SWIFT) + if ok { + // the swift object is of the latest type, simply return it + return swift.DeepCopy(), nil + } + + return nil, fmt.Errorf("not a known swift object: %+v", obj) +} diff --git a/pkg/operator/edgefs/swift/swift.go b/pkg/operator/edgefs/swift/swift.go new file mode 100644 index 000000000..5aee2364e --- /dev/null +++ b/pkg/operator/edgefs/swift/swift.go @@ -0,0 +1,379 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package SWIFT for the Edgefs manager. +package swift + +import ( + "fmt" + "strings" + + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" + apps "k8s.io/api/apps/v1" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + appName = "rook-edgefs-swift" + + /* Volumes definitions */ + defaultSWIFTImage = "edgefs/edgefs-restapi" + sslCertVolumeName = "ssl-cert-volume" + sslMountPath = "/opt/nedge/etc/ssl/" + dataVolumeName = "edgefs-datadir" + stateVolumeFolder = ".state" + etcVolumeFolder = ".etc" + defaultPort = 9981 + defaultSecurePort = 443 +) + +// Start the SWIFT manager +func (c *SWIFTController) CreateService(s edgefsv1alpha1.SWIFT, ownerRefs []metav1.OwnerReference) error { + return c.CreateOrUpdate(s, false, ownerRefs) +} + +func (c *SWIFTController) UpdateService(s edgefsv1alpha1.SWIFT, ownerRefs []metav1.OwnerReference) error { + return c.CreateOrUpdate(s, true, ownerRefs) +} + +// Start the swift instance +func (c *SWIFTController) CreateOrUpdate(s edgefsv1alpha1.SWIFT, update bool, ownerRefs []metav1.OwnerReference) error { + logger.Debugf("starting update=%v service=%s", update, s.Name) + + // validate SWIFT service settings + if err := validateService(c.context, s); err != nil { + return fmt.Errorf("invalid SWIFT service %s arguments. %+v", s.Name, err) + } + + //check http settings + if s.Spec.Port == 0 { + s.Spec.Port = defaultPort + } + + if s.Spec.SecurePort == 0 { + s.Spec.SecurePort = defaultSecurePort + } + + // all rest APIs coming from edgefs-restapi image, that + // includes mgmt, s3, s3s and swift + imageArgs := "swift" + rookImage := defaultSWIFTImage + + var rookImageVer string + rookImageComponents := strings.Split(rookImage, ":") + if len(rookImageComponents) == 2 { + rookImageVer = rookImageComponents[1] + } else { + rookImageVer = "latest" + } + + // check if SWIFT service already exists + exists, err := serviceExists(c.context, s) + if err == nil && exists { + if !update { + logger.Infof("SWIFT service %s exists in namespace %s", s.Name, s.Namespace) + return nil + } + logger.Infof("SWIFT service %s exists in namespace %s. checking for updates", s.Name, s.Namespace) + } + + // start the deployment + deployment := c.makeDeployment(s.Name, s.Namespace, rookImage+":"+rookImageVer, imageArgs, s.Spec) + if _, err := c.context.Clientset.Apps().Deployments(s.Namespace).Create(deployment); err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create %s deployment. %+v", appName, err) + } + logger.Infof("%s deployment already exists", appName) + } else { + logger.Infof("%s deployment started", appName) + } + + // create the swift service + service := c.makeSWIFTService(instanceName(s.Name), s.Name, s.Namespace, s.Spec) + if _, err := c.context.Clientset.CoreV1().Services(s.Namespace).Create(service); err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create swift service. %+v", err) + } + logger.Infof("swift service %s already exists", service) + } else { + logger.Infof("swift service %s started", service) + } + + return nil +} + +func (c *SWIFTController) makeSWIFTService(name, svcname, namespace string, swiftSpec edgefsv1alpha1.SWIFTSpec) *v1.Service { + labels := getLabels(name, svcname, namespace) + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Spec: v1.ServiceSpec{ + Selector: labels, + Type: v1.ServiceTypeNodePort, + Ports: []v1.ServicePort{ + {Name: "grpc", Port: 49000, Protocol: v1.ProtocolTCP}, + {Name: "port", Port: int32(swiftSpec.Port), Protocol: v1.ProtocolTCP}, + {Name: "secure-port", Port: int32(swiftSpec.SecurePort), Protocol: v1.ProtocolTCP}, + }, + }, + } + + k8sutil.SetOwnerRef(c.context.Clientset, namespace, &svc.ObjectMeta, &c.ownerRef) + return svc +} + +func (c *SWIFTController) makeDeployment(svcname, namespace, rookImage, imageArgs string, swiftSpec edgefsv1alpha1.SWIFTSpec) *apps.Deployment { + + name := instanceName(svcname) + volumes := []v1.Volume{} + + // add ssl certificate volume if defined + if len(swiftSpec.SSLCertificateRef) > 0 { + volumes = append(volumes, v1.Volume{ + Name: sslCertVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: swiftSpec.SSLCertificateRef, + Items: []v1.KeyToPath{ + { + Key: "sslkey", + Path: "ssl.key", + }, + { + Key: "sslcert", + Path: "ssl.crt", + }, + }, + }, + }, + }) + } + + if c.dataVolumeSize.Value() > 0 { + // dataVolume case + volumes = append(volumes, v1.Volume{ + Name: dataVolumeName, + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: dataVolumeName, + }, + }, + }) + } else { + // dataDir case + volumes = append(volumes, v1.Volume{ + Name: dataVolumeName, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: c.dataDirHostPath, + }, + }, + }) + } + + podSpec := v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: getLabels(name, svcname, namespace), + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{c.swiftContainer(svcname, name, rookImage, imageArgs, swiftSpec)}, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: volumes, + HostIPC: true, + HostNetwork: c.hostNetwork, + NodeSelector: map[string]string{namespace: "cluster"}, + }, + } + if c.hostNetwork { + podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } + + // apply current SWIFT CRD options to pod's specification + swiftSpec.Placement.ApplyToPodSpec(&podSpec.Spec) + + d := &apps.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: apps.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: podSpec.Labels, + }, + Template: podSpec, + Replicas: &swiftSpec.Instances, + }, + } + k8sutil.SetOwnerRef(c.context.Clientset, namespace, &d.ObjectMeta, &c.ownerRef) + return d +} + +func (c *SWIFTController) swiftContainer(svcname, name, containerImage, args string, swiftSpec edgefsv1alpha1.SWIFTSpec) v1.Container { + + runAsUser := int64(0) + readOnlyRootFilesystem := false + securityContext := &v1.SecurityContext{ + RunAsUser: &runAsUser, + ReadOnlyRootFilesystem: &readOnlyRootFilesystem, + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"}, + }, + } + + volumeMounts := []v1.VolumeMount{ + { + Name: dataVolumeName, + MountPath: "/opt/nedge/etc.target", + SubPath: etcVolumeFolder, + }, + { + Name: dataVolumeName, + MountPath: "/opt/nedge/var/run", + SubPath: stateVolumeFolder, + }, + } + + if len(swiftSpec.SSLCertificateRef) > 0 { + volumeMounts = append(volumeMounts, v1.VolumeMount{Name: sslCertVolumeName, MountPath: sslMountPath}) + } + + return v1.Container{ + Name: name, + Image: containerImage, + ImagePullPolicy: v1.PullAlways, + Args: []string{args}, + Env: []v1.EnvVar{ + { + Name: "CCOW_LOG_LEVEL", + Value: "5", + }, + { + Name: "CCOW_SVCNAME", + Value: svcname, + }, + { + Name: "DEBUG", + Value: "alert,error,info", + }, + { + Name: "HOST_HOSTNAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: "EFSSWIFT_HTTP_PORT", + Value: fmt.Sprint(swiftSpec.Port), + }, + { + Name: "EFSSWIFT_HTTPS_PORT", + Value: fmt.Sprint(swiftSpec.SecurePort), + }, + }, + SecurityContext: securityContext, + Resources: swiftSpec.Resources, + Ports: []v1.ContainerPort{ + {Name: "grpc", ContainerPort: 49000, Protocol: v1.ProtocolTCP}, + {Name: "port", ContainerPort: int32(swiftSpec.Port), Protocol: v1.ProtocolTCP}, + {Name: "secure-port", ContainerPort: int32(swiftSpec.SecurePort), Protocol: v1.ProtocolTCP}, + }, + VolumeMounts: volumeMounts, + } +} + +// Delete SWIFT service and possibly some artifacts. +func (c *SWIFTController) DeleteService(s edgefsv1alpha1.SWIFT) error { + // check if service exists + exists, err := serviceExists(c.context, s) + if err != nil { + return fmt.Errorf("failed to detect if there is a SWIFT service to delete. %+v", err) + } + if !exists { + logger.Infof("SWIFT service %s does not exist in namespace %s", s.Name, s.Namespace) + return nil + } + + logger.Infof("Deleting SWIFT service %s from namespace %s", s.Name, s.Namespace) + + var gracePeriod int64 + propagation := metav1.DeletePropagationForeground + options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} + + // Delete the swift service + err = c.context.Clientset.CoreV1().Services(s.Namespace).Delete(instanceName(s.Name), options) + if err != nil && !errors.IsNotFound(err) { + logger.Warningf("failed to delete SWIFT service. %+v", err) + } + + // Make a best effort to delete the SWIFT pods + err = k8sutil.DeleteDeployment(c.context.Clientset, s.Namespace, instanceName(s.Name)) + if err != nil { + logger.Warningf(err.Error()) + } + + logger.Infof("Completed deleting SWIFT service %s", s.Name) + return nil +} + +func getLabels(name, svcname, namespace string) map[string]string { + return map[string]string{ + k8sutil.AppAttr: appName, + k8sutil.ClusterAttr: namespace, + "edgefs_svcname": svcname, + "edgefs_svctype": "swift", + } +} + +// Validate the SWIFT arguments +func validateService(context *clusterd.Context, s edgefsv1alpha1.SWIFT) error { + if s.Name == "" { + return fmt.Errorf("missing name") + } + if s.Namespace == "" { + return fmt.Errorf("missing namespace") + } + + return nil +} + +func instanceName(svcname string) string { + return fmt.Sprintf("%s-%s", appName, svcname) +} + +// Check if the SWIFT service exists +func serviceExists(context *clusterd.Context, s edgefsv1alpha1.SWIFT) (bool, error) { + _, err := context.Clientset.Apps().Deployments(s.Namespace).Get(instanceName(s.Name), metav1.GetOptions{}) + if err == nil { + // the deployment was found + return true, nil + } + if err != nil && !errors.IsNotFound(err) { + return false, err + } + + // not found + return false, nil +}