Initial commit
This commit is contained in:
1 parent
09fe9553d7
commit
d45d84462d
32 files changed
+2401
-2
No files matched your search
@@ -0,0 +1,188 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/handler"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
mikrotikv1alpha1 "github.com/feedc0de/mikrotik-operator/api/v1alpha1"
|
||||
"github.com/feedc0de/mikrotik-operator/internal/routeros"
|
||||
)
|
||||
|
||||
const (
|
||||
syncInterval = 2 * time.Minute
|
||||
routerFinalizer = "mikrotik.brunner.ninja/nat-cleanup"
|
||||
)
|
||||
|
||||
type RouterClientFactory func(endpoint, username, password string, tlsConfig routeros.TLSConfig) (routeros.API, error)
|
||||
|
||||
type MikroTikRouterReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
ClientFactory RouterClientFactory
|
||||
}
|
||||
|
||||
func (r *MikroTikRouterReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
|
||||
var router mikrotikv1alpha1.MikroTikRouter
|
||||
if err := r.Get(ctx, request.NamespacedName, &router); err != nil {
|
||||
return ctrl.Result{}, client.IgnoreNotFound(err)
|
||||
}
|
||||
if !router.DeletionTimestamp.IsZero() {
|
||||
return r.finalize(ctx, &router)
|
||||
}
|
||||
if !controllerutil.ContainsFinalizer(&router, routerFinalizer) {
|
||||
controllerutil.AddFinalizer(&router, routerFinalizer)
|
||||
if err := r.Update(ctx, &router); err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("add cleanup finalizer: %w", err)
|
||||
}
|
||||
return ctrl.Result{Requeue: true}, nil
|
||||
}
|
||||
|
||||
api, err := r.routerClient(ctx, &router)
|
||||
if err != nil {
|
||||
if statusErr := r.setStatus(ctx, &router, 0, metav1.ConditionFalse, "ConfigurationError", err.Error()); statusErr != nil {
|
||||
return ctrl.Result{}, errors.Join(err, statusErr)
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: syncInterval}, nil
|
||||
}
|
||||
|
||||
var serviceList corev1.ServiceList
|
||||
if err := r.List(ctx, &serviceList); err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("list Services: %w", err)
|
||||
}
|
||||
desired, preserve, issues := DesiredRules(&router, serviceList.Items)
|
||||
managedRules, err := api.Sync(ctx, desired, preserve)
|
||||
if err != nil {
|
||||
if statusErr := r.setStatus(ctx, &router, router.Status.ManagedRules, metav1.ConditionFalse, "RouterAPIError", err.Error()); statusErr != nil {
|
||||
return ctrl.Result{}, errors.Join(err, statusErr)
|
||||
}
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if len(issues) > 0 {
|
||||
message := issueMessage(issues)
|
||||
if err := r.setStatus(ctx, &router, int32(managedRules), metav1.ConditionFalse, "ServiceConfigurationError", message); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: syncInterval}, nil
|
||||
}
|
||||
if err := r.setStatus(ctx, &router, int32(managedRules), metav1.ConditionTrue, "Synchronized", "RouterOS NAT rules are synchronized"); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: syncInterval}, nil
|
||||
}
|
||||
|
||||
func (r *MikroTikRouterReconciler) finalize(ctx context.Context, router *mikrotikv1alpha1.MikroTikRouter) (ctrl.Result, error) {
|
||||
if !controllerutil.ContainsFinalizer(router, routerFinalizer) {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
api, err := r.routerClient(ctx, router)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("initialize RouterOS client for cleanup: %w", err)
|
||||
}
|
||||
if _, err := api.Sync(ctx, nil, map[string]struct{}{}); err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("remove managed NAT rules during router deletion: %w", err)
|
||||
}
|
||||
controllerutil.RemoveFinalizer(router, routerFinalizer)
|
||||
if err := r.Update(ctx, router); err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("remove cleanup finalizer: %w", err)
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
func (r *MikroTikRouterReconciler) routerClient(ctx context.Context, router *mikrotikv1alpha1.MikroTikRouter) (routeros.API, error) {
|
||||
ref := router.Spec.CredentialsSecretRef
|
||||
var credentials corev1.Secret
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: ref.Namespace, Name: ref.Name}, &credentials); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return nil, fmt.Errorf("credentials Secret %s/%s does not exist", ref.Namespace, ref.Name)
|
||||
}
|
||||
return nil, fmt.Errorf("read credentials Secret: %w", err)
|
||||
}
|
||||
username, usernameOK := credentials.Data["username"]
|
||||
password, passwordOK := credentials.Data["password"]
|
||||
if !usernameOK || !passwordOK {
|
||||
return nil, errors.New("credentials Secret must contain username and password keys")
|
||||
}
|
||||
if len(username) == 0 || len(password) == 0 {
|
||||
return nil, errors.New("credentials Secret username and password must not be empty")
|
||||
}
|
||||
|
||||
tlsConfig := routeros.TLSConfig{InsecureSkipVerify: router.Spec.TLS.InsecureSkipVerify}
|
||||
if caRef := router.Spec.TLS.CASecretRef; caRef != nil {
|
||||
var caSecret corev1.Secret
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: caRef.Namespace, Name: caRef.Name}, &caSecret); err != nil {
|
||||
return nil, fmt.Errorf("read CA Secret %s/%s: %w", caRef.Namespace, caRef.Name, err)
|
||||
}
|
||||
var ok bool
|
||||
tlsConfig.CABundle, ok = caSecret.Data["ca.crt"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("CA Secret %s/%s must contain ca.crt", caRef.Namespace, caRef.Name)
|
||||
}
|
||||
}
|
||||
return r.ClientFactory(router.Spec.Endpoint, string(username), string(password), tlsConfig)
|
||||
}
|
||||
|
||||
func (r *MikroTikRouterReconciler) setStatus(ctx context.Context, router *mikrotikv1alpha1.MikroTikRouter, count int32, conditionStatus metav1.ConditionStatus, reason, message string) error {
|
||||
now := metav1.Now()
|
||||
router.Status.ObservedGeneration = router.Generation
|
||||
router.Status.ManagedRules = count
|
||||
router.Status.LastSyncTime = &now
|
||||
meta.SetStatusCondition(&router.Status.Conditions, metav1.Condition{
|
||||
Type: "Ready", Status: conditionStatus, Reason: reason, Message: message,
|
||||
ObservedGeneration: router.Generation,
|
||||
})
|
||||
return r.Status().Update(ctx, router)
|
||||
}
|
||||
|
||||
func issueMessage(issues []RuleIssue) string {
|
||||
const maxIssues = 3
|
||||
messages := make([]string, 0, min(len(issues), maxIssues))
|
||||
for i, issue := range issues {
|
||||
if i == maxIssues {
|
||||
break
|
||||
}
|
||||
messages = append(messages, issue.Service+": "+issue.Message)
|
||||
}
|
||||
if len(issues) > maxIssues {
|
||||
messages = append(messages, fmt.Sprintf("and %d more", len(issues)-maxIssues))
|
||||
}
|
||||
return strings.Join(messages, "; ")
|
||||
}
|
||||
|
||||
func (r *MikroTikRouterReconciler) enqueueAllRouters(ctx context.Context, _ client.Object) []reconcile.Request {
|
||||
var routers mikrotikv1alpha1.MikroTikRouterList
|
||||
if err := r.List(ctx, &routers); err != nil {
|
||||
return nil
|
||||
}
|
||||
requests := make([]reconcile.Request, 0, len(routers.Items))
|
||||
for i := range routers.Items {
|
||||
requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{Name: routers.Items[i].Name}})
|
||||
}
|
||||
return requests
|
||||
}
|
||||
|
||||
func (r *MikroTikRouterReconciler) SetupWithManager(manager ctrl.Manager) error {
|
||||
if r.ClientFactory == nil {
|
||||
r.ClientFactory = func(endpoint, username, password string, tlsConfig routeros.TLSConfig) (routeros.API, error) {
|
||||
return routeros.NewClient(endpoint, username, password, tlsConfig)
|
||||
}
|
||||
}
|
||||
return ctrl.NewControllerManagedBy(manager).
|
||||
For(&mikrotikv1alpha1.MikroTikRouter{}).
|
||||
Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(r.enqueueAllRouters)).
|
||||
Complete(r)
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
|
||||
mikrotikv1alpha1 "github.com/feedc0de/mikrotik-operator/api/v1alpha1"
|
||||
"github.com/feedc0de/mikrotik-operator/internal/routeros"
|
||||
)
|
||||
|
||||
const AnnotationPrefix = "mikrotik.brunner.ninja/"
|
||||
|
||||
const (
|
||||
AnnotationRouter = AnnotationPrefix + "router"
|
||||
AnnotationServicePort = AnnotationPrefix + "service-port"
|
||||
AnnotationDestinationPort = AnnotationPrefix + "dst-port"
|
||||
AnnotationDestinationIP = AnnotationPrefix + "dst-address"
|
||||
AnnotationDestinationList = AnnotationPrefix + "dst-address-list"
|
||||
AnnotationInInterface = AnnotationPrefix + "in-interface"
|
||||
AnnotationInInterfaceList = AnnotationPrefix + "in-interface-list"
|
||||
AnnotationSourceAddress = AnnotationPrefix + "src-address"
|
||||
AnnotationSourceList = AnnotationPrefix + "src-address-list"
|
||||
)
|
||||
|
||||
type RuleIssue struct {
|
||||
Service string
|
||||
Message string
|
||||
}
|
||||
|
||||
func DesiredRules(router *mikrotikv1alpha1.MikroTikRouter, services []corev1.Service) ([]routeros.Rule, map[string]struct{}, []RuleIssue) {
|
||||
desired := make([]routeros.Rule, 0)
|
||||
preserve := make(map[string]struct{})
|
||||
issues := make([]RuleIssue, 0)
|
||||
for i := range services {
|
||||
service := &services[i]
|
||||
if service.Annotations[AnnotationRouter] != router.Name || !namespaceAllowed(router.Spec.AllowedNamespaces, service.Namespace) {
|
||||
continue
|
||||
}
|
||||
comment := routeros.ManagedComment(service.Namespace, service.Name)
|
||||
rule, err := ruleForService(service, comment)
|
||||
if err != nil {
|
||||
preserve[comment] = struct{}{}
|
||||
issues = append(issues, RuleIssue{Service: service.Namespace + "/" + service.Name, Message: err.Error()})
|
||||
continue
|
||||
}
|
||||
desired = append(desired, rule)
|
||||
}
|
||||
return desired, preserve, issues
|
||||
}
|
||||
|
||||
func namespaceAllowed(allowed []string, namespace string) bool {
|
||||
for _, item := range allowed {
|
||||
if item == "*" || item == namespace {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ruleForService(service *corev1.Service, comment string) (routeros.Rule, error) {
|
||||
if service.Spec.Type != corev1.ServiceTypeLoadBalancer {
|
||||
return routeros.Rule{}, fmt.Errorf("Service must have type LoadBalancer")
|
||||
}
|
||||
port, err := selectServicePort(service)
|
||||
if err != nil {
|
||||
return routeros.Rule{}, err
|
||||
}
|
||||
toAddress := ""
|
||||
for _, ingress := range service.Status.LoadBalancer.Ingress {
|
||||
address, parseErr := netip.ParseAddr(ingress.IP)
|
||||
if parseErr == nil && address.Is4() {
|
||||
toAddress = address.String()
|
||||
break
|
||||
}
|
||||
}
|
||||
if toAddress == "" {
|
||||
return routeros.Rule{}, fmt.Errorf("waiting for an IPv4 address in status.loadBalancer.ingress")
|
||||
}
|
||||
|
||||
destinationPort := service.Annotations[AnnotationDestinationPort]
|
||||
if destinationPort == "" {
|
||||
destinationPort = strconv.Itoa(int(port.Port))
|
||||
}
|
||||
parsedDestinationPort, err := strconv.ParseUint(destinationPort, 10, 16)
|
||||
if err != nil || parsedDestinationPort == 0 {
|
||||
return routeros.Rule{}, fmt.Errorf("%s must be a port from 1 through 65535", AnnotationDestinationPort)
|
||||
}
|
||||
|
||||
return routeros.Rule{
|
||||
Comment: comment, Protocol: strings.ToLower(string(port.Protocol)),
|
||||
DestinationPort: destinationPort,
|
||||
DestinationIP: service.Annotations[AnnotationDestinationIP],
|
||||
DestinationList: service.Annotations[AnnotationDestinationList],
|
||||
InInterface: service.Annotations[AnnotationInInterface],
|
||||
InInterfaceList: service.Annotations[AnnotationInInterfaceList],
|
||||
SourceAddress: service.Annotations[AnnotationSourceAddress],
|
||||
SourceList: service.Annotations[AnnotationSourceList],
|
||||
ToAddress: toAddress, ToPort: strconv.Itoa(int(port.Port)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func selectServicePort(service *corev1.Service) (corev1.ServicePort, error) {
|
||||
selector := service.Annotations[AnnotationServicePort]
|
||||
if selector == "" {
|
||||
if len(service.Spec.Ports) != 1 {
|
||||
return corev1.ServicePort{}, fmt.Errorf("%s is required when the Service has more than one port", AnnotationServicePort)
|
||||
}
|
||||
return service.Spec.Ports[0], nil
|
||||
}
|
||||
for _, port := range service.Spec.Ports {
|
||||
if port.Name == selector || strconv.Itoa(int(port.Port)) == selector {
|
||||
return port, nil
|
||||
}
|
||||
}
|
||||
return corev1.ServicePort{}, fmt.Errorf("%s %q does not match a Service port name or number", AnnotationServicePort, selector)
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
mikrotikv1alpha1 "github.com/feedc0de/mikrotik-operator/api/v1alpha1"
|
||||
"github.com/feedc0de/mikrotik-operator/internal/routeros"
|
||||
)
|
||||
|
||||
func TestDesiredRules(t *testing.T) {
|
||||
router := &mikrotikv1alpha1.MikroTikRouter{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "edge"},
|
||||
Spec: mikrotikv1alpha1.MikroTikRouterSpec{AllowedNamespaces: []string{"games"}},
|
||||
}
|
||||
service := corev1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: "games", Name: "factorio",
|
||||
Annotations: map[string]string{
|
||||
AnnotationRouter: "edge", AnnotationServicePort: "game",
|
||||
AnnotationDestinationPort: "40000", AnnotationInInterfaceList: "WAN",
|
||||
},
|
||||
},
|
||||
Spec: corev1.ServiceSpec{
|
||||
Type: corev1.ServiceTypeLoadBalancer,
|
||||
Ports: []corev1.ServicePort{
|
||||
{Name: "game", Port: 34197, Protocol: corev1.ProtocolUDP},
|
||||
{Name: "rcon", Port: 27015, Protocol: corev1.ProtocolTCP},
|
||||
},
|
||||
},
|
||||
Status: corev1.ServiceStatus{LoadBalancer: corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{
|
||||
{IP: "2001:db8::1"}, {IP: "192.168.0.17"},
|
||||
}}},
|
||||
}
|
||||
|
||||
rules, preserve, issues := DesiredRules(router, []corev1.Service{service})
|
||||
if len(issues) != 0 {
|
||||
t.Fatalf("unexpected issues: %#v", issues)
|
||||
}
|
||||
if len(preserve) != 0 {
|
||||
t.Fatalf("unexpected preserve set: %#v", preserve)
|
||||
}
|
||||
if len(rules) != 1 {
|
||||
t.Fatalf("got %d rules, want 1", len(rules))
|
||||
}
|
||||
want := routeros.Rule{
|
||||
Comment: routeros.ManagedComment("games", "factorio"), Protocol: "udp",
|
||||
DestinationPort: "40000", InInterfaceList: "WAN",
|
||||
ToAddress: "192.168.0.17", ToPort: "34197",
|
||||
}
|
||||
if rules[0] != want {
|
||||
t.Fatalf("rule = %#v, want %#v", rules[0], want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDesiredRulesPreservesRuleWhileLoadBalancerIsPending(t *testing.T) {
|
||||
router := &mikrotikv1alpha1.MikroTikRouter{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "edge"},
|
||||
Spec: mikrotikv1alpha1.MikroTikRouterSpec{AllowedNamespaces: []string{"games"}},
|
||||
}
|
||||
service := corev1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "games", Name: "factorio", Annotations: map[string]string{AnnotationRouter: "edge"}},
|
||||
Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer, Ports: []corev1.ServicePort{{Port: 34197, Protocol: corev1.ProtocolUDP}}},
|
||||
}
|
||||
rules, preserve, issues := DesiredRules(router, []corev1.Service{service})
|
||||
if len(rules) != 0 || len(issues) != 1 {
|
||||
t.Fatalf("rules=%#v issues=%#v", rules, issues)
|
||||
}
|
||||
if _, ok := preserve[routeros.ManagedComment("games", "factorio")]; !ok {
|
||||
t.Fatal("pending rule was not preserved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDesiredRulesDoesNotHonorUnlistedNamespace(t *testing.T) {
|
||||
router := &mikrotikv1alpha1.MikroTikRouter{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "edge"},
|
||||
Spec: mikrotikv1alpha1.MikroTikRouterSpec{AllowedNamespaces: []string{"trusted"}},
|
||||
}
|
||||
service := corev1.Service{ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: "untrusted", Name: "bad", Annotations: map[string]string{AnnotationRouter: "edge"},
|
||||
}}
|
||||
rules, preserve, issues := DesiredRules(router, []corev1.Service{service})
|
||||
if len(rules) != 0 || len(preserve) != 0 || len(issues) != 0 {
|
||||
t.Fatalf("unlisted namespace affected reconciliation: rules=%#v preserve=%#v issues=%#v", rules, preserve, issues)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user