feat(agent): externals as VPCs

api/agent/v1beta1/catalog_types.go

@@ -26,14 +26,14 @@ type CatalogSpec struct {
 
 	// ConnectionSystemIDs stores connection name -> ID, globally unique for the fabric
 	ConnectionIDs map[string]uint32 `json:"connectionIDs,omitempty"`
-	// VPCVNIs stores VPC name -> VPC VNI, globally unique for the fabric
+	// VPCVNIs stores VPC name -> VPC VNI, globally unique for the fabric, it includes Externals too with ext@ prefix
 	VPCVNIs map[string]uint32 `json:"vpcVNIs,omitempty"`
 	// VPCSubnetVNIs stores VPC name -> subnet name -> VPC Subnet VNI, globally unique for the fabric
 	VPCSubnetVNIs map[string]map[string]uint32 `json:"vpcSubnetVNIs,omitempty"`
 
 	// Per redundancy group (or switch if no redundancy group)
 
-	// IRBVLANs stores VPC name -> IRB VLAN ID, unique per redundancy group (or switch)
+	// IRBVLANs stores VPC name -> IRB VLAN ID, unique per redundancy group (or switch), it includes Externals too with ext@ prefix
 	IRBVLANs map[string]uint16 `json:"irbVLANs,omitempty"`
 	// PortChannelIDs stores Connection name -> PortChannel ID, unique per redundancy group (or switch)
 	PortChannelIDs map[string]uint16 `json:"portChannelIDs,omitempty"`

api/vpc/v1beta1/external_types.go

@@ -28,6 +28,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 )
 
+const (
+	VPCInfoExtPrefix = "ext."
+)
+
 var communityCheck = regexp.MustCompile("^(6553[0-5]|655[0-2][0-9]|654[0-9]{2}|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{1,3}|[0-9]):(6553[0-5]|655[0-2][0-9]|654[0-9]{2}|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{1,3}|[0-9])$")
 
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.

api/vpc/v1beta1/vpc_types.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"net"
 	"slices"
+	"strings"
 
 	"github.com/pkg/errors"
 	"go.githedgehog.com/fabric/api/meta"
@@ -336,6 +337,9 @@ func (vpc *VPC) Validate(ctx context.Context, kube kclient.Reader, fabricCfg *me
 	if len(vpc.Name) > 11 {
 		return nil, errors.Errorf("name %s is too long, must be <= 11 characters", vpc.Name)
 	}
+	if strings.HasPrefix(vpc.Name, VPCInfoExtPrefix) {
+		return nil, errors.Errorf("vpc name cannot start with '%s': %s", VPCInfoExtPrefix, vpc.Name)
+	}
 	if vpc.Spec.IPv4Namespace == "" {
 		return nil, errors.Errorf("ipv4Namespace is required")
 	}

cmd/main.go

@@ -195,6 +195,9 @@ func run() error {
 		if err := ctrl.SetupGwVPCSyncReconcilerWith(mgr, cfg, libMngr); err != nil {
 			return fmt.Errorf("setting up gateway vpc sync controller: %w", err)
 		}
+		if err := ctrl.SetupGwExternalSyncReconcilerWith(mgr, cfg, libMngr); err != nil {
+			return fmt.Errorf("setting up gateway external sync controller: %w", err)
+		}
 	}
 
 	if err = connectionwh.SetupWithManager(mgr, cfg); err != nil {

config/crd/bases/agent.githedgehog.com_agents.yaml

@@ -207,7 +207,8 @@ spec:
                     additionalProperties:
                       type: integer
                     description: IRBVLANs stores VPC name -> IRB VLAN ID, unique per
-                      redundancy group (or switch)
+                      redundancy group (or switch), it includes Externals too with
+                      ext@ prefix
                     type: object
                   loopbackWorkaroundLinks:
                     additionalProperties:
@@ -248,7 +249,7 @@ spec:
                       format: int32
                       type: integer
                     description: VPCVNIs stores VPC name -> VPC VNI, globally unique
-                      for the fabric
+                      for the fabric, it includes Externals too with ext@ prefix
                     type: object
                 type: object
               config:

config/crd/bases/agent.githedgehog.com_catalogs.yaml

@@ -57,7 +57,7 @@ spec:
                 additionalProperties:
                   type: integer
                 description: IRBVLANs stores VPC name -> IRB VLAN ID, unique per redundancy
-                  group (or switch)
+                  group (or switch), it includes Externals too with ext@ prefix
                 type: object
               loopbackWorkaroundLinks:
                 additionalProperties:
@@ -98,7 +98,7 @@ spec:
                   format: int32
                   type: integer
                 description: VPCVNIs stores VPC name -> VPC VNI, globally unique for
-                  the fabric
+                  the fabric, it includes Externals too with ext@ prefix
                 type: object
             type: object
           status:

config/rbac/role.yaml

@@ -130,6 +130,13 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - vpc.githedgehog.com
+  resources:
+  - externals/finalizers
+  - vpcs/finalizers
+  verbs:
+  - update
 - apiGroups:
   - vpc.githedgehog.com
   resources:
@@ -142,12 +149,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - vpc.githedgehog.com
-  resources:
-  - vpcs/finalizers
-  verbs:
-  - update
 - apiGroups:
   - wiring.githedgehog.com
   resources:

pkg/agent/dozer/bcm/plan.go

@@ -983,7 +983,9 @@ func planExternals(agent *agentapi.Agent, spec *dozer.Spec) error {
 						ImportVRFs: map[string]*dozer.SpecVRFBGPImportVRF{},
 					},
 					L2VPNEVPN: dozer.SpecVRFBGPL2VPNEVPN{
-						Enabled: agent.IsSpineLeaf(),
+						Enabled:                       agent.IsSpineLeaf(),
+						AdvertiseIPv4Unicast:          pointer.To(true),
+						AdvertiseIPv4UnicastRouteMaps: []string{extInboundRouteMapName(externalName)},
 					},
 					Neighbors: map[string]*dozer.SpecVRFBGPNeighbor{},
 				},
@@ -1015,16 +1017,11 @@ func planExternals(agent *agentapi.Agent, spec *dozer.Spec) error {
 			},
 		}
 
-		prefList := extOutboundPrefixList(externalName)
-		spec.PrefixLists[prefList] = &dozer.SpecPrefixList{
-			Prefixes: map[uint32]*dozer.SpecPrefixListEntry{},
-		}
-
 		spec.RouteMaps[extOutboundRouteMapName(externalName)] = &dozer.SpecRouteMap{
 			Statements: map[string]*dozer.SpecRouteMapStatement{
 				"10": {
 					Conditions: dozer.SpecRouteMapConditions{
-						MatchPrefixList: pointer.To(prefList),
+						MatchPrefixList: pointer.To(ipnsSubnetsPrefixListName(external.IPv4Namespace)),
 					},
 					SetCommunities: []string{external.OutboundCommunity},
 					Result:         dozer.SpecRouteMapResultAccept,
@@ -1034,6 +1031,29 @@ func planExternals(agent *agentapi.Agent, spec *dozer.Spec) error {
 				},
 			},
 		}
+
+		irbVLAN := agent.Spec.Catalog.IRBVLANs[librarian.ReqForExt(externalName)]
+		extVNI := agent.Spec.Catalog.VPCVNIs[librarian.ReqForExt(externalName)]
+		if irbVLAN == 0 {
+			return fmt.Errorf("IRB VLAN for external %s not found in catalog", externalName) //nolint:goerr113
+		}
+		if extVNI == 0 {
+			return fmt.Errorf("VNI for external %s not found in catalog", externalName) //nolint:goerr113
+		}
+		irbIface := vlanName(irbVLAN)
+		spec.Interfaces[irbIface] = &dozer.SpecInterface{
+			Enabled:     pointer.To(true),
+			Description: pointer.To(fmt.Sprintf("External %s IRB", externalName)),
+		}
+		spec.VRFs[extVrfName].Interfaces[irbIface] = &dozer.SpecVRFInterface{}
+		spec.VRFVNIMap[extVrfName] = &dozer.SpecVRFVNIEntry{
+			VNI: pointer.To(extVNI),
+		}
+		spec.VXLANTunnelMap[fmt.Sprintf("map_%d_%s", extVNI, irbIface)] = &dozer.SpecVXLANTunnelMap{
+			VTEP: pointer.To(VTEPFabric),
+			VNI:  pointer.To(extVNI),
+			VLAN: pointer.To(irbVLAN),
+		}
 	}
 
 	for name, attach := range agent.Spec.ExternalAttachments {
@@ -2813,22 +2833,10 @@ func planExternalPeerings(agent *agentapi.Agent, spec *dozer.Spec) error {
 		}
 
 		for _, subnetName := range peering.Permit.VPC.Subnets {
-			subnet, exists := vpc.Subnets[subnetName]
+			_, exists := vpc.Subnets[subnetName]
 			if !exists {
 				return errors.Errorf("VPC %s subnet %s not found for external peering %s", vpcName, subnetName, name)
 			}
-
-			vni, exists := agent.Spec.Catalog.GetVPCSubnetVNI(vpcName, subnetName)
-			if vni == 0 || !exists {
-				return errors.Errorf("VNI for VPC %s subnet %s not found for external peering %s", vpcName, subnetName, name)
-			}
-
-			spec.PrefixLists[extOutboundPrefixList(externalName)].Prefixes[vni] = &dozer.SpecPrefixListEntry{
-				Prefix: dozer.SpecPrefixListPrefix{
-					Prefix: subnet.Subnet,
-				},
-				Action: dozer.SpecPrefixListActionPermit,
-			}
 		}
 
 		extPrefixesName := vpcExtPrefixesPrefixListName(vpcName)
@@ -2915,7 +2923,7 @@ func planExternalPeerings(agent *agentapi.Agent, spec *dozer.Spec) error {
 			spec.VRFs[extVrf].BGP.IPv4Unicast.ImportVRFs[vpcVrf] = &dozer.SpecVRFBGPImportVRF{}
 			spec.VRFs[vpcVrf].BGP.IPv4Unicast.ImportVRFs[extVrf] = &dozer.SpecVRFBGPImportVRF{}
 		} else {
-			sub1, sub2, ip1, ip2, err := planLoopbackWorkaround(agent, spec, librarian.LoWReqForExt(name))
+			sub1, sub2, ip1, ip2, err := planLoopbackWorkaround(agent, spec, librarian.ReqForExt(name))
 			if err != nil {
 				return errors.Wrapf(err, "failed to plan loopback workaround for external peering %s", name)
 			}
@@ -3115,10 +3123,6 @@ func extInboundRouteMapName(external string) string {
 	return fmt.Sprintf("ext-inbound--%s", external)
 }
 
-func extOutboundPrefixList(external string) string {
-	return fmt.Sprintf("ext-outbound--%s", external)
-}
-
 func extOutboundRouteMapName(external string) string {
 	return fmt.Sprintf("ext-outbound--%s", external)
 }

pkg/ctrl/agent_ctrl.go

@@ -485,6 +485,7 @@ func (r *AgentReconciler) Reconcile(ctx context.Context, req kctrl.Request) (kct
 	externals := map[string]vpcapi.ExternalSpec{}
 	externalsToConfig := map[string]vpcapi.ExternalSpec{}
 	externalList := &vpcapi.ExternalList{}
+	externalsReq := map[string]bool{}
 	err = r.List(ctx, externalList, kclient.InNamespace(sw.Namespace))
 	if err != nil {
 		return kctrl.Result{}, errors.Wrapf(err, "error listing externals")
@@ -493,6 +494,7 @@ func (r *AgentReconciler) Reconcile(ctx context.Context, req kctrl.Request) (kct
 		externals[ext.Name] = ext.Spec
 		if attachedExternals[ext.Name] {
 			externalsToConfig[ext.Name] = ext.Spec
+			externalsReq[ext.Name] = true
 		}
 	}
 
@@ -603,9 +605,14 @@ func (r *AgentReconciler) Reconcile(ctx context.Context, req kctrl.Request) (kct
 		idConns[name] = true
 	}
 
+	err = r.libr.UpdateVNIs(ctx, r.Client)
+	if err != nil {
+		return kctrl.Result{}, errors.Wrapf(err, "error updating VNIs catalog")
+	}
+
 	cat := &agentapi.CatalogSpec{}
 
-	err = r.libr.CatalogForRedundancyGroup(ctx, r.Client, cat, sw.Name, sw.Spec.Redundancy, usedVPCs, portChanConns, idConns)
+	err = r.libr.CatalogForRedundancyGroup(ctx, r.Client, cat, sw.Name, sw.Spec.Redundancy, usedVPCs, portChanConns, idConns, externalsReq)
 	if err != nil {
 		return kctrl.Result{}, errors.Wrapf(err, "error getting redundancy group catalog")
 	}
@@ -652,15 +659,10 @@ func (r *AgentReconciler) Reconcile(ctx context.Context, req kctrl.Request) (kct
 				continue
 			}
 
-			loWorkaroundReqs[librarian.LoWReqForExt(name)] = true
+			loWorkaroundReqs[librarian.ReqForExt(name)] = true
 		}
 	}
 
-	externalsReq := map[string]bool{}
-	for name := range externalsToConfig {
-		externalsReq[name] = true
-	}
-
 	subnetsReq := map[string]bool{}
 	for _, vpc := range vpcs {
 		for _, subnet := range vpc.Subnets {

pkg/ctrl/gw_vpc_sync.go → pkg/ctrl/gw_sync.go

@@ -6,6 +6,7 @@ package ctrl
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"go.githedgehog.com/fabric/api/meta"
 	vpcapi "go.githedgehog.com/fabric/api/vpc/v1beta1"
@@ -126,3 +127,113 @@ func (r *GwVPCSync) Reconcile(ctx context.Context, req kctrl.Request) (kctrl.Res
 
 	return kctrl.Result{}, nil
 }
+
+// External equivalent of the above code
+
+type GwExternalSync struct {
+	kclient.Client
+	cfg  *meta.FabricConfig
+	libr *librarian.Manager
+}
+
+func SetupGwExternalSyncReconcilerWith(mgr kctrl.Manager, cfg *meta.FabricConfig, libMngr *librarian.Manager) error {
+	if cfg == nil {
+		return fmt.Errorf("fabric config is nil") //nolint:goerr113
+	}
+	if libMngr == nil {
+		return fmt.Errorf("librarian manager is nil") //nolint:goerr113
+	}
+
+	r := &GwExternalSync{
+		Client: mgr.GetClient(),
+		cfg:    cfg,
+		libr:   libMngr,
+	}
+
+	if err := kctrl.NewControllerManagedBy(mgr).
+		Named("GwExternalSync").
+		For(&vpcapi.External{}).
+		// TODO consider relying on the owner reference
+		Watches(&gwapi.VPCInfo{}, handler.EnqueueRequestsFromMapFunc(r.enqueueForVPCInfo)).
+		Complete(r); err != nil {
+		return fmt.Errorf("failed to setup controller: %w", err)
+	}
+
+	return nil
+}
+
+func (r *GwExternalSync) enqueueForVPCInfo(ctx context.Context, obj kclient.Object) []reconcile.Request {
+	vpcInfo, ok := obj.(*gwapi.VPCInfo)
+	if !ok {
+		kctrllog.FromContext(ctx).Info("Enqueue: object is not a VPCInfo", "obj", obj)
+
+		return nil
+	}
+
+	if !strings.HasPrefix(vpcInfo.Name, vpcapi.VPCInfoExtPrefix) {
+		return nil
+	}
+
+	return []reconcile.Request{
+		{NamespacedName: ktypes.NamespacedName{
+			Namespace: vpcInfo.Namespace,
+			Name:      strings.TrimPrefix(vpcInfo.Name, vpcapi.VPCInfoExtPrefix),
+		}},
+	}
+}
+
+//+kubebuilder:rbac:groups=vpc.githedgehog.com,resources=externals,verbs=get;list;watch
+//+kubebuilder:rbac:groups=vpc.githedgehog.com,resources=externals/status,verbs=get;update;patch
+//+kubebuilder:rbac:groups=vpc.githedgehog.com,resources=externals/finalizers,verbs=update
+
+//+kubebuilder:rbac:groups=gateway.githedgehog.com,resources=vpcinfos,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=gateway.githedgehog.com,resources=vpcinfos/status,verbs=get;update;patch
+//+kubebuilder:rbac:groups=gateway.githedgehog.com,resources=vpcinfos/finalizers,verbs=update
+
+func (r *GwExternalSync) Reconcile(ctx context.Context, req kctrl.Request) (kctrl.Result, error) {
+	l := kctrllog.FromContext(ctx)
+
+	external := &vpcapi.External{}
+	if err := r.Get(ctx, req.NamespacedName, external); err != nil {
+		if kapierrors.IsNotFound(err) {
+			return kctrl.Result{}, nil
+		}
+
+		return kctrl.Result{}, fmt.Errorf("getting External %s: %w", req.NamespacedName, err)
+	}
+
+	vni, err := r.libr.GetExternalVNI(ctx, r.Client, external.Name)
+	if err != nil {
+		return kctrl.Result{}, fmt.Errorf("getting External %s VNI: %w", external.Name, err)
+	}
+
+	subnets := map[string]*gwapi.VPCInfoSubnet{}
+	// FIXME: the external spec does not have the prefixes we are importing, they are part of the externalPeering
+	subnets["external"] = &gwapi.VPCInfoSubnet{
+		CIDR: "0.0.0.0/0",
+	}
+
+	vpcInfo := &gwapi.VPCInfo{ObjectMeta: kmetav1.ObjectMeta{
+		Name:      vpcapi.VPCInfoExtPrefix + external.Name,
+		Namespace: external.Namespace,
+	}}
+	if op, err := ctrlutil.CreateOrUpdate(ctx, r.Client, vpcInfo, func() error {
+		if err := ctrlutil.SetControllerReference(external, vpcInfo, r.Scheme(),
+			ctrlutil.WithBlockOwnerDeletion(false)); err != nil {
+			return fmt.Errorf("setting controller reference: %w", err)
+		}
+
+		vpcInfo.Spec = gwapi.VPCInfoSpec{
+			VNI:     vni,
+			Subnets: subnets,
+		}
+
+		return nil
+	}); err != nil {
+		return kctrl.Result{}, fmt.Errorf("creating/updating VPCInfo %s: %w", req.NamespacedName, err)
+	} else if op == ctrlutil.OperationResultCreated || op == ctrlutil.OperationResultUpdated {
+		l.Info("Gateway VPCInfo synced", "op", op)
+	}
+
+	return kctrl.Result{}, nil
+}