Skip to content

Commit e453d20

Browse files
guvencguvenc
authored
Merge pull request #47 from onmetal/new_vni_handling_vnet_peering
New vni handling vnet peering
2 parents 13db515 + 521b185 commit e453d20

13 files changed

Lines changed: 317 additions & 65 deletions

‎api/v1alpha1/network_types.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ type NetworkSpec struct {
2929
// +kubebuilder:validation:Minimum=1
3030
// ID is the unique identifier of the Network
3131
ID int32 `json:"id"`
32+
33+
// PeeredIDs are the IDs of networks to peer with.
34+
PeeredIDs []int32 `json:"peeredIDs,omitempty"`
3235
}
3336

3437
//+kubebuilder:object:root=true

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 6 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/networking.metalnet.onmetal.de_networks.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,12 @@ spec:
5151
maximum: 16777215
5252
minimum: 1
5353
type: integer
54+
peeredIDs:
55+
description: PeeredIDs are the IDs of networks to peer with.
56+
items:
57+
format: int32
58+
type: integer
59+
type: array
5460
required:
5561
- id
5662
type: object

‎config/samples/network2.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: networking.metalnet.onmetal.de/v1alpha1
2+
kind: Network
3+
metadata:
4+
name: network-sample2
5+
spec:
6+
id: 200
7+
# peeredIDs:
8+
# - 123

‎config/samples/networking_v1alpha1_network.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,5 @@ metadata:
44
name: network-sample
55
spec:
66
id: 123
7+
# peeredIDs:
8+
# - 200

‎config/samples/networking_v1alpha1_networkinterface.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,4 @@ spec:
1616
# loadBalancerTargets:
1717
# - 194.11.242.110/32
1818
networkRef:
19-
name: network-sample
19+
name: network-sample2

‎controllers/loadbalancer_controller.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,11 @@ type LoadBalancerReconciler struct {
4747
record.EventRecorder
4848
Scheme *runtime.Scheme
4949

50-
DPDK dpdk.Client
51-
LBServer dpdkmetalbond.LBServerAccess
52-
Metalbond metalbond.Client
53-
NodeName string
54-
PublicVNI int
50+
DPDK dpdk.Client
51+
MBInternal dpdkmetalbond.MbInternalAccess
52+
Metalbond metalbond.Client
53+
NodeName string
54+
PublicVNI int
5555
}
5656

5757
//+kubebuilder:rbac:groups=networking.metalnet.onmetal.de,resources=loadbalancers,verbs=get;list;watch;create;update;patch;delete
@@ -119,7 +119,7 @@ func (r *LoadBalancerReconciler) delete(ctx context.Context, log logr.Logger, lb
119119
}
120120

121121
log.V(1).Info("Remove LoadBalancer server", "vni", vni, "ip", ip)
122-
if err := r.LBServer.RemoveLoadBalancerServer(vni, ip, lb.UID); err != nil {
122+
if err := r.MBInternal.RemoveLoadBalancerServer(vni, ip, lb.UID); err != nil {
123123
return ctrl.Result{}, fmt.Errorf("error deleting dpdk loadbalancer from internal cache: %w", err)
124124
}
125125

@@ -138,7 +138,7 @@ func (r *LoadBalancerReconciler) delete(ctx context.Context, log logr.Logger, lb
138138
}
139139
log.V(1).Info("Deleted Loadbalancer")
140140
log.V(1).Info("Remove LoadBalancer server", "vni", vni, "ip", ip)
141-
if err := r.LBServer.RemoveLoadBalancerServer(vni, ip, lb.UID); err != nil {
141+
if err := r.MBInternal.RemoveLoadBalancerServer(vni, ip, lb.UID); err != nil {
142142
return ctrl.Result{}, fmt.Errorf("error deleting dpdk loadbalancer from internal cache: %w", err)
143143
}
144144

@@ -318,7 +318,7 @@ func (r *LoadBalancerReconciler) applyLoadBalancer(ctx context.Context, log logr
318318
return netip.Addr{}, fmt.Errorf("error creating dpdk loadbalancer: %w", err)
319319
}
320320
log.V(1).Info("Adding loadbalancer server", "vni", vni, "ip", ip)
321-
if err := r.LBServer.AddLoadBalancerServer(vni, ip, lb.UID); err != nil {
321+
if err := r.MBInternal.AddLoadBalancerServer(vni, ip, lb.UID); err != nil {
322322
return netip.Addr{}, fmt.Errorf("error adding dpdk loadbalancer to internal cache: %w", err)
323323
}
324324
log.V(1).Info("Adding loadbalancer route if not exists")
@@ -331,7 +331,7 @@ func (r *LoadBalancerReconciler) applyLoadBalancer(ctx context.Context, log logr
331331

332332
log.V(1).Info("DPDK loadbalancer exists")
333333
log.V(1).Info("Adding loadbalancer server", "vni", vni, "ip", ip)
334-
if err := r.LBServer.AddLoadBalancerServer(vni, ip, lb.UID); err != nil {
334+
if err := r.MBInternal.AddLoadBalancerServer(vni, ip, lb.UID); err != nil {
335335
return netip.Addr{}, fmt.Errorf("error adding dpdk loadbalancer to internal cache: %w", err)
336336
}
337337
log.V(1).Info("Adding loadbalancer route if not exists")

‎controllers/network_controller.go‎

Lines changed: 164 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,11 @@ import (
2525
"github.com/onmetal/controller-utils/clientutils"
2626
metalnetv1alpha1 "github.com/onmetal/metalnet/api/v1alpha1"
2727
"github.com/onmetal/metalnet/dpdk"
28+
"github.com/onmetal/metalnet/dpdkmetalbond"
2829
"github.com/onmetal/metalnet/metalbond"
2930
"k8s.io/apimachinery/pkg/runtime"
3031
"k8s.io/apimachinery/pkg/types"
32+
"k8s.io/apimachinery/pkg/util/sets"
3133
ctrl "sigs.k8s.io/controller-runtime"
3234
"sigs.k8s.io/controller-runtime/pkg/builder"
3335
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -47,9 +49,9 @@ type NetworkReconciler struct {
4749
client.Client
4850
Scheme *runtime.Scheme
4951

50-
DPDK dpdk.Client
51-
Metalbond metalbond.Client
52-
52+
DPDK dpdk.Client
53+
Metalbond metalbond.Client
54+
MBInternal dpdkmetalbond.MbInternalAccess
5355
RouterAddress netip.Addr
5456
NodeName string
5557
}
@@ -103,6 +105,12 @@ func (r *NetworkReconciler) delete(ctx context.Context, log logr.Logger, network
103105
}
104106
log.V(1).Info("Deleted default route if existed")
105107

108+
log.V(1).Info("Deleting peered VNIs")
109+
if err := r.deletePeeredVNIs(ctx, log, network, vni); err != nil {
110+
return ctrl.Result{}, err
111+
}
112+
log.V(1).Info("Deleted peered VNIs")
113+
106114
log.V(1).Info("Cleanup done, removing finalizer")
107115
if err := clientutils.PatchRemoveFinalizer(ctx, r.Client, network, r.networkFinalizer()); err != nil {
108116
return ctrl.Result{}, fmt.Errorf("error removing finalizer: %w", err)
@@ -135,11 +143,19 @@ func (r *NetworkReconciler) reconcile(ctx context.Context, log logr.Logger, netw
135143
}
136144

137145
if !vniAvail {
138-
log.V(1).Info("VNI doesn't exist in dp-service, unsubscribe from it")
139-
if err := r.unsubscribeIfSubscribed(ctx, vni); err != nil {
146+
if !r.MBInternal.IsVniPeered(vni) {
147+
log.V(1).Info("VNI doesn't exist in dp-service and no peering, unsubscribe from it")
148+
if err := r.unsubscribeIfSubscribed(ctx, vni); err != nil {
149+
return ctrl.Result{}, err
150+
}
151+
log.V(1).Info("VNI doesn't exist in dp-service and no peering, unsubscribed from it")
152+
}
153+
154+
log.V(1).Info("Reconciling peered VNIs")
155+
if err := r.reconcilePeeredVNIs(ctx, log, network, vni, vniAvail); err != nil {
140156
return ctrl.Result{}, err
141157
}
142-
log.V(1).Info("VNI doesn't exist in dp-service, unsubscribed from it")
158+
log.V(1).Info("Reconciled peered VNIs")
143159
return ctrl.Result{}, nil
144160
}
145161
log.V(1).Info("Checked existence of the VNI")
@@ -150,6 +166,12 @@ func (r *NetworkReconciler) reconcile(ctx context.Context, log logr.Logger, netw
150166
}
151167
log.V(1).Info("Created dpdk default route if not existed")
152168

169+
log.V(1).Info("Reconciling peered VNIs")
170+
if err := r.reconcilePeeredVNIs(ctx, log, network, vni, vniAvail); err != nil {
171+
return ctrl.Result{}, err
172+
}
173+
log.V(1).Info("Reconciled peered VNIs")
174+
153175
log.V(1).Info("Subscribing to metalbond if not subscribed")
154176
if err := r.subscribeIfNotSubscribed(ctx, vni); err != nil {
155177
return ctrl.Result{}, err
@@ -202,6 +224,142 @@ func (r *NetworkReconciler) subscribeIfNotSubscribed(ctx context.Context, vni ui
202224
return nil
203225
}
204226

227+
func (r *NetworkReconciler) setDifference(s1, s2 sets.Set[uint32]) sets.Set[uint32] {
228+
diff := sets.New[uint32]()
229+
for k := range s1 {
230+
if _, ok := s2[k]; !ok {
231+
diff.Insert(k)
232+
}
233+
}
234+
return diff
235+
}
236+
237+
func (r *NetworkReconciler) reconcilePeeredVNIs(ctx context.Context, log logr.Logger, network *metalnetv1alpha1.Network, vni uint32, ownVniAvail bool) error {
238+
mbPeerVnis, err := r.MBInternal.GetPeerVnis(vni)
239+
if err != nil {
240+
return err
241+
}
242+
specPeerVnis := sets.New[uint32]()
243+
if network.Spec.PeeredIDs != nil {
244+
for _, v := range network.Spec.PeeredIDs {
245+
specPeerVnis.Insert(uint32(v))
246+
}
247+
}
248+
missing := r.setDifference(mbPeerVnis, specPeerVnis)
249+
added := r.setDifference(specPeerVnis, mbPeerVnis)
250+
251+
if missing.Len() == 0 && added.Len() == 0 {
252+
if mbPeerVnis.Len() == 0 {
253+
return nil
254+
}
255+
256+
for _, peeredVNI := range mbPeerVnis.UnsortedList() {
257+
if !ownVniAvail {
258+
if err := r.MBInternal.RemoveVniFromPeerVnis(log, vni, peeredVNI); err != nil {
259+
return err
260+
}
261+
}
262+
}
263+
}
264+
265+
if missing.Len() != 0 || added.Len() != 0 {
266+
for _, peeredVNI := range missing.UnsortedList() {
267+
log.V(1).Info("Checking the existence of the peeredVNI in dp-service", "peeredVNI", peeredVNI)
268+
peeredVniAvail, err := r.DPDK.IsVniAvailable(ctx, peeredVNI)
269+
if err != nil {
270+
return err
271+
}
272+
log.V(1).Info("Checked the existence of the peeredVNI in dp-service", "peeredVNI", peeredVNI)
273+
274+
if !peeredVniAvail {
275+
if err := r.unsubscribeIfSubscribed(ctx, peeredVNI); err != nil {
276+
return err
277+
}
278+
} else if peeredVniAvail && ownVniAvail {
279+
if err := r.recycleVNISubscription(ctx, vni); err != nil {
280+
return err
281+
}
282+
if err := r.recycleVNISubscription(ctx, peeredVNI); err != nil {
283+
return err
284+
}
285+
} else {
286+
if err := r.recycleVNISubscription(ctx, peeredVNI); err != nil {
287+
return err
288+
}
289+
}
290+
if err := r.MBInternal.RemoveVniFromPeerVnis(log, vni, peeredVNI); err != nil {
291+
return err
292+
}
293+
}
294+
295+
for _, peeredVNI := range added.UnsortedList() {
296+
if !ownVniAvail {
297+
return nil
298+
}
299+
log.V(1).Info("Checking the existence of the peeredVNI in dp-service", "peeredVNI", peeredVNI)
300+
peeredVniAvail, err := r.DPDK.IsVniAvailable(ctx, peeredVNI)
301+
if err != nil {
302+
return err
303+
}
304+
log.V(1).Info("Checked the existence of the peeredVNI in dp-service", "peeredVNI", peeredVNI)
305+
if ownVniAvail && !peeredVniAvail {
306+
if err := r.subscribeIfNotSubscribed(ctx, peeredVNI); err != nil {
307+
return err
308+
}
309+
}
310+
if ownVniAvail && peeredVniAvail {
311+
if err := r.recycleVNISubscription(ctx, vni); err != nil {
312+
return err
313+
}
314+
if err := r.recycleVNISubscription(ctx, peeredVNI); err != nil {
315+
return err
316+
}
317+
}
318+
if err := r.MBInternal.AddVniToPeerVnis(log, vni, peeredVNI); err != nil {
319+
return err
320+
}
321+
}
322+
}
323+
324+
return nil
325+
}
326+
327+
func (r *NetworkReconciler) deletePeeredVNIs(ctx context.Context, log logr.Logger, network *metalnetv1alpha1.Network, vni uint32) error {
328+
mbPeerVnis, _ := r.MBInternal.GetPeerVnis(vni)
329+
330+
for _, peeredVNI := range mbPeerVnis.UnsortedList() {
331+
log.V(1).Info("Checking existence of the ", "peered VNI", peeredVNI)
332+
vniAvail, err := r.DPDK.IsVniAvailable(ctx, peeredVNI)
333+
if err != nil {
334+
return err
335+
}
336+
if !vniAvail {
337+
if err := r.unsubscribeIfSubscribed(ctx, peeredVNI); err != nil {
338+
return err
339+
}
340+
}
341+
if err := r.MBInternal.RemoveVniFromPeerVnis(log, vni, peeredVNI); err != nil {
342+
return err
343+
}
344+
}
345+
return nil
346+
}
347+
348+
func (r *NetworkReconciler) recycleVNISubscription(ctx context.Context, vni uint32) error {
349+
if err := r.unsubscribeIfSubscribed(ctx, vni); err != nil {
350+
return err
351+
}
352+
353+
if err := r.DPDK.ResetVni(ctx, vni); err != nil {
354+
return fmt.Errorf("error resetting vni: %w", err)
355+
}
356+
357+
if err := r.subscribeIfNotSubscribed(ctx, vni); err != nil {
358+
return err
359+
}
360+
return nil
361+
}
362+
205363
func (r *NetworkReconciler) unsubscribeIfSubscribed(ctx context.Context, vni uint32) error {
206364
if err := r.Metalbond.Unsubscribe(ctx, metalbond.VNI(vni)); metalbond.IgnoreNotSubscribedToVNIError(err) != nil {
207365
return fmt.Errorf("error subscribing to vni: %w", err)

‎dpdk/client.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ type Client interface {
6161
DeleteNATRoute(ctx context.Context, route *NATRoute) error
6262

6363
IsVniAvailable(ctx context.Context, vni uint32) (bool, error)
64+
ResetVni(ctx context.Context, vni uint32) error
6465
}
6566

6667
type NATRoute struct {
@@ -357,6 +358,19 @@ func (c *client) IsVniAvailable(ctx context.Context, vni uint32) (bool, error) {
357358
return res.InUse, nil
358359
}
359360

361+
func (c *client) ResetVni(ctx context.Context, vni uint32) error {
362+
res, err := c.DPDKonmetalClient.ResetVni(ctx, &dpdkproto.ResetVniRequest{
363+
Vni: vni,
364+
Type: dpdkproto.VniType_VniIpv4AndIpv6})
365+
if err != nil {
366+
return err
367+
}
368+
if errorCode := res.GetError(); errorCode != 0 {
369+
return &StatusError{errorCode: errorCode, message: res.GetMessage()}
370+
}
371+
return nil
372+
}
373+
360374
func (c *client) GetInterface(ctx context.Context, uid types.UID) (*Interface, error) {
361375
res, err := c.DPDKonmetalClient.GetInterface(ctx, &dpdkproto.InterfaceIDMsg{InterfaceID: []byte(uid)})
362376
if err != nil {

0 commit comments

Comments
 (0)