Add routes propagation for VRF plugin

Up until now, if previous plugin assigned routes to interface, movement of
this interface to new VRF cause routes to be deleted.

This patch adds funtionality to VRF plugin to save the routes before
interface is assgined to VRF, and then re-apply all saved routes to new VRF.

plugins/meta/vrf/vrf.go

@@ -104,6 +104,14 @@ func addInterface(vrf *netlink.Vrf, intf string) error {
 	if err != nil {
 		return fmt.Errorf("failed getting ipv6 addresses for %s", intf)
 	}
+
+	// Save routes that are setup for the interface, before setting master,
+	// because otherwise the routes will be deleted after interface is moved.
+	routes, err := netlink.RouteList(i, netlink.FAMILY_ALL)
+	if err != nil {
+		return fmt.Errorf("failed getting ipv4 routes for %s", intf)
+	}
+
 	err = netlink.LinkSetMaster(i, vrf)
 	if err != nil {
 		return fmt.Errorf("could not set vrf %s as master of %s: %v", vrf.Name, intf, err)
@@ -130,6 +138,14 @@ CONTINUE:
 		}
 	}
 
+	// Apply all saved routes for the interface that was moved to the VRF
+	for _, route := range routes {
+		// modify original table to vrf one
+		// equivalent of 'ip route add <address> table <int>'
+		route.Table = int(vrf.Table)
+		netlink.RouteAdd(&route)
+	}
+
 	return nil
 }
 

plugins/meta/vrf/vrf_test.go

@@ -17,10 +17,13 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"net"
+	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
@@ -107,7 +110,7 @@ var _ = Describe("vrf plugin", func() {
 				},
 			})
 			Expect(err).NotTo(HaveOccurred())
-			_, err = netlink.LinkByName(IF0Name)
+			_, err = netlink.LinkByName(IF1Name)
 			Expect(err).NotTo(HaveOccurred())
 			return nil
 		})
@@ -177,6 +180,84 @@ var _ = Describe("vrf plugin", func() {
 		Expect(err).NotTo(HaveOccurred())
 	})
 
+	It("adds the interface and custom routing to new VRF", func() {
+		conf := configWithRouteFor("test", IF0Name, VRF0Name, "10.0.0.2/24", "10.10.10.0/24")
+
+		By("Setting custom routing first", func() {
+			err := targetNS.Do(func(ns.NetNS) error {
+				defer GinkgoRecover()
+
+				intfIp, intfNet, err := net.ParseCIDR("10.0.0.2/24")
+				Expect(err).NotTo(HaveOccurred())
+				_, destNet, err := net.ParseCIDR("10.10.10.0/24")
+				Expect(err).NotTo(HaveOccurred())
+
+				intfNet.IP = intfIp
+
+				link, err := netlink.LinkByName(IF0Name)
+				Expect(err).NotTo(HaveOccurred())
+
+				// Add 10.0.0.2/24 address for network reachability
+				netlink.AddrAdd(link, &netlink.Addr{IPNet: intfNet})
+
+				ipAddrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
+				Expect(err).NotTo(HaveOccurred())
+				// Check if address was assigned properly
+				Expect(ipAddrs[0].IP.String()).To(Equal("10.0.0.2"))
+
+				// Set interface UP, otherwise local route to 10.0.0.0/24 is not present
+				err = netlink.LinkSetUp(link)
+				Expect(err).NotTo(HaveOccurred())
+
+				// Add additional route to 10.10.10.0/24 via 10.0.0.1 gateway
+				r := netlink.Route{
+					LinkIndex: link.Attrs().Index,
+					Src: intfIp,
+					Dst:   destNet,
+					Gw:    net.ParseIP("10.0.0.1"),
+				}
+				err = netlink.RouteAdd(&r)
+				Expect(err).NotTo(HaveOccurred())
+
+				return nil
+			})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		args := &skel.CmdArgs{
+			ContainerID: "dummy",
+			Netns:       targetNS.Path(),
+			IfName:      IF0Name,
+			StdinData:   conf,
+		}
+
+		err := originalNS.Do(func(ns.NetNS) error {
+			defer GinkgoRecover()
+			r, _, err := testutils.CmdAddWithArgs(args, func() error {
+				return cmdAdd(args)
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			result, err := current.GetResult(r)
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(len(result.Interfaces)).To(Equal(1))
+			Expect(result.Interfaces[0].Name).To(Equal(IF0Name))
+			Expect(len(result.Routes)).To(Equal(1))
+			Expect(result.Routes[0].Dst.IP.String()).To(Equal("10.10.10.0"))
+			return nil
+		})
+		Expect(err).NotTo(HaveOccurred())
+
+		err = targetNS.Do(func(ns.NetNS) error {
+			defer GinkgoRecover()
+			checkInterfaceOnVRF(VRF0Name, IF0Name)
+			checkRoutesOnVRF(VRF0Name, IF0Name, "10.10.10.0/24")
+			return nil
+		})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	It("fails if the interface already has a master set", func() {
 		conf := configFor("test", IF0Name, VRF0Name, "10.0.0.2/24")
 
@@ -690,6 +771,35 @@ func configWithTableFor(name, intf, vrf, ip string, tableID int) []byte {
 	return []byte(conf)
 }
 
+func configWithRouteFor(name, intf, vrf, ip, route string) []byte {
+	conf := fmt.Sprintf(`{
+		"name": "%s",
+		"type": "vrf",
+		"cniVersion": "0.3.1",
+		"vrfName": "%s",
+		"prevResult": {
+			"interfaces": [
+				{"name": "%s", "sandbox":"netns"}
+			],
+			"ips": [
+				{
+					"version": "4",
+					"address": "%s",
+					"gateway": "10.0.0.1",
+					"interface": 0
+				}
+			],
+			"routes": [
+				{
+					"dst": "%s",
+					"gw": "10.0.0.1"
+				}
+			]
+		}
+	}`, name, vrf, intf, ip, route)
+	return []byte(conf)
+}
+
 func checkInterfaceOnVRF(vrfName, intfName string) {
 	vrf, err := netlink.LinkByName(vrfName)
 	Expect(err).NotTo(HaveOccurred())
@@ -702,3 +812,39 @@ func checkInterfaceOnVRF(vrfName, intfName string) {
 	Expect(err).NotTo(HaveOccurred())
 	Expect(master.Attrs().Name).To(Equal(vrfName))
 }
+
+func checkRoutesOnVRF(vrfName, intfName, routeAddress string) {
+	vrf, err := netlink.LinkByName(vrfName)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(vrf).To(BeAssignableToTypeOf(&netlink.Vrf{}))
+
+	link, err := netlink.LinkByName(intfName)
+	Expect(err).NotTo(HaveOccurred())
+
+	err = netlink.LinkSetUp(link)
+	Expect(err).NotTo(HaveOccurred())
+
+	ipAddrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(len(ipAddrs)).To(Equal(1))
+	Expect(ipAddrs[0].IP.String()).To(Equal("10.0.0.2"))
+
+
+	// Need to read all tables, so cannot use RouteList
+	routeFilter := &netlink.Route{
+		LinkIndex: link.Attrs().Index,
+		Table:     unix.RT_TABLE_UNSPEC,
+	}
+
+	routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4,
+		routeFilter,
+		netlink.RT_FILTER_OIF|netlink.RT_FILTER_TABLE)
+	Expect(err).NotTo(HaveOccurred())
+
+	routesStr := []string{}
+	for _, route := range routes {
+		routesStr= append(routesStr, route.String())
+		Expect(uint32(route.Table)).To(Equal(vrf.(*netlink.Vrf).Table))
+	}
+	Expect(strings.Join(routesStr, "\n")).To(ContainSubstring(routeAddress))
+}