User Tools

Site Tools


documentation:examples:ecmp

This is an old revision of the document!


Equal-cost multi-path routing (ECMP)

This lab is testing new ECMP routing code from FreeBSD's review D26449

Presentation

Bhyve doesn't support emulating multiqueue NIC, so RSS flow-id could not be tested using a bhyve based lab: Had to use physical lab

Network diagram

Here is the logical and physical view:

Setting-up the lab

Downloading BSD Router Project images

Download BSDRP serial image on Sourceforge and upload them to the 2 ECMP routers.

Static routing setup

VM 1 (client)

VM1 is configured as a simple host with multiples aliases:

sysrc hostname=VM1 \
  gateway_enable=NO \
  ipv6_gateway_enable=NO \
  ifconfig_igb1="inet 10.0.31.3/24" \
  ifconfig_igb1_alias1="inet 10.0.31.13/32" \
  ifconfig_igb1_alias2="inet 10.0.31.23/32" \
  ifconfig_igb1_alias3="inet 10.0.31.33/32" \
  ifconfig_igb1_alias4="inet 10.0.31.43/32" \
  ifconfig_igb1_alias5="inet 10.0.31.53/32" \
  ifconfig_igb1_ipv6="inet6 2001:db8:31::3 prefixlen 64" \
  ifconfig_igb1_alias61="inet6 2001:db8:31::13 prefixlen 64" \
  ifconfig_igb1_alias62="inet6 2001:db8:31::23 prefixlen 64" \
  ifconfig_igb1_alias63="inet6 2001:db8:31::33 prefixlen 64" \
  ifconfig_igb1_alias64="inet6 2001:db8:31::43 prefixlen 64" \
  ifconfig_igb1_alias65="inet6 2001:db8:31::53 prefixlen 64" \
  static_routes="LAB" \ 
  route_LAB="-net 10.0.0.0/16 10.0.31.1" \ 
  ipv6_static_routes="LAB" \ 
  ipv6_route_LAB="2001:db8:: -prefixlen 32 2001:db8:31::1"
service hostname restart
service netif restart
service routing restart
config save

VM 2 (ECMP router)

VM 2 is a router with ECMP: 2 static routes toward the same destination but using 2 different next-hop.

sysrc hostname=VM2 \
  gateway_enable=YES \
  ipv6_gateway_enable=YES \
  ifconfig_igb0="inet 10.0.31.1/24" \ 
  ifconfig_igb0_ipv6="inet6 2001:db8:31::1 prefixlen 64" \
  ifconfig_igb1="inet 10.0.112.1/24" \
  ifconfig_igb1_ipv6="inet6 2001:db8:112::1 prefixlen 64" \
  ifconfig_igb2="inet 10.0.212.1/24" \
  ifconfig_igb2_ipv6="inet6 2001:db8:212::1 prefixlen 64" \
  static_routes="MPATH1 MPATH2" \
  route_MPATH1="-net 10.0.0.0/16 10.0.112.2" \
  route_MPATH2="-net 10.0.0.0/16 10.0.212.2" \
  ipv6_static_routes="MPATH1 MPATH2" \
  ipv6_route_MPATH1="2001:db8:: -prefixlen 32 2001:db8:112::2" \
  ipv6_route_MPATH2="2001:db8:: -prefixlen 32 2001:db8:212::2"
service hostname restart
service netif restart
service routing restart
config save

Checking static route with multiple next-hop:

[root@VM2]~# # netstat -rn4 | grep 10.0.0.0/16
10.0.0.0/16        10.0.112.2         UGS        igb1
10.0.0.0/16        10.0.212.2         UGS        igb2

[root@VM2]~# route -n get 10.0.0.0/16
   route to: 10.0.0.0
destination: 10.0.0.0
       mask: 255.255.0.0
    gateway: 10.0.112.2
        fib: 0
  interface: igb1
      flags: <UP,GATEWAY,DONE,STATIC>
 recvpipe  sendpipe  ssthresh  rtt,msec    mtu        weight    expire
       0         0         0         0      1500         0         0

[root@VM2]~# netstat -4onW
Nexthop data

Internet:
Idx   Type         IFA                Gateway             Flags      Use Mtu         Netif     Addrif Refcnt Prepend
1       v4/resolve 127.0.0.1          lo0/resolve        HS            0  16384        lo0               1
2       v4/resolve 10.0.31.1          igb0/resolve                     0   1500       igb0               1
3       v4/resolve 127.0.0.1          lo0/resolve        HS            0  16384        lo0      igb0     1
4       v4/resolve 10.0.112.1         igb1/resolve                     0   1500       igb1               1
5       v4/resolve 127.0.0.1          lo0/resolve        HS            0  16384        lo0      igb1     1
6       v4/resolve 10.0.212.1         igb2/resolve                     0   1500       igb2               1
7       v4/resolve 127.0.0.1          lo0/resolve        HS            0  16384        lo0      igb2     1
8            v4/gw 10.0.112.1         10.0.112.2         GS            0   1500       igb1               1
9            v4/gw 10.0.212.1         10.0.212.2         GS            0   1500       igb2               1
          
[root@VM2]~# route -n6 get 2001:db8::/32
    route to: 2001:db8::
destination: 2001:db8::
       mask: ffff:ffff::
    gateway: 2001:db8:212::2
        fib: 0
  interface: igb1
      flags: <UP,GATEWAY,DONE,STATIC>
 recvpipe  sendpipe  ssthresh  rtt,msec    mtu        weight    expire
       0         0         0         0      1500         1         0

[root@VM2]~# netstat -6onW
Nexthop data

Internet6:
Idx   Type         IFA                           Gateway                        Flags      Use Mtu       Netif   Addrif Refcnt Prepend
1       v6/resolve ::1                           lo0/resolve                   H             0  16384      lo0             1
2       v6/resolve fe80::1%lo0                   lo0/resolve                   HS            0  16384      lo0             1
3       v6/resolve fe80::1%lo0                   lo0/resolve                                 0  16384      lo0             1
4       v6/resolve ::1                           lo0/resolve                   HS            0  16384      lo0    igb0     1
5       v6/resolve fe80::20d:b9ff:fe45:79e4%igb0 igb0/resolve                                0   1500     igb0             1
6       v6/resolve ::1                           lo0/resolve                   HS            0  16384      lo0    igb1     2
7       v6/resolve fe80::20d:b9ff:fe45:79e5%igb1 igb1/resolve                                0   1500     igb1             1
8       v6/resolve ::1                           lo0/resolve                   HS            0  16384      lo0    igb2     1
9       v6/resolve fe80::20d:b9ff:fe45:79e6%igb2 igb2/resolve                                0   1500     igb2             1
10           v6/gw ::1                           ::1                           GRS           0  16384      lo0             4
11      v6/resolve 2001:db8:212::1               igb1/resolve                                0   1500     igb1             1
12           v6/gw 2001:db8:212::1               2001:db8:212::2               GS            0   1500     igb1             1

VM 5 (ECMP router)

VM 5 is like VM 2, a router with ECMP: 2 static routing toward the same destination but using 2 different next-hop..

sysrc hostname=VM5 \
  gateway_enable=YES \
  ipv6_gateway_enable=YES \
  ifconfig_igb0="inet 10.0.24.2/24" \
  ifconfig_igb1_ipv6="inet6 2001:db8:24::2 prefixlen 64" \
  ifconfig_igb1="inet 10.0.112.2/24" \
  ifconfig_igb1_ipv6="inet6 2001:db8:112::2 prefixlen 64" \
  ifconfig_igb2="inet 10.0.212.2/24" \
  ifconfig_igb1_ipv6="inet6 2001:db8:212::2 prefixlen 64" \
  static_routes="MPATH1 MPATH2" \
  route_MPATH1="-net 10.0.0.0/16 10.0.112.1" \
  route_MPATH2="-net 10.0.0.0/16 10.0.212.1" \
  ipv6_static_routes="MPATH1 MPATH2" \
  ipv6_route_MPATH1="2001:db8:: -prefixlen 32 2001:db8:112::1" \
  ipv6_route_MPATH2="2001:db8:: -prefixlen 32 2001:db8:212::1"
service hostname restart
service netif restart
service routing restart
config save

VM 6 (server)

VM 6, like VM 1 is configured as a simple host with multiple aliases:

sysrc hostname=VM6 \
  gateway_enable=NO \
  ipv6_gateway_enable=NO \
  ifconfig_vr0="inet 10.0.24.4/24" \
  ifconfig_vr0_alias1="inet 10.0.24.14/32" \
  ifconfig_vr0_alias2="inet 10.0.24.24/32" \
  ifconfig_vr0_alias3="inet 10.0.24.34/32" \
  ifconfig_vr0_alias4="inet 10.0.24.44/32" \
  ifconfig_vr0_alias5="inet 10.0.24.54/32" \
  ifconfig_vr0_ipv6="inet6 2001:db8:24::4 prefixlen 64" \
  ifconfig_vr0_alias61="inet6 2001:db8:24::14 prefixlen 64" \
  ifconfig_vr0_alias62="inet6 2001:db8:24::24 prefixlen 64" \
  ifconfig_vr0_alias63="inet6 2001:db8:24::34 prefixlen 64" \
  ifconfig_vr0_alias64="inet6 2001:db8:24::44 prefixlen 64" \
  ifconfig_vr0_alias65="inet6 2001:db8:24::54 prefixlen 64" \
  static_routes="LAB" \
  route_LAB="-net 10.0.0.0/16 10.0.24.2" \
  ipv6_static_routes="LAB" \
  ipv6_route_LAB="2001:db8:: -prefixlen 32 2001:db8:24::2"
service hostname restart
service netif restart
service routing restart
config save

Testing Load balancing

Flows from the client to the server should be “flow-id shared” between the 2 paths. Let's check using multiple sources and destination IP addresses. Using ping record feature like with this example:

[root@VM1]~# ping -c1 -R -S 10.0.12.21 10.0.56.26
PING 10.0.56.26 (10.0.56.26) from 10.0.12.21: 56 data bytes
64 bytes from 10.0.56.26: icmp_seq=0 ttl=61 time=0.455 ms
RR:     10.0.24.2
        10.0.45.4
        10.0.56.5
        10.0.56.26
        10.0.45.5
        10.0.24.4
        10.0.12.2
        10.0.12.21

--- 10.0.56.26 ping statistics ---
1 packets transmitted, 1 packets received, 0.0% packet loss
round-trip min/avg/max/stddev = 0.455/0.455/0.455/0.000 ms
Source Destination Ongoing path selected by VM2 Returning path selected by VM5
10.0.12.1 10.0.35.5 VM3 VM4
10.0.12.1 10.0.45.5 VM4 VM4
10.0.12.1 10.0.56.6 VM4 VM4
10.0.12.11 10.0.56.16 VM4 VM4
10.0.12.21 10.0.56.26 VM4 VM4
10.0.12.31 10.0.56.36 VM4 VM4
10.0.12.41 10.0.56.46 VM4 VM4
10.0.12.51 10.0.56.56 VM4 VM4
2001:db8:12::1 2001:db8:56::6 VM4 VM4
2001:db8:12::11 2001:db8:56::16 VM4 VM4
2001:db8:12::21 2001:db8:56::26 VM4 VM4
2001:db8:12::31 2001:db8:56::36 VM4 VM4
2001:db8:12::41 2001:db8:56::46 VM4 VM4
2001:db8:12::51 2001:db8:56::56 VM4 VM4
documentation/examples/ecmp.1600684902.txt.gz · Last modified: 2020/09/21 12:41 by olivier

Except where otherwise noted, content on this wiki is licensed under the following license: BSD 2-Clause
Donate Powered by PHP Valid HTML5 Valid CSS Driven by DokuWiki