Differences

This shows you the differences between two versions of the page.

--- documentation:examples:ecmp [2018/07/13 10:31] – external edit 127.0.0.1
+++ documentation:examples:ecmp [2020/01/26 19:46] – [Testing Load balancing] olivier
@@ Line 1: / Line 1: @@
 ====== Equal-cost multi-path routing (ECMP) ======
-This lab show an ECMP (feature called RADIX_MPATH on FreeBSD) routing example using BSDRP (FreeBSD 11).
+This lab is testing new ECMP routing code.
 ===== Presentation =====
@@ Line 79: / Line 79: @@
 ===== Static routing setup =====
-==== Router 1 (client) ====
+==== VM 1 (client) ====
-R1 is configured as a simple host with multiples aliases:
+VM1 is configured as a simple host with multiples aliases:
 <code>
-sysrc hostname=R1
+sysrc hostname=VM1 \
-sysrc gateway_enable=NO
+	gateway_enable=NO \
-sysrc ipv6_gateway_enable=NO
+	ipv6_gateway_enable=NO \
-sysrc ifconfig_vtnet0="inet 10.0.12.1/24"
+	ifconfig_vtnet0="inet 10.0.12.1/24" \
-sysrc ifconfig_vtnet0_alias1="inet 10.0.12.11/32"
+	ifconfig_vtnet0_alias1="inet 10.0.12.11/32" \
-sysrc ifconfig_vtnet0_alias2="inet 10.0.12.21/32"
+	ifconfig_vtnet0_alias2="inet 10.0.12.21/32" \
-sysrc ifconfig_vtnet0_alias3="inet 10.0.12.31/32"
+	ifconfig_vtnet0_alias3="inet 10.0.12.31/32" \
-sysrc ifconfig_vtnet0_alias4="inet 10.0.12.41/32"
+	ifconfig_vtnet0_alias4="inet 10.0.12.41/32" \
-sysrc ifconfig_vtnet0_alias5="inet 10.0.12.51/32"
+	ifconfig_vtnet0_alias5="inet 10.0.12.51/32" \
-sysrc ifconfig_vtnet0_ipv6="inet6 2001:db8:12::1 prefixlen 64"
+	ifconfig_vtnet0_ipv6="inet6 2001:db8:12::1 prefixlen 64" \
-sysrc ifconfig_vtnet0_alias61="inet6 2001:db8:12::11 prefixlen 64"
+	ifconfig_vtnet0_alias61="inet6 2001:db8:12::11 prefixlen 64" \
-sysrc ifconfig_vtnet0_alias62="inet6 2001:db8:12::21 prefixlen 64"
+	ifconfig_vtnet0_alias62="inet6 2001:db8:12::21 prefixlen 64" \
-sysrc ifconfig_vtnet0_alias63="inet6 2001:db8:12::31 prefixlen 64"
+	ifconfig_vtnet0_alias63="inet6 2001:db8:12::31 prefixlen 64" \
-sysrc ifconfig_vtnet0_alias64="inet6 2001:db8:12::41 prefixlen 64"
+	ifconfig_vtnet0_alias64="inet6 2001:db8:12::41 prefixlen 64" \
-sysrc ifconfig_vtnet0_alias65="inet6 2001:db8:12::51 prefixlen 64"
+	ifconfig_vtnet0_alias65="inet6 2001:db8:12::51 prefixlen 64" \
-sysrc defaultrouter="10.0.12.2"
+	defaultrouter="10.0.12.2" \
-sysrc ipv6_defaultrouter="2001:db8:12::2"
+	ipv6_defaultrouter="2001:db8:12::2"
 service hostname restart
 service netif restart
@@ Line 107: / Line 107: @@
 </code>
-==== Router 2 (ECMP router) ====
+==== VM 2 (ECMP router) ====
-R2 is a router with ECMP: 2 static routes toward the same destination but using 2 different next-hop.
+VM 2 is a router with ECMP: 2 static routes toward the same destination but using 2 different next-hop.
 <code>
-sysrc hostname=R2
+sysrc hostname=VM2 \
-sysrc ifconfig_vtnet0="inet 10.0.12.2/24"
+	ifconfig_vtnet0="inet 10.0.12.2/24" \
-sysrc ifconfig_vtnet1="inet 10.0.23.2/24"
+	ifconfig_vtnet1="inet 10.0.23.2/24" \
-sysrc ifconfig_vtnet2="inet 10.0.24.2/24"
+	ifconfig_vtnet2="inet 10.0.24.2/24" \
-sysrc ifconfig_vtnet0_ipv6="inet6 2001:db8:12::2 prefixlen 64"
+	ifconfig_vtnet0_ipv6="inet6 2001:db8:12::2 prefixlen 64" \
-sysrc ifconfig_vtnet1_ipv6="inet6 2001:db8:23::2 prefixlen 64"
+	ifconfig_vtnet1_ipv6="inet6 2001:db8:23::2 prefixlen 64" \
-sysrc ifconfig_vtnet2_ipv6="inet6 2001:db8:24::2 prefixlen 64"
+	ifconfig_vtnet2_ipv6="inet6 2001:db8:24::2 prefixlen 64" \
-sysrc static_routes="R3R6 R3R5 R4R6 R4R5"
+	static_routes="VM3VM6 VM3VM5 VM4VM6 VM4VM5" \
-sysrc route_R3R6="-net 10.0.56.0/24 10.0.23.3"
+	route_VM3VM6="-net 10.0.56.0/24 10.0.23.3" \
-sysrc route_R3R5="-net 10.0.35.0/24 10.0.23.3"
+	route_VM3VM5="-net 10.0.35.0/24 10.0.23.3" \
-sysrc route_R4R6="-net 10.0.56.0/24 10.0.24.4"
+	route_VM4VM6="-net 10.0.56.0/24 10.0.24.4" \
-sysrc route_R4R5="-net 10.0.45.0/24 10.0.24.4"
+	route_VM4VM5="-net 10.0.45.0/24 10.0.24.4" \
-sysrc ipv6_static_routes="R3R6 R3R5 R4R6 R4R5"
+	ipv6_static_routes="VM3VM6 VM3VM5 VM4VM6 VM4VM5" \
-sysrc ipv6_route_R3R6="2001:db8:56:: -prefixlen 64 2001:db8:23::3"
+	ipv6_route_VM3VM6="2001:db8:56:: -prefixlen 64 2001:db8:23::3" \
-sysrc ipv6_route_R3R5="2001:db8:35:: -prefixlen 64 2001:db8:23::3"
+	ipv6_route_VM3VM5="2001:db8:35:: -prefixlen 64 2001:db8:23::3" \
-sysrc ipv6_route_R4R6="2001:db8:56:: -prefixlen 64 2001:db8:24::4"
+	ipv6_route_VM4VM6="2001:db8:56:: -prefixlen 64 2001:db8:24::4" \
-sysrc ipv6_route_R4R5="2001:db8:45:: -prefixlen 64 2001:db8:24::4"
+	ipv6_route_VM4VM5="2001:db8:45:: -prefixlen 64 2001:db8:24::4"
 service hostname restart
 service netif restart
@@ Line 135: / Line 135: @@
 </code>
-==== Router 3 (simple router) ====
+Checking static route with multiple next-hop:
-R3 is a simple router.
+<code>
+[root@VM2]~# netstat -rn4 | grep 10.0.56.0
+.0.56.0/24       10.0.24.4          UGS      vtnet2
+.0.56.0/24       10.0.23.3          UGS      vtnet1
+[root@VM2]~# route -n get 10.0.56.0/24
+   route to: 10.0.56.0
+destination: 10.0.56.0
+       mask: 255.255.255.0
+    gateway: 10.0.24.4
+        fib: 0
+  interface: vtnet2
+      flags: <UP,GATEWAY,DONE,STATIC>
+ recvpipe  sendpipe  ssthresh  rtt,msec    mtu        weight    expire
+         0         0         0      1500       100         0
+[root@VM2]~# netstat -4onW
+Nexthop data
+Internet:
+Idx   Type         IFA                Gateway             Flags      Use Mtu         Netif     Addrif Refcnt Prepend
+       v4/resolve 10.0.23.2          vtnet1/resolve                   0   1500     vtnet1               1
+       v4/resolve 10.0.24.2          vtnet2/resolve                   0   1500     vtnet2               1
+       v4/resolve 10.0.12.2          vtnet0/resolve                   3   1500     vtnet0               1
+       v4/resolve 127.0.0.1          lo0/resolve        HS            6  16384        lo0               1
+       v4/resolve 127.0.0.1          lo0/resolve        HS            0  16384        lo0    vtnet2     1
+       v4/resolve 127.0.0.1          lo0/resolve        HS            0  16384        lo0    vtnet1     1
+       v4/resolve 127.0.0.1          lo0/resolve        HS            0  16384        lo0    vtnet0     1
+            v4/gw 10.0.24.2          10.0.24.4          GS            3   1500     vtnet2               2
+            v4/gw 10.0.23.2          10.0.23.3          GS            0   1500     vtnet1               2
+Nexthop groups data
+BUF: 192
+Internet:
+MpIdx NHIdx Weigh Slots            Gateway Netif      Refcnt
+      ---- ---- ----              ----      ----    1
+   100     1          10.0.24.4    vtnet2
+   100     1          10.0.23.3    vtnet1
+</code>
+==== VM 3 (router) ====
+VM 3 is a simple router.
 <code>
-sysrc hostname=R3
+sysrc hostname=VM3 \
-sysrc ifconfig_vtnet1="inet 10.0.23.3/24"
+	ifconfig_vtnet1="inet 10.0.23.3/24" \
-sysrc ifconfig_vtnet3="inet 10.0.35.3/24"
+	ifconfig_vtnet3="inet 10.0.35.3/24" \
-sysrc ifconfig_vtnet1_ipv6="inet6 2001:db8:23::3 prefixlen 64"
+	ifconfig_vtnet1_ipv6="inet6 2001:db8:23::3 prefixlen 64" \
-sysrc ifconfig_vtnet3_ipv6="inet6 2001:db8:35::3 prefixlen 64"
+	ifconfig_vtnet3_ipv6="inet6 2001:db8:35::3 prefixlen 64" \
-sysrc static_routes="R6 R1"
+	static_routes="VM6 VM1" \
-sysrc route_R1="-net 10.0.12.0/24 10.0.23.2"
+	route_VM1="-net 10.0.12.0/24 10.0.23.2" \
-sysrc route_R6="-net 10.0.56.0/24 10.0.35.5"
+	route_VM6="-net 10.0.56.0/24 10.0.35.5" \
-sysrc ipv6_static_routes="R6 R1"
+	ipv6_static_routes="VM6 VM1" \
-sysrc ipv6_route_R1="2001:db8:12:: -prefixlen 64 2001:db8:23::2"
+	ipv6_route_VM1="2001:db8:12:: -prefixlen 64 2001:db8:23::2" \
-sysrc ipv6_route_R6="2001:db8:56:: -prefixlen 64 2001:db8:35::5"
+	ipv6_route_VM6="2001:db8:56:: -prefixlen 64 2001:db8:35::5"
 service hostname restart
 service netif restart
@@ Line 157: / Line 196: @@
 </code>
-==== Router 4 (simple router) ====
+==== VM 4 (router) ====
-R4 is like R3, a simple router.
+VM 4 is like VM 3, a simple router.
 <code>
-sysrc hostname=R4
+sysrc hostname=VM4 \
-sysrc ifconfig_vtnet1="inet 10.0.24.4/24"
+	ifconfig_vtnet1="inet 10.0.24.4/24" \
-sysrc ifconfig_vtnet3="inet 10.0.45.4/24"
+	ifconfig_vtnet3="inet 10.0.45.4/24" \
-sysrc ifconfig_vtnet1_ipv6="inet6 2001:db8:24::4 prefixlen 64"
+	ifconfig_vtnet1_ipv6="inet6 2001:db8:24::4 prefixlen 64" \
-sysrc ifconfig_vtnet3_ipv6="inet6 2001:db8:45::4 prefixlen 64"
+	ifconfig_vtnet3_ipv6="inet6 2001:db8:45::4 prefixlen 64" \
-sysrc static_routes="R6 R1"
+	static_routes="VM6 VM1" \
-sysrc route_R1="-net 10.0.12.0/24 10.0.24.2"
+	route_VM1="-net 10.0.12.0/24 10.0.24.2" \
-sysrc route_R6="-net 10.0.56.0/24 10.0.45.5"
+	route_VM6="-net 10.0.56.0/24 10.0.45.5" \
-sysrc ipv6_static_routes="R6 R1"
+	ipv6_static_routes="VM6 VM1" \
-sysrc ipv6_route_R1="2001:db8:12:: -prefixlen 64 2001:db8:24::2"
+	ipv6_route_VM1="2001:db8:12:: -prefixlen 64 2001:db8:24::2" \
-sysrc ipv6_route_R6="2001:db8:56:: -prefixlen 64 2001:db8:45::5"
+	ipv6_route_VM6="2001:db8:56:: -prefixlen 64 2001:db8:45::5"
 service hostname restart
 service netif restart
@@ Line 179: / Line 218: @@
 </code>
-==== Router 5 (ECMP router) ====
+==== VM 5 (ECMP router) ====
-R5 is like R2, a router with ECMP: 2 static routing toward the same destination but using 2 different next-hop..
+VM 5 is like VM 2, a router with ECMP: 2 static routing toward the same destination but using 2 different next-hop..
 <code>
-sysrc hostname=R5
+sysrc hostname=VM5 \
-sysrc ifconfig_vtnet2="inet 10.0.35.5/24"
+	ifconfig_vtnet2="inet 10.0.35.5/24" \
-sysrc ifconfig_vtnet3="inet 10.0.45.5/24"
+	ifconfig_vtnet3="inet 10.0.45.5/24" \
-sysrc ifconfig_vtnet4="inet 10.0.56.5/24"
+	ifconfig_vtnet4="inet 10.0.56.5/24" \
-sysrc ifconfig_vtnet2_ipv6="inet6 2001:db8:35::5 prefixlen 64"
+	ifconfig_vtnet2_ipv6="inet6 2001:db8:35::5 prefixlen 64" \
-sysrc ifconfig_vtnet3_ipv6="inet6 2001:db8:45::5 prefixlen 64"
+	ifconfig_vtnet3_ipv6="inet6 2001:db8:45::5 prefixlen 64" \
-sysrc ifconfig_vtnet4_ipv6="inet6 2001:db8:56::5 prefixlen 64"
+	ifconfig_vtnet4_ipv6="inet6 2001:db8:56::5 prefixlen 64" \
-sysrc static_routes="R3R1 R3R2 R4R1 R4R2"
+	static_routes="VM3VM1 VM3VM2 VM4VM1 VM4VM2" \
-sysrc route_R3R1="-net 10.0.12.0/24 10.0.35.3"
+	route_VM3VM1="-net 10.0.12.0/24 10.0.35.3" \
-sysrc route_R3R2="-net 10.0.23.0/24 10.0.35.3"
+	route_VM3VM2="-net 10.0.23.0/24 10.0.35.3" \
-sysrc route_R4R1="-net 10.0.12.0/24 10.0.45.4"
+	route_VM4VM1="-net 10.0.12.0/24 10.0.45.4" \
-sysrc route_R4R2="-net 10.0.24.0/24 10.0.45.4"
+	route_VM4VM2="-net 10.0.24.0/24 10.0.45.4" \
-sysrc ipv6_static_routes="R3R1 R3R2 R4R1 R4R2"
+	ipv6_static_routes="VM3VM1 VM3VM2 VM4VM1 VM4VM2" \
-sysrc ipv6_route_R3R1="2001:db8:12:: -prefixlen 64 2001:db8:35::3"
+	ipv6_route_VM3VM1="2001:db8:12:: -prefixlen 64 2001:db8:35::3" \
-sysrc ipv6_route_R3R2="2001:db8:23:: -prefixlen 64 2001:db8:35::3"
+	ipv6_route_VM3VM2="2001:db8:23:: -prefixlen 64 2001:db8:35::3" \
-sysrc ipv6_route_R4R1="2001:db8:12:: -prefixlen 64 2001:db8:45::4"
+	ipv6_route_VM4VM1="2001:db8:12:: -prefixlen 64 2001:db8:45::4" \
-sysrc ipv6_route_R4R2="2001:db8:24:: -prefixlen 64 2001:db8:45::4"
+	ipv6_route_VM4VM2="2001:db8:24:: -prefixlen 64 2001:db8:45::4"
 service hostname restart
 service netif restart
@@ Line 207: / Line 246: @@
 </code>
-==== Router 6 (server) ====
+==== VM 6 (server) ====
-R6, like R1 is configured as a simple host with multiple aliases:
+VM 6, like VM 1 is configured as a simple host with multiple aliases:
 <code>
-sysrc hostname=R6
+sysrc hostname=VM6 \
-sysrc gateway_enable=NO
+	gateway_enable=NO \
-sysrc ipv6_gateway_enable=NO
+	ipv6_gateway_enable=NO \
-sysrc ifconfig_vtnet4="inet 10.0.56.6/24"
+	ifconfig_vtnet4="inet 10.0.56.6/24" \
-sysrc ifconfig_vtnet4_alias1="inet 10.0.56.16/32"
+	ifconfig_vtnet4_alias1="inet 10.0.56.16/32" \
-sysrc ifconfig_vtnet4_alias2="inet 10.0.56.26/32"
+	ifconfig_vtnet4_alias2="inet 10.0.56.26/32" \
-sysrc ifconfig_vtnet4_alias3="inet 10.0.56.36/32"
+	ifconfig_vtnet4_alias3="inet 10.0.56.36/32" \
-sysrc ifconfig_vtnet4_alias4="inet 10.0.56.46/32"
+	ifconfig_vtnet4_alias4="inet 10.0.56.46/32" \
-sysrc ifconfig_vtnet4_alias5="inet 10.0.56.56/32"
+	ifconfig_vtnet4_alias5="inet 10.0.56.56/32" \
-sysrc ifconfig_vtnet4_ipv6="inet6 2001:db8:56::6 prefixlen 64"
+	ifconfig_vtnet4_ipv6="inet6 2001:db8:56::6 prefixlen 64" \
-sysrc ifconfig_vtnet4_alias61="inet6 2001:db8:56::16 prefixlen 64"
+	ifconfig_vtnet4_alias61="inet6 2001:db8:56::16 prefixlen 64" \
-sysrc ifconfig_vtnet4_alias62="inet6 2001:db8:56::26 prefixlen 64"
+	ifconfig_vtnet4_alias62="inet6 2001:db8:56::26 prefixlen 64" \
-sysrc ifconfig_vtnet4_alias63="inet6 2001:db8:56::36 prefixlen 64"
+	ifconfig_vtnet4_alias63="inet6 2001:db8:56::36 prefixlen 64" \
-sysrc ifconfig_vtnet4_alias64="inet6 2001:db8:56::46 prefixlen 64"
+	ifconfig_vtnet4_alias64="inet6 2001:db8:56::46 prefixlen 64" \
-sysrc ifconfig_vtnet4_alias65="inet6 2001:db8:56::56 prefixlen 64"
+	ifconfig_vtnet4_alias65="inet6 2001:db8:56::56 prefixlen 64" \
-sysrc defaultrouter="10.0.56.5"
+	defaultrouter="10.0.56.5" \
-sysrc ipv6_defaultrouter="2001:db8:56::5"
+	ipv6_defaultrouter="2001:db8:56::5"
 service hostname restart
 service netif restart
@@ Line 237: / Line 276: @@
 ==== Testing Load balancing ====
-Flows from R1 to R6 should be load-balanced by R2 toward R3 and R4. Let's check using multiple sources and destination IP addresses:
+Flows from VM1 to VM6 should be "flow-id shared" by VM2 toward VM3 and VM4. Let's check using multiple sources and destination IP addresses.
+Using ping record feature like with this example:
+<code>
+[root@VM1]~# ping -c1 -R -S 10.0.12.21 10.0.56.26
+PING 10.0.56.26 (10.0.56.26) from 10.0.12.21: 56 data bytes
+bytes from 10.0.56.26: icmp_seq=0 ttl=61 time=0.455 ms
+RR:     10.0.24.2
+.0.45.4
+.0.56.5
+.0.56.26
+.0.45.5
+.0.24.4
+.0.12.2
+.0.12.21
-^ Source ^ Destination ^ Ongoing path selected by R2 ^ Returning path selected by R5^
+--- 10.0.56.26 ping statistics ---
-| 10.0.12.1 | 10.0.56.6 | R3 | R3 |
+packets transmitted, 1 packets received, 0.0% packet loss
-| 10.0.12.11 | 10.0.56.16 | R3 | R3 |
+round-trip min/avg/max/stddev = 0.455/0.455/0.455/0.000 ms
-| 10.0.12.21 | 10.0.56.26 | R3 | R3 |
+</code>
-| 10.0.12.31 | 10.0.56.36 | R3 | R3 |
-| 10.0.12.41 | 10.0.56.46 | R3 | R3 |
-| 10.0.12.51 | 10.0.56.56 | R3 | R3 |
-| 2001:db8:12::1 | 2001:db8:56::6 | R3 | R3 |
-| 2001:db8:12::11 | 2001:db8:56::16 | R3 | R3 |
-| 2001:db8:12::21 | 2001:db8:56::26 | R3 | R3 |
-| 2001:db8:12::31 | 2001:db8:56::36 | R3 | R3 |
-| 2001:db8:12::41 | 2001:db8:56::46 | R3 | R3 |
-| 2001:db8:12::51 | 2001:db8:56::56 | R3 | R3 |
-| 10.0.12.1 | 10.0.45.5 | R4 | R3 |
-<note>There is no flow load-balancing with RADIX MPATH on FreeBSD 11, [[https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=225792|seems broken with tryforward]], put proposed patch disable tryforward(fastforward) if RADIX_MPATH is enabled.</note>
+^ Source ^ Destination ^ Ongoing path selected by VM2 ^ Returning path selected by VM5^
+| 10.0.12.1 | 10.0.35.5 | VM3 | VM4 |
+| 10.0.12.1 | 10.0.45.5 | VM4 | VM4 |
+| 10.0.12.1 | 10.0.56.6 | VM3 | VM4 |
+| 10.0.12.11 | 10.0.56.16 | VM3 | VM4 |
+| 10.0.12.21 | 10.0.56.26 | VM3 | VM4 |
+| 10.0.12.31 | 10.0.56.36 | VM3 | VM4 |
+| 10.0.12.41 | 10.0.56.46 | VM3 | VM4 |
+| 10.0.12.51 | 10.0.56.56 | VM3 | VM4 |
+| 2001:db8:12::1 | 2001:db8:56::6 | error | error |
+| 2001:db8:12::11 | 2001:db8:56::16 | error | error |
+| 2001:db8:12::21 | 2001:db8:56::26 | error | error |
+| 2001:db8:12::31 | 2001:db8:56::36 | error | error |
+| 2001:db8:12::41 | 2001:db8:56::46 | error | error |
+| 2001:db8:12::51 | 2001:db8:56::56 | error | error |
+Notice inet6 bug: Cannot ping itself neither neighbor
+<code>
+[root@VM1]~# ifconfig vtnet0 inet6
+vtnet0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
+        options=80028<VLAN_MTU,JUMBO_MTU,LINKSTATE>
+        inet6 fe80::5a9c:fcff:fe01:201%vtnet0 prefixlen 64 scopeid 0x1
+        inet6 2001:db8:12::1 prefixlen 64
+        inet6 2001:db8:12::11 prefixlen 64
+        inet6 2001:db8:12::21 prefixlen 64
+        inet6 2001:db8:12::31 prefixlen 64
+        inet6 2001:db8:12::41 prefixlen 64
+        inet6 2001:db8:12::51 prefixlen 64
+        nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
+[root@VM1]~# ping6 -c 1 fe80::5a9c:fcff:fe01:201%vtnet0
+PING6(56=40+8+8 bytes) fe80::5a9c:fcff:fe01:201%vtnet0 --> fe80::5a9c:fcff:fe01:201%vtnet0
+--- fe80::5a9c:fcff:fe01:201%vtnet0 ping6 statistics ---
+packets transmitted, 0 packets received, 100.0% packet loss
+[root@VM1]~# ping6 -c 12001:db8:12::2
+ping6: illegal number of packets -- 12001:db8:12::2
+[root@VM1]~# ping6 -c 1 2001:db8:12::2
+PING6(56=40+8+8 bytes) 2001:db8:12::1 --> 2001:db8:12::2
+--- 2001:db8:12::2 ping6 statistics ---
+packets transmitted, 0 packets received, 100.0% packet loss
+</code>
+==== Testing Fail Over ====
+VM4 is the preferred hop, so shutdown this VM to check behavior:
+<code>
+[root@VM4]~# shutdown -p now
+Shutdown NOW!
+</code>
+<code>
+[root@VM2]~# netstat -4onW
+Nexthop data
+Internet:
+Idx   Type         IFA                Gateway             Flags      Use Mtu         Netif     Addrif Refcnt Prepend
+       v4/resolve 10.0.23.2          vtnet1/resolve                   0   1500     vtnet1               1
+       v4/resolve 10.0.24.2          vtnet2/resolve                   0   1500     vtnet2               1
+       v4/resolve 10.0.12.2          vtnet0/resolve                  11   1500     vtnet0               1
+       v4/resolve 127.0.0.1          lo0/resolve        HS            6  16384        lo0               1
+       v4/resolve 127.0.0.1          lo0/resolve        HS            0  16384        lo0    vtnet2     1
+       v4/resolve 127.0.0.1          lo0/resolve        HS            0  16384        lo0    vtnet1     1
+       v4/resolve 127.0.0.1          lo0/resolve        HS            0  16384        lo0    vtnet0     1
+            v4/gw 10.0.24.2          10.0.24.4          GS           10   1500     vtnet2               2
+            v4/gw 10.0.23.2          10.0.23.3          GS            2   1500     vtnet1               2
+Nexthop groups data
+BUF: 192
+Internet:
+MpIdx NHIdx Weigh Slots            Gateway Netif      Refcnt
+      ---- ---- ----              ----      ----    1
+   100     1          10.0.24.4    vtnet2
+   100     1          10.0.23.3    vtnet1
+</code>
+<code>
+[root@VM1]~# ping -c1 -R -S 10.0.12.21 10.0.56.26
+PING 10.0.56.26 (10.0.56.26) from 10.0.12.21: 56 data bytes
+--- 10.0.56.26 ping statistics ---
+packets transmitted, 0 packets received, 100.0% packet loss
+</code>