documentation:examples:multi-tenant_router_and_firewall
no way to compare when less than two revisions
Differences
This shows you the differences between two versions of the page.
| — | documentation:examples:multi-tenant_router_and_firewall [2017/08/31 16:25] (current) – created - external edit 127.0.0.1 | ||
|---|---|---|---|
| Line 1: | Line 1: | ||
| + | ====== Multi-tenant router or firewall ====== | ||
| + | {{description> | ||
| + | This lab shows how to create multi-tenant router or firewall using jail/vnet (available since BSDRP 1.80). | ||
| + | |||
| + | ===== Presentation ===== | ||
| + | |||
| + | ==== Network diagram ==== | ||
| + | |||
| + | Lab build following [[documentation: | ||
| + | |||
| + | Here is the logical and physical view: | ||
| + | |||
| + | {{: | ||
| + | |||
| + | ==== Setting-up a virtual lab ==== | ||
| + | |||
| + | === Downloading BSD Router Project images === | ||
| + | |||
| + | Download BSDRP serial image (prevent to have to use an X display) on Sourceforge. | ||
| + | |||
| + | === Download Lab scripts ==== | ||
| + | |||
| + | More information on these BSDRP lab scripts available on [[documentation: | ||
| + | |||
| + | Start the lab with full-meshed 5 routers and one shared LAN, on this example using bhyve lab script on FreeBSD: | ||
| + | |||
| + | < | ||
| + | [root@FreeBSD]~# | ||
| + | BSD Router Project (http:// | ||
| + | Setting-up a virtual lab with 5 VM(s): | ||
| + | - Working directory: /tmp/BSDRP | ||
| + | - Each VM have 1 core(s) and 256M RAM | ||
| + | - Switch mode: bridge + tap | ||
| + | - 1 LAN(s) between all VM | ||
| + | - Full mesh Ethernet links between each VM | ||
| + | VM 1 have the following NIC: | ||
| + | - vtnet0 connected to VM 2 | ||
| + | - vtnet1 connected to VM 3 | ||
| + | - vtnet2 connected to VM 4 | ||
| + | - vtnet3 connected to VM 5 | ||
| + | - vtnet4 connected to LAN number 1 | ||
| + | VM 2 have the following NIC: | ||
| + | - vtnet0 connected to VM 1 | ||
| + | - vtnet1 connected to VM 3 | ||
| + | - vtnet2 connected to VM 4 | ||
| + | - vtnet3 connected to VM 5 | ||
| + | - vtnet4 connected to LAN number 1 | ||
| + | VM 3 have the following NIC: | ||
| + | - vtnet0 connected to VM 1 | ||
| + | - vtnet1 connected to VM 2 | ||
| + | - vtnet2 connected to VM 4 | ||
| + | - vtnet3 connected to VM 5 | ||
| + | - vtnet4 connected to LAN number 1 | ||
| + | VM 4 have the following NIC: | ||
| + | - vtnet0 connected to VM 1 | ||
| + | - vtnet1 connected to VM 2 | ||
| + | - vtnet2 connected to VM 3 | ||
| + | - vtnet3 connected to VM 5 | ||
| + | - vtnet4 connected to LAN number 1 | ||
| + | VM 5 have the following NIC: | ||
| + | - vtnet0 connected to VM 1 | ||
| + | - vtnet1 connected to VM 2 | ||
| + | - vtnet2 connected to VM 3 | ||
| + | - vtnet3 connected to VM 4 | ||
| + | - vtnet4 connected to LAN number 1 | ||
| + | For connecting to VM' | ||
| + | - VM 1 : cu -l /dev/nmdm1B | ||
| + | - VM 2 : cu -l /dev/nmdm2B | ||
| + | - VM 3 : cu -l /dev/nmdm3B | ||
| + | - VM 4 : cu -l /dev/nmdm4B | ||
| + | - VM 5 : cu -l /dev/nmdm5B | ||
| + | </ | ||
| + | ===== Configuration ===== | ||
| + | |||
| + | * Router 4 (R4) hosts the 3 routers/ | ||
| + | * Router 1 (R1) belongs to customer 1, router 2 (R2) to customer 2 and router 3 (R3) to customer 3. | ||
| + | * Router 5 (R5) simulates a simple Internet host. | ||
| + | |||
| + | ==== Router 5: Simple Internet host ==== | ||
| + | |||
| + | R5 simulate a simple Internet host: | ||
| + | |||
| + | < | ||
| + | sysrc hostname=R5 | ||
| + | hostname R5 | ||
| + | sysrc ifconfig_vtnet3=" | ||
| + | sysrc -x gateway_enable | ||
| + | sysrc -x ipv6_gateway_enable | ||
| + | service netif restart | ||
| + | service routing restart | ||
| + | config save | ||
| + | </ | ||
| + | |||
| + | ==== Router 1: Customer 1 workstation==== | ||
| + | |||
| + | R1 simulate customer 1's workstation, | ||
| + | |||
| + | < | ||
| + | sysrc hostname=R1 | ||
| + | hostname R1 | ||
| + | sysrc ifconfig_vtnet4=" | ||
| + | sysrc vlans_vtnet4=" | ||
| + | sysrc ifconfig_vtnet4_1=" | ||
| + | sysrc defaultrouter=" | ||
| + | sysrc -x gateway_enable | ||
| + | sysrc -x ipv6_gateway_enable | ||
| + | service netif restart | ||
| + | service routing restart | ||
| + | ssh-keygen -f / | ||
| + | config save | ||
| + | </ | ||
| + | |||
| + | Then display the public SSH key (need to declare it into the customer 1's firewall): | ||
| + | < | ||
| + | cat .ssh/ | ||
| + | ssh-rsa (...) root@R1 | ||
| + | </ | ||
| + | |||
| + | ==== Router 2: Customer 2 workstation==== | ||
| + | |||
| + | R2 simulate customer 2's workstation, | ||
| + | |||
| + | < | ||
| + | sysrc hostname=R2 | ||
| + | hostname R2 | ||
| + | sysrc ifconfig_vtnet4=" | ||
| + | sysrc vlans_vtnet4=" | ||
| + | sysrc ifconfig_vtnet4_2=" | ||
| + | sysrc defaultrouter=" | ||
| + | sysrc -x gateway_enable | ||
| + | sysrc -x ipv6_gateway_enable | ||
| + | service netif restart | ||
| + | service routing restart | ||
| + | ssh-keygen -f / | ||
| + | config save | ||
| + | </ | ||
| + | |||
| + | Then display the public SSH key (need to declare it into the customer 2's firewall): | ||
| + | < | ||
| + | cat .ssh/ | ||
| + | ssh-rsa (...) root@R2 | ||
| + | </ | ||
| + | ==== Router 3: Customer 3 workstation==== | ||
| + | |||
| + | R3 simulate customer 3's workstation, | ||
| + | |||
| + | < | ||
| + | sysrc hostname=R3 | ||
| + | hostname R3 | ||
| + | sysrc ifconfig_vtnet4=" | ||
| + | sysrc vlans_vtnet4=" | ||
| + | sysrc ifconfig_vtnet4_3=" | ||
| + | sysrc defaultrouter=" | ||
| + | sysrc -x gateway_enable | ||
| + | sysrc -x ipv6_gateway_enable | ||
| + | service netif restart | ||
| + | service routing restart | ||
| + | ssh-keygen -f / | ||
| + | config save | ||
| + | </ | ||
| + | |||
| + | Then display the public SSH key (need to declare it into the customer 3's firewall): | ||
| + | < | ||
| + | cat .ssh/ | ||
| + | ssh-rsa (...) root@R3 | ||
| + | </ | ||
| + | ==== Router 4: multi-tenant ipfw firewall ==== | ||
| + | |||
| + | Router 4 is a multi-tenant ipfw firewall: It hosts 3 firewalls for each customer. | ||
| + | |||
| + | Then we will configure: | ||
| + | * Bridge and VLAN interfaces | ||
| + | * Enable ipfw (firewall modules needs to be loaded on the host for being available into jails) | ||
| + | |||
| + | < | ||
| + | sysrc hostname=R4 | ||
| + | sysrc cloned_interfaces=" | ||
| + | sysrc ifconfig_bridge0=" | ||
| + | sysrc ifconfig_vtnet3=" | ||
| + | sysrc ifconfig_vtnet4=" | ||
| + | sysrc vlans_vtnet4=" | ||
| + | sysrc ifconfig_vtnet4_1=" | ||
| + | sysrc ifconfig_vtnet4_2=" | ||
| + | sysrc ifconfig_vtnet4_3=" | ||
| + | sysrc firewall_enable=" | ||
| + | sysrc firewall_nat_enable=" | ||
| + | sysrc firewall_type=" | ||
| + | service netif restart | ||
| + | hostname R4 | ||
| + | service ipfw start | ||
| + | config save | ||
| + | </ | ||
| + | |||
| + | Then install the customers SSH public keys: | ||
| + | |||
| + | < | ||
| + | echo " | ||
| + | echo " | ||
| + | echo " | ||
| + | </ | ||
| + | |||
| + | Create 3 jailed firewalls, one for each customers: | ||
| + | < | ||
| + | tenant -c -j customer1 -f / | ||
| + | tenant -c -j customer2 -f / | ||
| + | tenant -c -j customer3 -f / | ||
| + | </ | ||
| + | |||
| + | Last step, because they are virtual firewalls and not simple routers, we will enable firewall in open mode into their internal rc.conf for allowing customers to SSH into them: | ||
| + | |||
| + | < | ||
| + | sysrc -f / | ||
| + | sysrc -f / | ||
| + | sysrc -f / | ||
| + | sysrc -f / | ||
| + | sysrc -f / | ||
| + | sysrc -f / | ||
| + | sysrc -f / | ||
| + | sysrc -f / | ||
| + | sysrc -f / | ||
| + | </ | ||
| + | |||
| + | Configuration will be now automatically saved when changed detected into /etc, then you do not need to use " | ||
| + | |||
| + | Then to start the jails: | ||
| + | |||
| + | < | ||
| + | service jail start | ||
| + | </ | ||
| + | |||
| + | |||
| + | ===== Customers firewalls configuration ===== | ||
| + | |||
| + | Each customer should be able to ssh into their new firewalls using their SSH keys. | ||
| + | |||
| + | ==== Customer 1 ==== | ||
| + | |||
| + | From customer 1's workstation R1: | ||
| + | < | ||
| + | [root@R1]~# ssh 10.0.0.254 | ||
| + | The authenticity of host ' | ||
| + | ECDSA key fingerprint is SHA256: | ||
| + | No matching host key fingerprint found in DNS. | ||
| + | Are you sure you want to continue connecting (yes/no)? yes | ||
| + | Warning: Permanently added ' | ||
| + | BSD Router project (BSDRP) (c) 2009-2017, The BSDRP Development Team | ||
| + | All rights reserved. | ||
| + | BSDRP is under the Simplified BSD license. | ||
| + | |||
| + | Documentation: | ||
| + | |||
| + | Discover BSDRP tools with " | ||
| + | |||
| + | Keyboard layout can be changed with this command: | ||
| + | kbdcontrol -l keymap_file (< | ||
| + | root has logged on pts/0 from 10.0.0.1. | ||
| + | [root@customer1]~# | ||
| + | </ | ||
| + | |||
| + | Now connected to his firewall, this customer can configure its own firewall rules: | ||
| + | |||
| + | < | ||
| + | sysrc -x firewall_type | ||
| + | sysrc firewall_script="/ | ||
| + | |||
| + | cat > / | ||
| + | #!/bin/sh | ||
| + | fwcmd="/ | ||
| + | ext_if=" | ||
| + | int_if=" | ||
| + | ${fwcmd} -f flush | ||
| + | ${fwcmd} nat 1 config if ${ext_if} same_ports deny_in unreg_only reset | ||
| + | ${fwcmd} add pass ip from any to any via lo0 | ||
| + | ${fwcmd} add pass ip from any to any via ${int_if} | ||
| + | ${fwcmd} add nat 1 ip from any to any via ${ext_if} | ||
| + | ' | ||
| + | service ipfw restart | ||
| + | config save | ||
| + | </ | ||
| + | |||
| + | Check firewall rules: | ||
| + | < | ||
| + | [root@customer1]~# | ||
| + | 00100 0 0 allow ip from any to any via lo0 | ||
| + | 00200 91 7756 allow ip from any to any via vtnet4.1 | ||
| + | 00300 0 0 nat 1 ip from any to any via epair1b | ||
| + | 65535 0 0 deny ip from any to any | ||
| + | </ | ||
| + | |||
| + | Now, from R1, try to reach public Internet server R5: | ||
| + | |||
| + | < | ||
| + | [root@R1]~# ping -c 3 10.254.254.5 | ||
| + | PING 10.254.254.5 (10.254.254.5): | ||
| + | 64 bytes from 10.254.254.5: | ||
| + | 64 bytes from 10.254.254.5: | ||
| + | 64 bytes from 10.254.254.5: | ||
| + | |||
| + | --- 10.254.254.5 ping statistics --- | ||
| + | 3 packets transmitted, | ||
| + | round-trip min/ | ||
| + | </ | ||
| + | |||
| + | ==== Customer 2 ==== | ||
| + | |||
| + | From customer 2's workstation R1: | ||
| + | < | ||
| + | [root@R2]~# ssh 10.0.0.254 | ||
| + | The authenticity of host ' | ||
| + | ECDSA key fingerprint is SHA256: | ||
| + | No matching host key fingerprint found in DNS. | ||
| + | Are you sure you want to continue connecting (yes/no)? yes | ||
| + | Warning: Permanently added ' | ||
| + | BSD Router project (BSDRP) (c) 2009-2017, The BSDRP Development Team | ||
| + | All rights reserved. | ||
| + | BSDRP is under the Simplified BSD license. | ||
| + | |||
| + | Documentation: | ||
| + | |||
| + | Discover BSDRP tools with " | ||
| + | |||
| + | Keyboard layout can be changed with this command: | ||
| + | kbdcontrol -l keymap_file (< | ||
| + | root has logged on pts/0 from 10.0.0.1. | ||
| + | [root@customer2]~# | ||
| + | </ | ||
| + | |||
| + | Now connected to his firewall, this customer can configure its own firewall rules: | ||
| + | |||
| + | < | ||
| + | sysrc -x firewall_type | ||
| + | sysrc firewall_script="/ | ||
| + | |||
| + | cat > / | ||
| + | #!/bin/sh | ||
| + | fwcmd="/ | ||
| + | ext_if=" | ||
| + | int_if=" | ||
| + | ${fwcmd} -f flush | ||
| + | ${fwcmd} nat 1 config if ${ext_if} same_ports deny_in unreg_only reset | ||
| + | ${fwcmd} add pass ip from any to any via lo0 | ||
| + | ${fwcmd} add pass ip from any to any via ${int_if} | ||
| + | ${fwcmd} add nat 1 ip from any to any via ${ext_if} | ||
| + | ' | ||
| + | service ipfw restart | ||
| + | config save | ||
| + | </ | ||
| + | |||
| + | Check firewall rules: | ||
| + | < | ||
| + | [root@customer2]~# | ||
| + | 00100 0 0 allow ip from any to any via lo0 | ||
| + | 00200 91 7756 allow ip from any to any via vtnet4.2 | ||
| + | 00300 0 0 nat 1 ip from any to any via epair2b | ||
| + | 65535 0 0 deny ip from any to any | ||
| + | </ | ||
| + | |||
| + | Now, from R2, try to public Internet server R5: | ||
| + | |||
| + | < | ||
| + | [root@R2]~# ping -c 3 10.254.254.5 | ||
| + | PING 10.254.254.5 (10.254.254.5): | ||
| + | 64 bytes from 10.254.254.5: | ||
| + | 64 bytes from 10.254.254.5: | ||
| + | 64 bytes from 10.254.254.5: | ||
| + | |||
| + | --- 10.254.254.5 ping statistics --- | ||
| + | 3 packets transmitted, | ||
| + | round-trip min/ | ||
| + | </ | ||
| + | |||
| + | ==== Customer 3 ==== | ||
| + | |||
| + | From customer 3's workstation R1: | ||
| + | < | ||
| + | [root@R3]~# ssh 10.0.0.254 | ||
| + | The authenticity of host ' | ||
| + | ECDSA key fingerprint is SHA256: | ||
| + | No matching host key fingerprint found in DNS. | ||
| + | Are you sure you want to continue connecting (yes/no)? yes | ||
| + | Warning: Permanently added ' | ||
| + | BSD Router project (BSDRP) (c) 2009-2017, The BSDRP Development Team | ||
| + | All rights reserved. | ||
| + | BSDRP is under the Simplified BSD license. | ||
| + | |||
| + | Documentation: | ||
| + | |||
| + | Discover BSDRP tools with " | ||
| + | |||
| + | Keyboard layout can be changed with this command: | ||
| + | kbdcontrol -l keymap_file (< | ||
| + | root has logged on pts/0 from 10.0.0.1. | ||
| + | [root@customer3]~# | ||
| + | </ | ||
| + | |||
| + | Now connected to his firewall, this customer can configure its own firewall rules: | ||
| + | |||
| + | < | ||
| + | sysrc -x firewall_type | ||
| + | sysrc firewall_script="/ | ||
| + | |||
| + | cat > / | ||
| + | #!/bin/sh | ||
| + | fwcmd="/ | ||
| + | ext_if=" | ||
| + | int_if=" | ||
| + | ${fwcmd} -f flush | ||
| + | ${fwcmd} nat 1 config if ${ext_if} same_ports deny_in unreg_only reset | ||
| + | ${fwcmd} add pass ip from any to any via lo0 | ||
| + | ${fwcmd} add pass ip from any to any via ${int_if} | ||
| + | ${fwcmd} add nat 1 ip from any to any via ${ext_if} | ||
| + | ' | ||
| + | service ipfw restart | ||
| + | config save | ||
| + | </ | ||
| + | |||
| + | Check firewall rules: | ||
| + | < | ||
| + | [root@customer3]~# | ||
| + | 00100 0 0 allow ip from any to any via lo0 | ||
| + | 00200 91 7756 allow ip from any to any via vtnet4.3 | ||
| + | 00300 0 0 nat 1 ip from any to any via epair3b | ||
| + | 65535 0 0 deny ip from any to any | ||
| + | </ | ||
| + | |||
| + | Now, from R3, try to public Internet server R5: | ||
| + | |||
| + | < | ||
| + | [root@R3]~# ping -c 3 10.254.254.5 | ||
| + | PING 10.254.254.5 (10.254.254.5): | ||
| + | 64 bytes from 10.254.254.5: | ||
| + | 64 bytes from 10.254.254.5: | ||
| + | 64 bytes from 10.254.254.5: | ||
| + | |||
| + | --- 10.254.254.5 ping statistics --- | ||
| + | 3 packets transmitted, | ||
| + | round-trip min/ | ||
| + | </ | ||
| + | ===== Using pf firewall in place of ipfw ===== | ||
| + | |||
| + | pf need a little more configuration because by default /dev/pf is hidden from jail. | ||
| + | |||
| + | Then, on the host we need to: | ||
| + | - In place of loading the ipfw/ | ||
| + | - Modify default devd rules for allowing jails to see /dev/pf (if you want to use tcpdump inside your jail, you should use bpf device too) | ||
| + | - Replacing nojail tag by nojailvnet tag into / | ||
| + | |||
| + | Preparing configuration: | ||
| + | |||
| + | < | ||
| + | cat > / | ||
| + | [devfsrules_jailpf=4] | ||
| + | add include $devfsrules_hide_all | ||
| + | add include $devfsrules_unhide_basic | ||
| + | add include $devfsrules_unhide_login | ||
| + | add path ' | ||
| + | ' | ||
| + | pf_enable=" | ||
| + | pf_flags=" | ||
| + | echo "set skip on {lo1 vtnet4}" | ||
| + | </ | ||
| + | |||
| + | Now reloading devd and loading pf module: | ||
| + | < | ||
| + | service pf start | ||
| + | service devfs restart | ||
| + | </ | ||
| + | |||
| + | You can now declare pf in place of ipfw into your jails rc.conf: | ||
| + | |||
| + | < | ||
| + | sysrc -f / | ||
| + | echo "pass all" > / | ||
| + | sysrc -f / | ||
| + | echo "pass all" > / | ||
| + | sysrc -f / | ||
| + | echo "pass all" > / | ||
| + | </ | ||
| + | |||
| + | Now you can start customers jails, and let customers SSH into their firewalls and configure their own rules: | ||
| + | < | ||
| + | [root@customer2]~# | ||
| + | pass all flags S/SA keep state | ||
| + | </ | ||
| + | |||
| + | |||
| + | ===== Under the hood: jails-on-nanobsd ===== | ||
| + | |||
| + | [[https:// | ||
| + | |||
| + | Then these jails need to be configured for a nanobsd: | ||
| + | - Being nullfs based for being hosted on a read-only root filesystem | ||
| + | - Have their /etc and /var into tmpfs disks (then we need to populate these directory before each start) | ||
| + | - Configuration changes need to be saved with nanobsd configuration tools, like " | ||
| + | And on the host: | ||
| + | - [[https:// | ||
| + | |||
| + | Here are examples of configuration files generated: | ||
| + | |||
| + | host jail.conf: | ||
| + | |||
| + | < | ||
| + | customer1 { | ||
| + | jid = 1; | ||
| + | path = "/ | ||
| + | # Because we are using jail on nanobsd, the jail directories are volatil (mounted into /var/jails) | ||
| + | # They didn't exist after a reboot, then we need to create jail directories with exec.prestart | ||
| + | # But mount.* instructions are called before exec.prestart, | ||
| + | # into the exec.prestart | ||
| + | # | ||
| + | # | ||
| + | # | ||
| + | host.hostname = " | ||
| + | vnet new; | ||
| + | allow.chflags = 1; | ||
| + | exec.start | ||
| + | exec.stop | ||
| + | exec.clean; | ||
| + | exec.consolelog = "/ | ||
| + | exec.poststop | ||
| + | # Commands to run on host before jail is created | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | |||
| + | # Copy reference and backuped files to /etc | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | # Prevent diskless | ||
| + | exec.prestart | ||
| + | || true"; | ||
| + | vnet.interface | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | exec.prestart | ||
| + | # fix bug that assing same MAC to all epairXb interface | ||
| + | # TO DO: convert this id into hexa | ||
| + | exec.prestart | ||
| + | exec.poststop | ||
| + | exec.poststop | ||
| + | vnet.interface | ||
| + | exec.poststop | ||
| + | exec.prestart | ||
| + | exec.poststop | ||
| + | exec.poststop | ||
| + | exec.poststop | ||
| + | } | ||
| + | </ | ||
| + | |||
| + | / | ||
| + | < | ||
| + | tmpfs / | ||
| + | tmpfs / | ||
| + | /root / | ||
| + | /bin / | ||
| + | /sbin / | ||
| + | /lib / | ||
| + | /libexec / | ||
| + | /usr / | ||
| + | /conf/base / | ||
| + | / | ||
| + | </ | ||
documentation/examples/multi-tenant_router_and_firewall.txt · Last modified: 2017/08/31 16:25 by 127.0.0.1
