Run FreeBSD on pysical/virtual servers by PXE booting mouting root over NFS4
Mount iSCSI zvol as /dev/da0 and create zpool for use by iocage
Great for testing fixes and network performance testing. This configuration prepares 5 FreeBSD enviroments that can be run on physical servers
Current Versions:
zfs create ship/pxe zfs set sync=always ship/pxe mkdir /mnt/ship/pxe/11 mkdir /mnt/ship/pxe/12 mkdir /mnt/ship/pxe/13 cp -r /root/FreeBSD/configs/pxe/ /mnt/ship/pxe zfs create ship/pxe/FreeBSD11 zfs create ship/pxe/FreeBSD12 zfs create ship/pxe/FreeBSD13 zfs create ship/pxe/FreeBSD12p zfs create ship/pxe/FreeBSD13p zfs create -s -V 128G -o refreservation=none ship/raw/FreeBSD11 zfs create -s -V 128G -o refreservation=none ship/raw/FreeBSD12 zfs create -s -V 128G -o refreservation=none ship/raw/FreeBSD13 zfs create -s -V 128G -o refreservation=none ship/raw/FreeBSD12p zfs create -s -V 128G -o refreservation=none ship/raw/FreeBSD13p edit /etc/exports edit /etc/ctl.conf pkg install tftp-hpa sysrc nfs_server_enable="YES" sysrc rpcbind_enable="YES" sysrc mountd_enable="YES" sysrc mountd_flags="-r" sysrc rpc_lockd_enable="YES" sysrc rpc_statd_enable="YES" sysrc ctld_enable="YES" sysrc tftpd_enable="YES" sysrc tftpd_flags="-s /mnt/ship/pxe" cd /mnt/ship/pxe/11 wget https://download.freebsd.org/ftp/releases/amd64/11.3-RELEASE/base.txz wget https://download.freebsd.org/ftp/releases/amd64/11.3-RELEASE/kernel.txz wget https://download.freebsd.org/ftp/releases/amd64/11.3-RELEASE/src.txz wget https://download.freebsd.org/ftp/releases/amd64/11.3-RELEASE/ports.txz tar Jxf base.txz -C /mnt/ship/pxe/FreeBSD11 tar Jxf kernel.txz -C /mnt/ship/pxe/FreeBSD11 tar Jxf src.txz -C /mnt/ship/pxe/FreeBSD11 #tar Jxf ports.txz -C /mnt/ship/pxe/FreeBSD11 cd /mnt/ship/pxe/12 wget https://download.freebsd.org/ftp/releases/amd64/12.1-RELEASE/base.txz wget https://download.freebsd.org/ftp/releases/amd64/12.1-RELEASE/kernel.txz wget https://download.freebsd.org/ftp/releases/amd64/12.1-RELEASE/src.txz wget https://download.freebsd.org/ftp/releases/amd64/12.1-RELEASE/ports.txz tar Jxf base.txz -C /mnt/ship/pxe/FreeBSD12 tar Jxf kernel.txz -C /mnt/ship/pxe/FreeBSD12 tar Jxf src.txz -C /mnt/ship/pxe/FreeBSD12 #tar Jxf ports.txz -C /mnt/ship/pxe/FreeBSD12 tar Jxf base.txz -C /mnt/ship/pxe/FreeBSD12p tar Jxf kernel.txz -C /mnt/ship/pxe/FreeBSD12p tar Jxf src.txz -C /mnt/ship/pxe/FreeBSD12p #tar Jxf ports.txz -C /mnt/ship/pxe/FreeBSD12p cd /mnt/ship/pxe/13 wget https://download.freebsd.org/ftp/snapshots/amd64/13.0-CURRENT/base.txz wget https://download.freebsd.org/ftp/snapshots/amd64/13.0-CURRENT/kernel.txz wget https://download.freebsd.org/ftp/snapshots/amd64/13.0-CURRENT/src.txz wget https://download.freebsd.org/ftp/snapshots/amd64/13.0-CURRENT/ports.txz tar Jxf base.txz -C /mnt/ship/pxe/FreeBSD13 tar Jxf kernel.txz -C /mnt/ship/pxe/FreeBSD13 tar Jxf src.txz -C /mnt/ship/pxe/FreeBSD13 #tar Jxf ports.txz -C /mnt/ship/pxe/FreeBSD13 tar Jxf base.txz -C /mnt/ship/pxe/FreeBSD13p tar Jxf kernel.txz -C /mnt/ship/pxe/FreeBSD13p tar Jxf src.txz -C /mnt/ship/pxe/FreeBSD13p #tar Jxf ports.txz -C /mnt/ship/pxe/FreeBSD13p cd /mnt/ship/pxe cp loader.conf FreeBSD11/boot cp loader.conf FreeBSD12/boot cp loader.conf FreeBSD13/boot cp loader.conf FreeBSD12p/boot cp loader.conf FreeBSD13p/boot cp sysctl.conf FreeBSD11/etc cp sysctl.conf FreeBSD12/etc cp sysctl.conf FreeBSD13/etc cp sysctl.conf FreeBSD12p/etc cp sysctl.conf FreeBSD13p/etc cp fstab.11 FreeBSD11/etc/fstab cp fstab.12 FreeBSD12/etc/fstab cp fstab.13 FreeBSD13/etc/fstab cp fstab.12p FreeBSD12p/etc/fstab cp fstab.13p FreeBSD13p/etc/fstab cp resolv.conf FreeBSD11/etc cp resolv.conf FreeBSD12/etc cp resolv.conf FreeBSD13/etc cp resolv.conf FreeBSD12p/etc cp resolv.conf FreeBSD13p/etc cp sshd_config FreeBSD11/etc/ssh cp sshd_config FreeBSD12/etc/ssh cp sshd_config FreeBSD13/etc/ssh cp sshd_config FreeBSD12p/etc/ssh cp sshd_config FreeBSD13p/etc/ssh cp rc.conf FreeBSD11/etc cp rc.conf FreeBSD12/etc cp rc.conf FreeBSD13/etc cp rc.conf FreeBSD12p/etc cp rc.conf FreeBSD13p/etc cp -r .ssh FreeBSD11/root cp -r .ssh FreeBSD12/root cp -r .ssh FreeBSD13/root cp -r .ssh FreeBSD12p/root cp -r .ssh FreeBSD13p/root cp iscsi.conf.11 FreeBSD11/etc/iscsi.conf cp iscsi.conf.12 FreeBSD12/etc/iscsi.conf cp iscsi.conf.13 FreeBSD13/etc/iscsi.conf cp iscsi.conf.12p FreeBSD12p/etc/iscsi.conf cp iscsi.conf.13p FreeBSD13p/etc/iscsi.conf cp iscsid FreeBSD11/etc/rc.d cp iscsid FreeBSD12/etc/rc.d cp iscsid FreeBSD13/etc/rc.d cp iscsid FreeBSD12p/etc/rc.d cp iscsid FreeBSD13p/etc/rc.d #currently I mount the root rw as its one directory per server and its easier to install packages. #If NFS mounting root ro cd /mnt/ship/pxe/FreeBSD11 mkdir -p conf/base/etc mkdir -p conf/base/var echo "16384"> conf/base/etc/md_size echo "32768"> conf/base/var/md_size tar -c -v -f conf/base/etc.cpio.gz --format cpio --gzip etc tar -c -v -f conf/base/var.cpio.gz --format cpio --gzip var cd /mnt/ship/pxe/FreeBSD12 mkdir -p conf/base/etc mkdir -p conf/base/var echo "16384"> conf/base/etc/md_size echo "32768"> conf/base/var/md_size tar -c -v -f conf/base/etc.cpio.gz --format cpio --gzip etc tar -c -v -f conf/base/var.cpio.gz --format cpio --gzip var cd /mnt/ship/pxe/FreeBSD13 mkdir -p conf/base/etc mkdir -p conf/base/var echo "16384"> conf/base/etc/md_size echo "32768"> conf/base/var/md_size tar -c -v -f conf/base/etc.cpio.gz --format cpio --gzip etc tar -c -v -f conf/base/var.cpio.gz --format cpio --gzip var cd /mnt/ship/pxe/FreeBSD12p mkdir -p conf/base/etc mkdir -p conf/base/var echo "16384"> conf/base/etc/md_size echo "32768"> conf/base/var/md_size tar -c -v -f conf/base/etc.cpio.gz --format cpio --gzip etc tar -c -v -f conf/base/var.cpio.gz --format cpio --gzip var cd /mnt/ship/pxe/FreeBSD13p mkdir -p conf/base/etc mkdir -p conf/base/var echo "16384"> conf/base/etc/md_size echo "32768"> conf/base/var/md_size tar -c -v -f conf/base/etc.cpio.gz --format cpio --gzip etc tar -c -v -f conf/base/var.cpio.gz --format cpio --gzip var #EndIf mounting NFS ro
pfsense dhcp configuration: Control which server boots which FreeBSD image using the DHCP options Common: Enable network booting Next Server: 192.168.10.10 Per Host: Netboot filename: FreeBSD12/boot/pxeboot Root Path: 192.168.10.10:/mnt/ship/pxe/FreeBSD12 Current mappings: lab17:FreeBSD12 / FreeBSD13p lab18:FreeBSD12p / FreeBSD13 zfs mount -a # to mount the ZFS datasets #zfs destroy ship/pxe/FreeBSD11 #zfs destroy ship/pxe/FreeBSD12 #zfs destroy ship/pxe/FreeBSD13 #zfs destroy ship/pxe/FreeBSD12p #zfs destroy ship/pxe/FreeBSD13p #iscsid includes the additional dependency on lockd to ensure NFS locking is in place before this service starts # After changing exports or recreating zfs datasets service mountd reload service ctld reload service tftpd restart Kernel space nfs does not work with jails. There is a userspace nfs alternative https://unfs3.github.io References: #https://www.freebsd.org/doc/en_US.ISO8859-1/books/handbook/network-diskless.html #https://www.freebsd.org/doc/en_US.ISO8859-1/books/handbook/network-nfs.html #https://www.freebsd.org/doc/en_US.ISO8859-1/articles/remote-install/preparation.html #https://www.freebsd.org/doc/handbook/network-iscsi.html #https://mfsbsd.vx.sk
On FreeBSD12(p) and FreeBSD13(p) #First time zpool create tank /dev/da0 pkg install -y nano bash bash-completion sudo wget htop py37-iocage bhyve-firmware grub2-bhyve iperf3 netperf iocage list #On FreeBSD13(p) iocage fetch -r 13.0-CURRENT -d ftp/snapshots/amd64/ After PXE boot: FreeBSD13(p) sometimes needs: service iscsid start FreeBSD12(p) and FreeBSD13(p) need: zfs mount -a iocage start ALL #manual steps: (not required if in rc.conf) #on both lab17 & lab18: ifconfig bridge1 create ifconfig bridge1 ether random addm bge0 up ifconfig bridge2 create ifconfig bridge2 ether random addm bge1 up ifconfig bridge11 create ifconfig bridge11 ether random addm bnxt0 up ifconfig bridge12 create #ifconfig bridge12 ether random addm bnxt1 up #ifconfig bridge12 deletem bnxt1 #on lab17: ifconfig bnxt0 10.0.11.17/24 up ifconfig bnxt1 10.0.12.17/24 up #on lab18: ifconfig bnxt0 10.0.11.18/24 up ifconfig bnxt1 10.0.12.18/24 up #iocage installation: mkdir -p /mnt/ship/pxe/root cp -r /root/FreeBSD /mnt/ship/pxe/root rpl "192.168.0" "192.168.10" /mnt/ship/pxe/root/FreeBSD/jails/create.sh rpl "fd01::" "fd0a::" /mnt/ship/pxe/root/FreeBSD/jails/create.sh cp -r /mnt/ship/pxe/root /mnt/ship/pxe/FreeBSD12 cp -r /mnt/ship/pxe/root /mnt/ship/pxe/FreeBSD13 cp -r /mnt/ship/pxe/root /mnt/ship/pxe/FreeBSD12p cp -r /mnt/ship/pxe/root /mnt/ship/pxe/FreeBSD13p rpl "12.2-RELEASE" "13.0-CURRENT" /mnt/ship/pxe/FreeBSD13/root/FreeBSD/jails/create.sh rpl "12.2-RELEASE" "13.0-CURRENT" /mnt/ship/pxe/FreeBSD13/root/FreeBSD/jails/update.sh rpl "12.2-RELEASE" "13.0-CURRENT" /mnt/ship/pxe/FreeBSD13p/root/FreeBSD/jails/create.sh rpl "12.2-RELEASE" "13.0-CURRENT" /mnt/ship/pxe/FreeBSD13p/root/FreeBSD/jails/update.sh rmdir -rf /mnt/ship/pxe/root #Create jail 117 on lab17 bash export JAIL=t-117 export JAILHOSTNAME=test-117 export JAILDOMAIN=diyit.org export JAILIP=117 export JAILUSER=X export JAILUSERID=1000 export JAILUSERVNC=false /root/FreeBSD/jails/create.sh $JAIL $JAILHOSTNAME $JAILDOMAIN $JAILIP $JAILUSER $JAILUSERID $JAILUSERVNC iocage exec $JAIL "pkg install -y iperf3 netperf" iocage set interfaces=vnet0:bridge1,vnet1:bridge11,vnet2:bridge12 ip4_addr="vnet0|192.168.10.$JAILIP/24,vnet1|10.0.11.$JAILIP/24,vnet2|10.0.12.$JAILIP/24" ip6_addr="vnet0|fd0a::$JAILIP/64,vnet1|2001:db8:11::$JAILIP/64,vnet2|2001:db8:12::$JAILIP/64" $JAIL iocage restart $JAIL #Create jail 127 on lab17 bash export JAIL=t-127 export JAILHOSTNAME=test-127 export JAILDOMAIN=diyit.org export JAILIP=127 export JAILUSER=X export JAILUSERID=1000 export JAILUSERVNC=false /root/FreeBSD/jails/create.sh $JAIL $JAILHOSTNAME $JAILDOMAIN $JAILIP $JAILUSER $JAILUSERID $JAILUSERVNC iocage exec $JAIL "pkg install -y iperf3 netperf" iocage set interfaces=vnet0:bridge1,vnet1:bridge11,vnet2:bridge12 ip4_addr="vnet0|192.168.10.$JAILIP/24,vnet1|10.0.11.$JAILIP/24,vnet2|10.0.12.$JAILIP/24" ip6_addr="vnet0|fd0a::$JAILIP/64,vnet1|2001:db8:11::$JAILIP/64,vnet2|2001:db8:12::$JAILIP/64" $JAIL iocage restart $JAIL #Create jail 118 on lab18 bash export JAIL=t-118 export JAILHOSTNAME=test-118 export JAILDOMAIN=diyit.org export JAILIP=118 export JAILUSER=X export JAILUSERID=1000 export JAILUSERVNC=false /root/FreeBSD/jails/create.sh $JAIL $JAILHOSTNAME $JAILDOMAIN $JAILIP $JAILUSER $JAILUSERID $JAILUSERVNC iocage exec $JAIL "pkg install -y iperf3 netperf" iocage set interfaces=vnet0:bridge1,vnet1:bridge11,vnet2:bridge12 ip4_addr="vnet0|192.168.10.$JAILIP/24,vnet1|10.0.11.$JAILIP/24,vnet2|10.0.12.$JAILIP/24" ip6_addr="vnet0|fd0a::$JAILIP/64,vnet1|2001:db8:11::$JAILIP/64,vnet2|2001:db8:12::$JAILIP/64" $JAIL iocage restart $JAIL #Create jail 128 on lab18 bash export JAIL=t-128 export JAILHOSTNAME=test-128 export JAILDOMAIN=diyit.org export JAILIP=128 export JAILUSER=X export JAILUSERID=1000 export JAILUSERVNC=false /root/FreeBSD/jails/create.sh $JAIL $JAILHOSTNAME $JAILDOMAIN $JAILIP $JAILUSER $JAILUSERID $JAILUSERVNC iocage exec $JAIL "pkg install -y iperf3 netperf" iocage set interfaces=vnet0:bridge1,vnet1:bridge11,vnet2:bridge12 ip4_addr="vnet0|192.168.10.$JAILIP/24,vnet1|10.0.11.$JAILIP/24,vnet2|10.0.12.$JAILIP/24" ip6_addr="vnet0|fd0a::$JAILIP/64,vnet1|2001:db8:11::$JAILIP/64,vnet2|2001:db8:12::$JAILIP/64" $JAIL iocage restart $JAIL #Notes: ipfw module is not loaded and configured on the test bed why ether random when configuring bridge: The if_bridge interface randomly chooses a link (MAC) address in the range reserved for locally administered addresses when it is created. This address is guaranteed to be unique only across all if_bridge inter- faces on the local machine. Thus you can theoretically have two bridges on the different machines with the same link addresses. ssh-keygen -R lab17.diyit.org; ssh -o "StrictHostKeyChecking no" root@lab17.diyit.org ssh-keygen -R lab18.diyit.org; ssh -o "StrictHostKeyChecking no" root@lab18.diyit.org THE TEST TRIGGER: https://people.freebsd.org/~kp/if_bridge/stable_12/
PerfTest.txt 13-current Vs 12.1-release - simple iperf test #on both lab17 & lab18: ifconfig bridge1 create ifconfig bridge1 addm bge0 up ifconfig bridge2 create ifconfig bridge2 addm bge1 up ifconfig bridge11 create ifconfig bridge11 addm bnxt0 up ifconfig bridge12 create #ifconfig bridge12 addm bnxt1 up #ifconfig bridge12 deletem bnxt1 #on lab17: ifconfig bnxt0 10.0.11.17/24 up ifconfig bnxt1 10.0.12.17/24 up #on lab18: ifconfig bnxt0 10.0.11.18/24 up ifconfig bnxt1 10.0.12.18/24 up lab17 & lab18 servers are connected over 25G SFP28 links via a switch. bnxt0 interfaces are in VLAN 11 and bnxt1 interfaces are in VLAN 12 root@lab17:~ # uname -a FreeBSD lab17 13.0-CURRENT FreeBSD 13.0-CURRENT #0 r361567: Thu May 28 04:59:08 UTC 2020 root@releng1.nyi.freebsd.org:/usr/obj/usr/src/amd64.amd64/sys/GENERIC amd64 root@lab17:~ # root@lab18:~ # uname -a FreeBSD lab18 13.0-CURRENT FreeBSD 13.0-CURRENT #0 r361567: Thu May 28 04:59:08 UTC 2020 root@releng1.nyi.freebsd.org:/usr/obj/usr/src/amd64.amd64/sys/GENERIC amd64 root@lab18:~ # Bridged Interfaces: server to server ----------------------------------------------------------- Accepted connection from 10.0.11.18, port 57363 [ 5] local 10.0.11.17 port 5201 connected to 10.0.11.18 port 20451 [ ID] Interval Transfer Bitrate [ 5] 0.00-1.00 sec 152 MBytes 1.28 Gbits/sec [ 5] 1.00-2.00 sec 258 MBytes 2.16 Gbits/sec [ 5] 2.00-3.00 sec 262 MBytes 2.20 Gbits/sec [ 5] 3.00-4.00 sec 270 MBytes 2.27 Gbits/sec [ 5] 4.00-5.00 sec 278 MBytes 2.33 Gbits/sec [ 5] 5.00-6.00 sec 281 MBytes 2.36 Gbits/sec [ 5] 6.00-7.00 sec 280 MBytes 2.35 Gbits/sec [ 5] 7.00-8.00 sec 276 MBytes 2.32 Gbits/sec [ 5] 8.00-9.00 sec 252 MBytes 2.12 Gbits/sec [ 5] 9.00-10.00 sec 265 MBytes 2.22 Gbits/sec [ 5] 10.00-10.00 sec 113 KBytes 2.56 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate [ 5] 0.00-10.00 sec 2.51 GBytes 2.16 Gbits/sec receiver ----------------------------------------------------------- Connecting to host 10.0.11.17, port 5201 [ 5] local 10.0.11.18 port 20451 connected to 10.0.11.17 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-1.00 sec 153 MBytes 1.28 Gbits/sec 7 218 KBytes [ 5] 1.00-2.00 sec 258 MBytes 2.16 Gbits/sec 15 299 KBytes [ 5] 2.00-3.00 sec 262 MBytes 2.20 Gbits/sec 12 242 KBytes [ 5] 3.00-4.00 sec 270 MBytes 2.27 Gbits/sec 10 229 KBytes [ 5] 4.00-5.00 sec 277 MBytes 2.33 Gbits/sec 13 252 KBytes [ 5] 5.00-6.00 sec 281 MBytes 2.36 Gbits/sec 15 286 KBytes [ 5] 6.00-7.00 sec 280 MBytes 2.35 Gbits/sec 12 321 KBytes [ 5] 7.00-8.00 sec 276 MBytes 2.32 Gbits/sec 11 336 KBytes [ 5] 8.00-9.00 sec 253 MBytes 2.12 Gbits/sec 135 348 KBytes [ 5] 9.00-10.00 sec 265 MBytes 2.22 Gbits/sec 13 308 KBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 2.52 GBytes 2.16 Gbits/sec 243 sender [ 5] 0.00-10.00 sec 2.51 GBytes 2.16 Gbits/sec receiver iperf Done. Direct Interfaces: server to server ----------------------------------------------------------- Accepted connection from 10.0.12.18, port 24137 [ 5] local 10.0.12.17 port 5201 connected to 10.0.12.18 port 19922 [ ID] Interval Transfer Bitrate [ 5] 0.00-1.00 sec 403 MBytes 3.38 Gbits/sec [ 5] 1.00-2.00 sec 757 MBytes 6.35 Gbits/sec [ 5] 2.00-3.00 sec 758 MBytes 6.35 Gbits/sec [ 5] 3.00-4.00 sec 760 MBytes 6.38 Gbits/sec [ 5] 4.00-5.00 sec 759 MBytes 6.37 Gbits/sec [ 5] 5.00-6.00 sec 760 MBytes 6.37 Gbits/sec [ 5] 6.00-7.00 sec 758 MBytes 6.36 Gbits/sec [ 5] 7.00-8.00 sec 759 MBytes 6.37 Gbits/sec [ 5] 8.00-9.00 sec 759 MBytes 6.37 Gbits/sec [ 5] 9.00-10.00 sec 759 MBytes 6.37 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate [ 5] 0.00-10.00 sec 7.06 GBytes 6.07 Gbits/sec receiver ----------------------------------------------------------- root@lab18:~ # iperf3 -c 10.0.12.17 Connecting to host 10.0.12.17, port 5201 [ 5] local 10.0.12.18 port 19922 connected to 10.0.12.17 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-1.00 sec 404 MBytes 3.39 Gbits/sec 14 268 KBytes [ 5] 1.00-2.00 sec 757 MBytes 6.35 Gbits/sec 2 354 KBytes [ 5] 2.00-3.00 sec 758 MBytes 6.35 Gbits/sec 6 259 KBytes [ 5] 3.00-4.00 sec 760 MBytes 6.38 Gbits/sec 4 349 KBytes [ 5] 4.00-5.00 sec 759 MBytes 6.37 Gbits/sec 6 254 KBytes [ 5] 5.00-6.00 sec 760 MBytes 6.37 Gbits/sec 4 344 KBytes [ 5] 6.00-7.00 sec 758 MBytes 6.36 Gbits/sec 6 247 KBytes [ 5] 7.00-8.00 sec 759 MBytes 6.37 Gbits/sec 4 338 KBytes [ 5] 8.00-9.00 sec 759 MBytes 6.37 Gbits/sec 15 269 KBytes [ 5] 9.00-10.00 sec 759 MBytes 6.37 Gbits/sec 4 356 KBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 7.06 GBytes 6.07 Gbits/sec 65 sender [ 5] 0.00-10.00 sec 7.06 GBytes 6.07 Gbits/sec receiver iperf Done. jail to jail on same server ----------------------------------------------------------- Accepted connection from 10.0.11.117, port 43039 [ 5] local 10.0.11.127 port 5201 connected to 10.0.11.117 port 55213 [ ID] Interval Transfer Bitrate [ 5] 0.00-1.00 sec 173 MBytes 1.45 Gbits/sec [ 5] 1.00-2.00 sec 182 MBytes 1.52 Gbits/sec [ 5] 2.00-3.00 sec 180 MBytes 1.51 Gbits/sec [ 5] 3.00-4.00 sec 185 MBytes 1.55 Gbits/sec [ 5] 4.00-5.00 sec 184 MBytes 1.54 Gbits/sec [ 5] 5.00-6.00 sec 181 MBytes 1.52 Gbits/sec [ 5] 6.00-7.00 sec 154 MBytes 1.29 Gbits/sec [ 5] 7.00-8.00 sec 133 MBytes 1.12 Gbits/sec [ 5] 8.00-9.00 sec 136 MBytes 1.14 Gbits/sec [ 5] 9.00-10.00 sec 137 MBytes 1.15 Gbits/sec [ 5] 10.00-10.00 sec 771 KBytes 1.70 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate [ 5] 0.00-10.00 sec 1.61 GBytes 1.38 Gbits/sec receiver ----------------------------------------------------------- test-117 ~ # iperf3 -c 10.0.11.127 Connecting to host 10.0.11.127, port 5201 [ 5] local 10.0.11.117 port 55213 connected to 10.0.11.127 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-1.00 sec 175 MBytes 1.46 Gbits/sec 0 1.61 MBytes [ 5] 1.00-2.00 sec 182 MBytes 1.52 Gbits/sec 0 1.61 MBytes [ 5] 2.00-3.00 sec 179 MBytes 1.50 Gbits/sec 0 1.61 MBytes [ 5] 3.00-4.00 sec 185 MBytes 1.55 Gbits/sec 0 1.61 MBytes [ 5] 4.00-5.00 sec 184 MBytes 1.55 Gbits/sec 0 1.61 MBytes [ 5] 5.00-6.00 sec 181 MBytes 1.52 Gbits/sec 0 1.61 MBytes [ 5] 6.00-7.00 sec 153 MBytes 1.29 Gbits/sec 0 1.61 MBytes [ 5] 7.00-8.00 sec 133 MBytes 1.12 Gbits/sec 0 1.61 MBytes [ 5] 8.00-9.00 sec 136 MBytes 1.14 Gbits/sec 0 1.61 MBytes [ 5] 9.00-10.00 sec 137 MBytes 1.15 Gbits/sec 0 1.61 MBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 1.61 GBytes 1.38 Gbits/sec 0 sender [ 5] 0.00-10.00 sec 1.61 GBytes 1.38 Gbits/sec receiver iperf Done. jail to jail on different servers ----------------------------------------------------------- Accepted connection from 10.0.12.128, port 56410 [ 5] local 10.0.12.127 port 5201 connected to 10.0.12.128 port 61265 [ ID] Interval Transfer Bitrate [ 5] 0.00-1.00 sec 126 MBytes 1.06 Gbits/sec [ 5] 1.00-2.00 sec 192 MBytes 1.61 Gbits/sec [ 5] 2.00-3.00 sec 173 MBytes 1.45 Gbits/sec [ 5] 3.00-4.00 sec 156 MBytes 1.31 Gbits/sec [ 5] 4.00-5.00 sec 170 MBytes 1.43 Gbits/sec [ 5] 5.00-6.00 sec 175 MBytes 1.46 Gbits/sec [ 5] 6.00-7.00 sec 173 MBytes 1.45 Gbits/sec [ 5] 7.00-8.00 sec 182 MBytes 1.53 Gbits/sec [ 5] 8.00-9.00 sec 195 MBytes 1.64 Gbits/sec [ 5] 9.00-10.00 sec 188 MBytes 1.58 Gbits/sec [ 5] 10.00-10.02 sec 2.47 MBytes 1.37 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate [ 5] 0.00-10.02 sec 1.69 GBytes 1.45 Gbits/sec receiver ----------------------------------------------------------- test-128 ~ # iperf3 -c 10.0.12.127 Connecting to host 10.0.12.127, port 5201 [ 5] local 10.0.12.128 port 61265 connected to 10.0.12.127 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-1.00 sec 129 MBytes 1.08 Gbits/sec 0 1.61 MBytes [ 5] 1.00-2.00 sec 192 MBytes 1.61 Gbits/sec 0 1.61 MBytes [ 5] 2.00-3.00 sec 173 MBytes 1.45 Gbits/sec 0 1.61 MBytes [ 5] 3.00-4.00 sec 155 MBytes 1.30 Gbits/sec 0 1.61 MBytes [ 5] 4.00-5.00 sec 171 MBytes 1.43 Gbits/sec 0 1.61 MBytes [ 5] 5.00-6.00 sec 174 MBytes 1.46 Gbits/sec 0 1.61 MBytes [ 5] 6.00-7.00 sec 173 MBytes 1.45 Gbits/sec 0 1.61 MBytes [ 5] 7.00-8.00 sec 183 MBytes 1.54 Gbits/sec 0 1.61 MBytes [ 5] 8.00-9.00 sec 195 MBytes 1.63 Gbits/sec 0 1.61 MBytes [ 5] 9.00-10.00 sec 188 MBytes 1.58 Gbits/sec 0 1.61 MBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 1.69 GBytes 1.45 Gbits/sec 0 sender [ 5] 0.00-10.02 sec 1.69 GBytes 1.45 Gbits/sec receiver iperf Done. root@lab17:~ # uname -a FreeBSD lab17 12.1-RELEASE FreeBSD 12.1-RELEASE r354233 GENERIC amd64 root@lab17:~ # root@lab18:~ # uname -a FreeBSD lab18 12.1-RELEASE FreeBSD 12.1-RELEASE r354233 GENERIC amd64 root@lab18:~ # Bridged Interfaces: server to server ----------------------------------------------------------- Accepted connection from 10.0.11.18, port 61273 [ 5] local 10.0.11.17 port 5201 connected to 10.0.11.18 port 17033 [ ID] Interval Transfer Bitrate [ 5] 0.00-1.00 sec 310 MBytes 2.60 Gbits/sec [ 5] 1.00-2.00 sec 551 MBytes 4.62 Gbits/sec [ 5] 2.00-3.00 sec 589 MBytes 4.94 Gbits/sec [ 5] 3.00-4.00 sec 591 MBytes 4.96 Gbits/sec [ 5] 4.00-5.00 sec 592 MBytes 4.97 Gbits/sec [ 5] 5.00-6.00 sec 594 MBytes 4.99 Gbits/sec [ 5] 6.00-7.00 sec 594 MBytes 4.98 Gbits/sec [ 5] 7.00-8.00 sec 593 MBytes 4.98 Gbits/sec [ 5] 8.00-9.00 sec 593 MBytes 4.97 Gbits/sec [ 5] 9.00-10.00 sec 593 MBytes 4.98 Gbits/sec [ 5] 10.00-10.00 sec 337 KBytes 4.93 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate [ 5] 0.00-10.00 sec 5.47 GBytes 4.70 Gbits/sec receiver ----------------------------------------------------------- root@lab18:~ # iperf3 -c 10.0.11.17 Connecting to host 10.0.11.17, port 5201 [ 5] local 10.0.11.18 port 17033 connected to 10.0.11.17 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-1.00 sec 312 MBytes 2.61 Gbits/sec 28 390 KBytes [ 5] 1.00-2.00 sec 551 MBytes 4.63 Gbits/sec 78 254 KBytes [ 5] 2.00-3.00 sec 589 MBytes 4.94 Gbits/sec 27 191 KBytes [ 5] 3.00-4.00 sec 591 MBytes 4.96 Gbits/sec 32 335 KBytes [ 5] 4.00-5.00 sec 592 MBytes 4.96 Gbits/sec 28 304 KBytes [ 5] 5.00-6.00 sec 595 MBytes 4.99 Gbits/sec 29 275 KBytes [ 5] 6.00-7.00 sec 593 MBytes 4.98 Gbits/sec 28 238 KBytes [ 5] 7.00-8.00 sec 593 MBytes 4.98 Gbits/sec 33 366 KBytes [ 5] 8.00-9.00 sec 593 MBytes 4.97 Gbits/sec 26 348 KBytes [ 5] 9.00-10.00 sec 593 MBytes 4.98 Gbits/sec 31 315 KBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 5.47 GBytes 4.70 Gbits/sec 340 sender [ 5] 0.00-10.00 sec 5.47 GBytes 4.70 Gbits/sec receiver iperf Done. Direct Interfaces: server to server ----------------------------------------------------------- Accepted connection from 10.0.12.18, port 24603 [ 5] local 10.0.12.17 port 5201 connected to 10.0.12.18 port 30690 [ ID] Interval Transfer Bitrate [ 5] 0.00-1.00 sec 997 MBytes 8.36 Gbits/sec [ 5] 1.00-2.00 sec 1.82 GBytes 15.6 Gbits/sec [ 5] 2.00-3.00 sec 1.83 GBytes 15.7 Gbits/sec [ 5] 3.00-4.00 sec 1.83 GBytes 15.7 Gbits/sec [ 5] 4.00-5.00 sec 1.83 GBytes 15.7 Gbits/sec [ 5] 5.00-6.00 sec 1.83 GBytes 15.7 Gbits/sec [ 5] 6.00-7.00 sec 1.83 GBytes 15.8 Gbits/sec [ 5] 7.00-8.00 sec 1.84 GBytes 15.8 Gbits/sec [ 5] 8.00-9.00 sec 1.83 GBytes 15.7 Gbits/sec [ 5] 9.00-10.00 sec 1.83 GBytes 15.7 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate [ 5] 0.00-10.00 sec 17.4 GBytes 15.0 Gbits/sec receiver ----------------------------------------------------------- root@lab18:~ # iperf3 -c 10.0.12.17 Connecting to host 10.0.12.17, port 5201 [ 5] local 10.0.12.18 port 30690 connected to 10.0.12.17 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-1.00 sec 998 MBytes 8.37 Gbits/sec 16 386 KBytes [ 5] 1.00-2.00 sec 1.82 GBytes 15.6 Gbits/sec 38 375 KBytes [ 5] 2.00-3.00 sec 1.83 GBytes 15.7 Gbits/sec 16 379 KBytes [ 5] 3.00-4.00 sec 1.83 GBytes 15.7 Gbits/sec 11 1.41 KBytes [ 5] 4.00-5.00 sec 1.83 GBytes 15.7 Gbits/sec 16 205 KBytes [ 5] 5.00-6.00 sec 1.83 GBytes 15.7 Gbits/sec 8 212 KBytes [ 5] 6.00-7.00 sec 1.83 GBytes 15.8 Gbits/sec 9 239 KBytes [ 5] 7.00-8.00 sec 1.84 GBytes 15.8 Gbits/sec 8 267 KBytes [ 5] 8.00-9.00 sec 1.83 GBytes 15.7 Gbits/sec 7 289 KBytes [ 5] 9.00-10.00 sec 1.83 GBytes 15.7 Gbits/sec 6 314 KBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 17.4 GBytes 15.0 Gbits/sec 135 sender [ 5] 0.00-10.00 sec 17.4 GBytes 15.0 Gbits/sec receiver iperf Done. jail to jail on same server ----------------------------------------------------------- Accepted connection from 10.0.11.117, port 31176 [ 5] local 10.0.11.127 port 5201 connected to 10.0.11.117 port 38861 [ ID] Interval Transfer Bitrate [ 5] 0.00-1.00 sec 203 MBytes 1.71 Gbits/sec [ 5] 1.00-2.00 sec 342 MBytes 2.87 Gbits/sec [ 5] 2.00-3.00 sec 343 MBytes 2.88 Gbits/sec [ 5] 3.00-4.00 sec 315 MBytes 2.64 Gbits/sec [ 5] 4.00-5.00 sec 224 MBytes 1.88 Gbits/sec [ 5] 5.00-6.00 sec 233 MBytes 1.96 Gbits/sec [ 5] 6.00-7.00 sec 241 MBytes 2.02 Gbits/sec [ 5] 7.00-8.00 sec 243 MBytes 2.04 Gbits/sec [ 5] 8.00-9.00 sec 243 MBytes 2.04 Gbits/sec [ 5] 9.00-10.00 sec 243 MBytes 2.04 Gbits/sec [ 5] 10.00-10.00 sec 851 KBytes 3.64 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate [ 5] 0.00-10.00 sec 2.57 GBytes 2.21 Gbits/sec receiver ----------------------------------------------------------- test-117 ~ # iperf3 -c 10.0.11.127 Connecting to host 10.0.11.127, port 5201 [ 5] local 10.0.11.117 port 38861 connected to 10.0.11.127 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-1.00 sec 204 MBytes 1.71 Gbits/sec 0 495 KBytes [ 5] 1.00-2.00 sec 342 MBytes 2.87 Gbits/sec 0 995 KBytes [ 5] 2.00-3.00 sec 343 MBytes 2.88 Gbits/sec 0 1.38 MBytes [ 5] 3.00-4.00 sec 315 MBytes 2.64 Gbits/sec 0 1.67 MBytes [ 5] 4.00-5.00 sec 224 MBytes 1.88 Gbits/sec 0 1.77 MBytes [ 5] 5.00-6.00 sec 233 MBytes 1.96 Gbits/sec 0 1.77 MBytes [ 5] 6.00-7.00 sec 241 MBytes 2.02 Gbits/sec 0 1.77 MBytes [ 5] 7.00-8.00 sec 243 MBytes 2.04 Gbits/sec 0 1.77 MBytes [ 5] 8.00-9.00 sec 243 MBytes 2.04 Gbits/sec 0 1.77 MBytes [ 5] 9.00-10.00 sec 243 MBytes 2.04 Gbits/sec 0 1.77 MBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 2.57 GBytes 2.21 Gbits/sec 0 sender [ 5] 0.00-10.00 sec 2.57 GBytes 2.21 Gbits/sec receiver iperf Done. jail to jail on different servers ----------------------------------------------------------- Accepted connection from 10.0.12.128, port 25861 [ 5] local 10.0.12.127 port 5201 connected to 10.0.12.128 port 18803 [ ID] Interval Transfer Bitrate [ 5] 0.00-1.00 sec 221 MBytes 1.85 Gbits/sec [ 5] 1.00-2.00 sec 340 MBytes 2.85 Gbits/sec [ 5] 2.00-3.00 sec 344 MBytes 2.89 Gbits/sec [ 5] 3.00-4.00 sec 392 MBytes 3.29 Gbits/sec [ 5] 4.00-5.00 sec 341 MBytes 2.86 Gbits/sec [ 5] 5.00-6.00 sec 262 MBytes 2.20 Gbits/sec [ 5] 6.00-7.00 sec 390 MBytes 3.27 Gbits/sec [ 5] 7.00-8.00 sec 392 MBytes 3.29 Gbits/sec [ 5] 8.00-9.00 sec 397 MBytes 3.33 Gbits/sec [ 5] 9.00-10.00 sec 398 MBytes 3.34 Gbits/sec [ 5] 10.00-10.00 sec 1014 KBytes 4.11 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate [ 5] 0.00-10.00 sec 3.40 GBytes 2.92 Gbits/sec receiver ----------------------------------------------------------- test-128 ~ # iperf3 -c 10.0.12.127 Connecting to host 10.0.12.127, port 5201 [ 5] local 10.0.12.128 port 18803 connected to 10.0.12.127 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-1.00 sec 221 MBytes 1.86 Gbits/sec 0 495 KBytes [ 5] 1.00-2.00 sec 340 MBytes 2.85 Gbits/sec 0 1007 KBytes [ 5] 2.00-3.00 sec 344 MBytes 2.89 Gbits/sec 0 1.47 MBytes [ 5] 3.00-4.00 sec 392 MBytes 3.29 Gbits/sec 0 1.77 MBytes [ 5] 4.00-5.00 sec 341 MBytes 2.86 Gbits/sec 0 1.77 MBytes [ 5] 5.00-6.00 sec 262 MBytes 2.20 Gbits/sec 0 1.77 MBytes [ 5] 6.00-7.00 sec 390 MBytes 3.27 Gbits/sec 0 1.77 MBytes [ 5] 7.00-8.00 sec 392 MBytes 3.29 Gbits/sec 0 1.77 MBytes [ 5] 8.00-9.00 sec 397 MBytes 3.33 Gbits/sec 0 1.77 MBytes [ 5] 9.00-10.00 sec 399 MBytes 3.34 Gbits/sec 0 1.77 MBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 3.40 GBytes 2.92 Gbits/sec 0 sender [ 5] 0.00-10.00 sec 3.40 GBytes 2.92 Gbits/sec receiver iperf Done.