1,601 (edited by Deus3X 2017-02-24 00:15:41)

Re: mwan3; multi-wan policy routing (general topic)

hi @Adze, hi openwrt members,
i was using mwan3 from 2 years with BB on an archer C7 v1
now i switched on dir-860L
i'm trying to replace the same configuration that i was using on BB but i having lots of problem.
mwan3 is working fine with ethernet interface but it does don't want to work with l2tp vpn client.
wan4 is the l2tp interface with metric 40.
The interface is online and is working if i uninstall mwan3.
mwan3 does not set ip rule and ip routing for that device and ofcorse is not able to ping and is marked ad offline.
the strange thing is that if i set the wan4 with metric 10 it will have ip rule and ip routing setting but the other interface will be marked as offline.

i have sent your test commands and this is the result:

root@DIR-860L:~# ACTION=ifup INTERFACE=wan4 /etc/hotplug.d/iface/15-mwan3
root@DIR-860L:~# echo $?
3

this is my configuration:

Software versions : 

OpenWrt - LEDE Reboot 17.01.0 r3205-59508e3
LuCI - git-17.051.53299-a100738

mwan3 - 2.0-3
mwan3-luci - 1.4-5

Output of "cat /etc/config/mwan3" : 

config interface 'wan'
    option enabled '1'
    list track_ip '8.8.4.4'
    list track_ip '8.8.8.8'
    list track_ip '208.67.222.222'
    list track_ip '208.67.220.220'
    option count '1'
    option timeout '2'
    option interval '5'
    option down '3'
    option up '8'
    option reliability '1'

config interface 'wan2'
    option reliability '1'
    option count '1'
    option timeout '2'
    option interval '5'
    option down '3'
    option up '8'
    option enabled '1'
    list track_ip '8.8.4.4'
    list track_ip '8.8.8.8'
    list track_ip '208.67.222.222'
    list track_ip '208.67.220.220'

config interface 'wan3'
    option enabled '1'
    option reliability '1'
    option count '1'
    option timeout '2'
    option interval '5'
    option down '3'
    option up '8'
    list track_ip '8.8.4.4'
    list track_ip '8.8.8.8'
    list track_ip '208.67.222.222'
    list track_ip '208.67.220.220'

config interface 'wan4'
    option enabled '1'
    option reliability '1'
    option count '1'
    option timeout '2'
    option interval '5'
    option down '3'
    option up '8'
    list track_ip '8.8.4.4'
    list track_ip '8.8.8.8'
    list track_ip '208.67.222.222'
    list track_ip '208.67.220.220'

config interface 'wan5'
    option enabled '1'
    list track_ip '8.8.4.4'
    list track_ip '8.8.8.8'
    list track_ip '208.67.222.222'
    list track_ip '208.67.220.220'
    option reliability '1'
    option count '1'
    option timeout '2'
    option interval '5'
    option down '3'
    option up '8'

config interface 'wan6'
    option enabled '1'
    list track_ip '8.8.4.4'
    list track_ip '8.8.8.8'
    list track_ip '208.67.222.222'
    list track_ip '208.67.220.220'
    option reliability '1'
    option count '1'
    option timeout '2'
    option interval '5'
    option down '3'
    option up '8'

config member 'wan_m1_w6'
    option interface 'wan'
    option metric '1'
    option weight '6'

config member 'wan_m2_w6'
    option interface 'wan'
    option metric '2'
    option weight '6'

config member 'wan_m3_w6'
    option interface 'wan'
    option metric '3'
    option weight '6'

config member 'wan_m4_w6'
    option interface 'wan'
    option metric '4'
    option weight '6'

config member 'wan_m5_w6'
    option interface 'wan'
    option metric '5'
    option weight '6'

config member 'wan_m6_w6'
    option interface 'wan'
    option metric '6'
    option weight '6'

config member 'wan2_m1_w5'
    option interface 'wan2'
    option metric '1'
    option weight '5'

config member 'wan2_m2_w5'
    option interface 'wan2'
    option metric '2'
    option weight '5'

config member 'wan2_m3_w5'
    option interface 'wan2'
    option metric '3'
    option weight '5'

config member 'wan2_m4_w5'
    option interface 'wan2'
    option metric '4'
    option weight '5'

config member 'wan2_m5_w5'
    option interface 'wan2'
    option metric '5'
    option weight '5'

config member 'wan2_m6_w5'
    option interface 'wan2'
    option metric '6'
    option weight '5'

config member 'wan3_m1_w4'
    option interface 'wan3'
    option metric '1'
    option weight '4'

config member 'wan3_m2_w4'
    option interface 'wan3'
    option metric '2'
    option weight '4'

config member 'wan3_m3_w4'
    option interface 'wan3'
    option metric '3'
    option weight '4'

config member 'wan3_m4_w4'
    option interface 'wan3'
    option metric '4'
    option weight '4'

config member 'wan3_m5_w4'
    option interface 'wan3'
    option metric '5'
    option weight '4'

config member 'wan3_m6_w4'
    option interface 'wan3'
    option metric '6'
    option weight '4'

config member 'wan4_m1_w3'
    option interface 'wan4'
    option metric '1'
    option weight '3'

config member 'wan4_m2_w3'
    option interface 'wan4'
    option metric '2'
    option weight '3'

config member 'wan4_m3_w3'
    option interface 'wan4'
    option metric '3'
    option weight '3'

config member 'wan4_m4_w3'
    option interface 'wan4'
    option metric '4'
    option weight '3'

config member 'wan4_m5_w3'
    option interface 'wan4'
    option metric '5'
    option weight '3'

config member 'wan4_m6_w3'
    option interface 'wan4'
    option metric '6'
    option weight '3'

config member 'wan5_m1_w2'
    option interface 'wan5'
    option metric '1'
    option weight '2'

config member 'wan5_m2_w2'
    option interface 'wan5'
    option metric '2'
    option weight '2'

config member 'wan5_m3_w2'
    option interface 'wan5'
    option metric '3'
    option weight '2'

config member 'wan5_m4_w2'
    option interface 'wan5'
    option metric '4'
    option weight '2'

config member 'wan5_m5_w2'
    option interface 'wan5'
    option metric '5'
    option weight '2'

config member 'wan5_m6_w2'
    option interface 'wan5'
    option metric '6'
    option weight '2'

config member 'wan6_m1_w1'
    option interface 'wan6'
    option metric '1'
    option weight '1'

config member 'wan6_m2_w1'
    option interface 'wan6'
    option metric '2'
    option weight '1'

config member 'wan6_m3_w1'
    option interface 'wan6'
    option metric '3'
    option weight '1'

config member 'wan6_m4_w1'
    option interface 'wan6'
    option metric '4'
    option weight '1'

config member 'wan6_m5_w1'
    option interface 'wan6'
    option metric '5'
    option weight '1'

config member 'wan6_m6_w1'
    option interface 'wan6'
    option metric '6'
    option weight '1'

config policy 'wan_only'
    list use_member 'wan_m1_w6'

config policy 'wan2_only'
    list use_member 'wan2_m1_w5'

config policy 'wan3_only'
    list use_member 'wan3_m1_w4'

config policy 'wan4_only'
    list use_member 'wan4_m1_w3'

config policy 'wan5_only'
    list use_member 'wan5_m1_w2'

config policy 'wan6_only'
    list use_member 'wan6_m1_w1'

config policy 'balanced'
    list use_member 'wan_m1_w6'
    list use_member 'wan2_m1_w5'
    list use_member 'wan3_m1_w4'

config policy 'wn_wn2_wn3'
    list use_member 'wan_m1_w6'
    list use_member 'wan2_m2_w5'
    list use_member 'wan3_m3_w4'

config policy 'wn3_wn2_wn'
    list use_member 'wan3_m1_w4'
    list use_member 'wan2_m2_w5'
    list use_member 'wan_m3_w6'

config policy 'wn_wn2'
    list use_member 'wan_m1_w6'
    list use_member 'wan2_m2_w5'

config policy 'wn_wn3'
    list use_member 'wan_m1_w6'
    list use_member 'wan3_m2_w4'

config policy 'wn_wn4'
    list use_member 'wan_m1_w6'
    list use_member 'wan4_m2_w3'

config policy 'wn_wn5'
    list use_member 'wan_m1_w6'
    list use_member 'wan5_m2_w2'

config policy 'wn_wn6'
        list use_member 'wan_m1_w6'
        list use_member 'wan6_m2_w1'

config policy 'wn2_wn'
    list use_member 'wan2_m1_w5'
    list use_member 'wan_m2_w6'

config policy 'wn3_wn'
    list use_member 'wan3_m1_w4'
    list use_member 'wan_m2_w6'

config policy 'wn4_wn'
    list use_member 'wan4_m1_w3'
    list use_member 'wan_m2_w6'

config policy 'wn4_wn2'
    list use_member 'wan4_m1_w3'
    list use_member 'wan2_m2_w5'

config policy 'wn4_wn3'
    list use_member 'wan4_m1_w3'
    list use_member 'wan3_m2_w4'

config policy 'wn5_wn'
    list use_member 'wan5_m1_w2'
    list use_member 'wan_m2_w6'

config policy 'wn5_wn2'
    list use_member 'wan5_m1_w2'
    list use_member 'wan2_m2_w5'

config policy 'wn5_wn3'
    list use_member 'wan5_m1_w2'
    list use_member 'wan3_m2_w4'

config policy 'wn6_wn'
        list use_member 'wan6_m1_w1'
        list use_member 'wan_m2_w6'

config policy 'wn6_wn2'
        list use_member 'wan6_m1_w1'
        list use_member 'wan2_m2_w5'

config policy 'wn6_wn3'
        list use_member 'wan6_m1_w1'
        list use_member 'wan3_m2_w4'

config policy 'wn4_wn_wn2_wn3'
    list use_member 'wan4_m1_w3'
    list use_member 'wan_m2_w6'
    list use_member 'wan2_m3_w5'
    list use_member 'wan3_m4_w4'

config policy 'wn5_wn_wn2_wn3'
    list use_member 'wan5_m1_w2'
    list use_member 'wan_m2_w6'
    list use_member 'wan2_m3_w5'
    list use_member 'wan3_m4_w4'

config policy 'wn6_wn_wn2_wn3'
        list use_member 'wan6_m1_w1'
        list use_member 'wan_m2_w6'
        list use_member 'wan2_m3_w5'
        list use_member 'wan3_m4_w4'

config rule 'Vodafone'
    option dest_ip '192.168.4.1'
    option proto 'all'
    option use_policy 'wan2_only'

config rule 'DNS_WAN1'
    option proto 'all'
    option use_policy 'wan_only'
    option dest_ip 'checkip.dyndns.com'

config rule 'DNS_WAN2'
    option proto 'all'
    option use_policy 'wan2_only'
    option dest_ip 'ipv4.ident.me'

config rule 'DNS_WAN3'
    option proto 'all'
    option use_policy 'wan3_only'
    option dest_ip 'ipv4.icanhazip.com'

config rule 'SKY_Francesco'
    option src_ip '192.168.2.24'
    option proto 'all'
    option use_policy 'wan_only'

config rule 'SKY_Soggiorno'
    option src_ip '192.168.2.25'
    option proto 'all'
    option use_policy 'wan_only'

config rule 'TV_Francesco'
    option src_ip '192.168.2.22'
    option proto 'all'
    option use_policy 'wan4_only'

config rule 'TV_Emanuele'
    option src_ip '192.168.2.23'
    option proto 'all'
    option use_policy 'wan4_only'

config rule 'NAS'
    option src_ip '192.168.2.44'
    option proto 'all'
    option use_policy 'wan4_only'

config rule 'NAS_2'
    option dest_ip '192.168.2.44'
    option proto 'all'
    option use_policy 'wan4_only'

config rule 'Vodafone_3G'
    option dest_ip 'contatori.vodafone.it'
    option proto 'all'
    option use_policy 'wan3_only'

config rule 'Francesco'
    option src_ip '192.168.2.2'
    option proto 'all'
    option use_policy 'wn_wn2_wn3'

config rule 'Francesco_2'
    option src_ip '192.168.2.4'
    option proto 'all'
    option use_policy 'wn_wn2_wn3'

config rule 'default_rule'
    option dest_ip '0.0.0.0/0'
    option proto 'all'
    option use_policy 'wn4_wn_wn2_wn3'

Output of "cat /etc/config/network" : 

config interface 'loopback'
    option ifname 'lo'
    option proto 'static'
    option ipaddr '127.0.0.1'
    option netmask '255.0.0.0'

config globals 'globals'
    option ula_prefix 'fdfd:ad9b:27bf::/48'

config switch
    option name 'switch0'
    option reset '1'
    option enable_vlan '1'

config switch_vlan
    option device 'switch0'
    option vlan '1'
    option vid '1'
    option ports '2 3 4 6t'

config switch_vlan
    option device 'switch0'
    option vlan '2'
    option ports '0 6t'
    option vid '2'

config switch_vlan
    option device 'switch0'
    option vlan '3'
    option vid '3'
    option ports '1 6t'

config interface 'lan'
    option type 'bridge'
    option ifname 'eth0.1'
    option proto 'static'
    option netmask '255.255.255.0'
    option delegate '0'
    option ipaddr '192.168.2.1'
    option dns '8.8.8.8 8.8.4.4'

config device 'lan_dev'
    option name 'eth0.1'
    option macaddr 'e4:6f:13:2e:a5:10'

config interface 'wan'
    option ifname 'eth0.2'
    option proto 'dhcp'
    option delegate '0'
    option peerdns '0'
    option dns '8.8.8.8 8.8.4.4'
    option metric '10'

config device 'wan_dev'
    option name 'eth0.2'
    option macaddr 'e4:6f:13:2e:a5:13'

config interface 'wan2'
    option proto 'static'
    option ifname 'eth0.3'
    option delegate '0'
    option ipaddr '192.168.5.2'
    option netmask '255.255.255.0'
    option gateway '192.168.5.1'
    option dns '8.8.8.8 8.8.4.4'
    option metric '20'

config interface 'wan3'
    option proto '3g'
    option delegate '0'
    option device '/dev/ttyUSB0'
    option service 'umts_only'
    option app 'APN'
    option pincode '5204'
    option dialnumber '*99#'
    option ipv6 '0'
    option metric '30'
    option peerdns '0'
    option dns '8.8.8.8 8.8.4.4'

config interface 'wan4'
    option proto 'l2tp'
    option ifname 'eth0'
    option delegate '0'
    option server 'SERVER_IP'
    USERNAME HIDDEN
    PASSWORD HIDDEN
    option metric '40'
    option peerdns '0'
    option dns '8.8.8.8 8.8.4.4'
    option ipv6 '0'

config interface 'vpn'
    option proto 'none'
    option ifname 'tun0'
    option auto '1'
    option delegate '0'

config interface 'guest'
    option type 'bridge'
    option proto 'static'
    option delegate '0'
    option ipaddr '10.0.0.1'
    option netmask '255.255.255.0'
    option dns '8.8.8.8 8.8.4.4'

Output of "cat /etc/config/wireless" : 

config wifi-device 'radio0'
    option type 'mac80211'
    option hwmode '11a'
    option path 'pci0000:00/0000:00:00.0/0000:01:00.0'
    option htmode 'VHT80'
    option country '00'
    option txpower '20'
    option channel '40'

config wifi-iface 'default_radio0'
    option device 'radio0'
    option network 'lan'
    option mode 'ap'
    option encryption 'psk2'
    KEY HIDDEN
    option ssid 'PARADISE 5GHz'

config wifi-device 'radio1'
    option type 'mac80211'
    option hwmode '11g'
    option path 'pci0000:00/0000:00:01.0/0000:02:00.0'
    option htmode 'HT20'
    option channel '5'
    option txpower '20'
    option country '00'

config wifi-iface 'default_radio1'
    option device 'radio1'
    option network 'lan'
    option mode 'ap'
    option ssid 'PARADISE'
    option encryption 'psk2'
    KEY HIDDEN

config wifi-iface
    option device 'radio0'
    option mode 'ap'
    option ssid 'HEAVEN 5GHz'
    option encryption 'psk2'
    KEY HIDDEN
    option network 'guest'

config wifi-iface
    option device 'radio1'
    option mode 'ap'
    option ssid 'HEAVEN'
    option encryption 'psk2'
    KEY HIDDEN
    option network 'guest'

Output of "ifconfig" : 

3g-wan3   Link encap:Point-to-Point Protocol  
          inet addr:5.92.1.146  P-t-P:10.64.64.64  Mask:255.255.255.255
          UP POINTOPOINT RUNNING NOARP MULTICAST  MTU:1500  Metric:1
          RX packets:1305 errors:2060 dropped:0 overruns:0 frame:0
          TX packets:1319 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:3 
          RX bytes:109737 (107.1 KiB)  TX bytes:109521 (106.9 KiB)

br-guest  Link encap:Ethernet  HWaddr E6:6F:13:2E:A5:10  
          inet addr:10.0.0.1  Bcast:10.0.0.255  Mask:255.255.255.0
          inet6 addr: fe80::e46f:13ff:fe2e:a510/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:0 (0.0 B)  TX bytes:1080 (1.0 KiB)

br-lan    Link encap:Ethernet  HWaddr E4:6F:13:2E:A5:10  
          inet addr:192.168.2.1  Bcast:192.168.2.255  Mask:255.255.255.0
          inet6 addr: fe80::e66f:13ff:fe2e:a510/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:844640 errors:0 dropped:0 overruns:0 frame:0
          TX packets:633598 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:853271271 (813.7 MiB)  TX bytes:458891085 (437.6 MiB)

eth0      Link encap:Ethernet  HWaddr BA:BB:F0:29:F9:D3  
          inet6 addr: fe80::b8bb:f0ff:fe29:f9d3/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:1485563 errors:0 dropped:0 overruns:0 frame:0
          TX packets:1473084 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:1330817566 (1.2 GiB)  TX bytes:1334221310 (1.2 GiB)
          Interrupt:10 

eth0.1    Link encap:Ethernet  HWaddr E4:6F:13:2E:A5:10  
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:844677 errors:0 dropped:0 overruns:0 frame:0
          TX packets:633599 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:853290865 (813.7 MiB)  TX bytes:458891105 (437.6 MiB)

eth0.2    Link encap:Ethernet  HWaddr E4:6F:13:2E:A5:13  
          inet addr:192.168.3.1  Bcast:192.168.3.255  Mask:255.255.255.0
          inet6 addr: fe80::e66f:13ff:fe2e:a513/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:23565 errors:0 dropped:0 overruns:0 frame:0
          TX packets:19985 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:21854352 (20.8 MiB)  TX bytes:5926022 (5.6 MiB)

eth0.3    Link encap:Ethernet  HWaddr BA:BB:F0:29:F9:D3  
          inet addr:192.168.5.2  Bcast:192.168.5.255  Mask:255.255.255.0
          inet6 addr: fe80::b8bb:f0ff:fe29:f9d3/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:617000 errors:0 dropped:0 overruns:0 frame:0
          TX packets:818781 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:428914826 (409.0 MiB)  TX bytes:857536986 (817.8 MiB)

ifb0      Link encap:Ethernet  HWaddr B2:58:B3:36:89:04  
          inet6 addr: fe80::b058:b3ff:fe36:8904/64 Scope:Link
          UP BROADCAST RUNNING NOARP  MTU:1500  Metric:1
          RX packets:23561 errors:0 dropped:0 overruns:0 frame:0
          TX packets:23561 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:32 
          RX bytes:22182430 (21.1 MiB)  TX bytes:22182430 (21.1 MiB)

lo        Link encap:Local Loopback  
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:511 errors:0 dropped:0 overruns:0 frame:0
          TX packets:511 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1 
          RX bytes:42069 (41.0 KiB)  TX bytes:42069 (41.0 KiB)

tun0      Link encap:UNSPEC  HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00  
          inet addr:192.168.100.1  P-t-P:192.168.100.1  Mask:255.255.255.0
          UP POINTOPOINT RUNNING NOARP MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:100 
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

wlan0     Link encap:Ethernet  HWaddr E4:6F:13:2E:A5:12  
          inet6 addr: fe80::e66f:13ff:fe2e:a512/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:7652 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:0 (0.0 B)  TX bytes:1959250 (1.8 MiB)

wlan0-1   Link encap:Ethernet  HWaddr E6:6F:13:2E:A5:12  
          inet6 addr: fe80::e46f:13ff:fe2e:a512/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:13 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:0 (0.0 B)  TX bytes:1824 (1.7 KiB)

wlan1     Link encap:Ethernet  HWaddr E4:6F:13:2E:A5:10  
          inet6 addr: fe80::e66f:13ff:fe2e:a510/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:4 errors:0 dropped:0 overruns:0 frame:0
          TX packets:7670 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:474 (474.0 B)  TX bytes:1961505 (1.8 MiB)

wlan1-1   Link encap:Ethernet  HWaddr E6:6F:13:2E:A5:10  
          inet6 addr: fe80::e46f:13ff:fe2e:a510/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:12 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:0 (0.0 B)  TX bytes:1736 (1.6 KiB)

Output of "route -n" : 

Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.3.254   0.0.0.0         UG    10     0        0 eth0.2
0.0.0.0         192.168.5.1     0.0.0.0         UG    20     0        0 eth0.3
0.0.0.0         10.64.64.64     0.0.0.0         UG    30     0        0 3g-wan3
10.0.0.0        0.0.0.0         255.255.255.0   U     0      0        0 br-guest
10.64.64.64     0.0.0.0         255.255.255.255 UH    0      0        0 3g-wan3
94.242.246.38   192.168.3.254   255.255.255.255 UGH   10     0        0 eth0.2
94.242.246.45   192.168.3.254   255.255.255.255 UGH   10     0        0 eth0.2
94.242.246.46   192.168.3.254   255.255.255.255 UGH   10     0        0 eth0.2
94.242.246.47   192.168.3.254   255.255.255.255 UGH   10     0        0 eth0.2
192.168.2.0     0.0.0.0         255.255.255.0   U     0      0        0 br-lan
192.168.3.0     0.0.0.0         255.255.255.0   U     10     0        0 eth0.2
192.168.3.254   0.0.0.0         255.255.255.255 UH    10     0        0 eth0.2
192.168.5.0     0.0.0.0         255.255.255.0   U     20     0        0 eth0.3
192.168.15.0    192.168.100.2   255.255.255.0   UG    0      0        0 tun0
192.168.100.0   0.0.0.0         255.255.255.0   U     0      0        0 tun0

Output of "ip rule show" : 

0:    from all lookup local 
1001:    from all iif eth0.2 lookup main 
1002:    from all iif eth0.3 lookup main 
1003:    from all iif 3g-wan3 lookup main 
2001:    from all fwmark 0x100/0xff00 lookup 1 
2002:    from all fwmark 0x200/0xff00 lookup 2 
2003:    from all fwmark 0x300/0xff00 lookup 3 
2253:    from all fwmark 0xfd00/0xff00 blackhole
2254:    from all fwmark 0xfe00/0xff00 unreachable
32766:    from all lookup main 
32767:    from all lookup default

Output of "ip route list table 1-250" : 

1
default via 192.168.3.254 dev eth0.2 
2
default via 192.168.5.1 dev eth0.3 
3
default via 10.64.64.64 dev 3g-wan3

Firewall default output policy (must be ACCEPT) : 

ACCEPT

Output of "iptables -L -t mangle -v -n" : 

Chain PREROUTING (policy ACCEPT 1468K packets, 1294M bytes)
 pkts bytes target     prot opt in     out     source               destination         
1480K 1304M mwan3_hook  all  --  *      *       0.0.0.0/0            0.0.0.0/0           

Chain INPUT (policy ACCEPT 11208 packets, 1063K bytes)
 pkts bytes target     prot opt in     out     source               destination         

Chain FORWARD (policy ACCEPT 1449K packets, 1290M bytes)
 pkts bytes target     prot opt in     out     source               destination         
15317 5211K qos_Default  all  --  *      eth0.2  0.0.0.0/0            0.0.0.0/0           
  390 22828 TCPMSS     tcp  --  *      eth0.2  0.0.0.0/0            0.0.0.0/0            tcp flags:0x06/0x02 /* !fw3: wan (mtu_fix) */ TCPMSS clamp to PMTU
   35  2112 TCPMSS     tcp  --  *      eth0.3  0.0.0.0/0            0.0.0.0/0            tcp flags:0x06/0x02 /* !fw3: wan (mtu_fix) */ TCPMSS clamp to PMTU
    0     0 TCPMSS     tcp  --  *      3g-wan3  0.0.0.0/0            0.0.0.0/0            tcp flags:0x06/0x02 /* !fw3: wan (mtu_fix) */ TCPMSS clamp to PMTU
    0     0 TCPMSS     tcp  --  *      br-guest  0.0.0.0/0            0.0.0.0/0            tcp flags:0x06/0x02 /* !fw3: guest (mtu_fix) */ TCPMSS clamp to PMTU

Chain OUTPUT (policy ACCEPT 10844 packets, 2057K bytes)
 pkts bytes target     prot opt in     out     source               destination         
 4663  441K qos_Default  all  --  *      eth0.2  0.0.0.0/0            0.0.0.0/0           
11898 2168K mwan3_hook  all  --  *      *       0.0.0.0/0            0.0.0.0/0           

Chain POSTROUTING (policy ACCEPT 1460K packets, 1292M bytes)
 pkts bytes target     prot opt in     out     source               destination         

Chain mwan3_connected (2 references)
 pkts bytes target     prot opt in     out     source               destination         
31489   24M MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            match-set mwan3_connected dst MARK or 0xff00

Chain mwan3_hook (2 references)
 pkts bytes target     prot opt in     out     source               destination         
1492K 1306M CONNMARK   all  --  *      *       0.0.0.0/0            0.0.0.0/0            CONNMARK restore mask 0xff00
15096 3601K mwan3_ifaces_in  all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00
11489 2132K mwan3_connected  all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00
 6797  530K mwan3_ifaces_out  all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00
 1538  101K mwan3_rules  all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00
1492K 1306M CONNMARK   all  --  *      *       0.0.0.0/0            0.0.0.0/0            CONNMARK save mask 0xff00
50885   28M mwan3_connected  all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match ! 0xff00/0xff00

Chain mwan3_iface_in_wan (1 references)
 pkts bytes target     prot opt in     out     source               destination         
    1    36 MARK       all  --  eth0.2 *       0.0.0.0/0            0.0.0.0/0            match-set mwan3_connected src mark match 0x0/0xff00 /* default */ MARK or 0xff00
  138 53629 MARK       all  --  eth0.2 *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan */ MARK xset 0x100/0xff00

Chain mwan3_iface_in_wan2 (1 references)
 pkts bytes target     prot opt in     out     source               destination         
 3429 1411K MARK       all  --  eth0.3 *       0.0.0.0/0            0.0.0.0/0            match-set mwan3_connected src mark match 0x0/0xff00 /* default */ MARK or 0xff00
   39  3065 MARK       all  --  eth0.3 *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan2 */ MARK xset 0x200/0xff00

Chain mwan3_iface_in_wan3 (1 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  3g-wan3 *       0.0.0.0/0            0.0.0.0/0            match-set mwan3_connected src mark match 0x0/0xff00 /* default */ MARK or 0xff00
    0     0 MARK       all  --  3g-wan3 *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan3 */ MARK xset 0x300/0xff00

Chain mwan3_iface_out_wan (1 references)
 pkts bytes target     prot opt in     out     source               destination         
 2494  197K MARK       all  --  *      *       192.168.3.1          0.0.0.0/0            mark match 0x0/0xff00 /* wan */ MARK xset 0x100/0xff00

Chain mwan3_iface_out_wan2 (1 references)
 pkts bytes target     prot opt in     out     source               destination         
 1478  124K MARK       all  --  *      *       192.168.5.2          0.0.0.0/0            mark match 0x0/0xff00 /* wan2 */ MARK xset 0x200/0xff00

Chain mwan3_iface_out_wan3 (1 references)
 pkts bytes target     prot opt in     out     source               destination         
 1274  107K MARK       all  --  *      *       5.92.1.146           0.0.0.0/0            mark match 0x0/0xff00 /* wan3 */ MARK xset 0x300/0xff00

Chain mwan3_ifaces_in (1 references)
 pkts bytes target     prot opt in     out     source               destination         
14970 3580K mwan3_iface_in_wan  all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00
14695 3509K mwan3_iface_in_wan3  all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00
13990 3369K mwan3_iface_in_wan2  all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00

Chain mwan3_ifaces_out (1 references)
 pkts bytes target     prot opt in     out     source               destination         
 6741  524K mwan3_iface_out_wan  all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00
 4232  326K mwan3_iface_out_wan3  all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00
 2830  206K mwan3_iface_out_wan2  all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00

Chain mwan3_policy_balanced (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 statistic mode random probability 0.26599999983 /* wan3 4 15 */ MARK xset 0x300/0xff00
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 statistic mode random probability 0.45399999991 /* wan2 5 11 */ MARK xset 0x200/0xff00
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan 6 6 */ MARK xset 0x100/0xff00

Chain mwan3_policy_wan2_only (2 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan2 5 5 */ MARK xset 0x200/0xff00

Chain mwan3_policy_wan3_only (5 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan3 4 4 */ MARK xset 0x300/0xff00

Chain mwan3_policy_wan4_only (4 references)
 pkts bytes target     prot opt in     out     source               destination         
  664 39840 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* unreachable */ MARK xset 0xfe00/0xff00

Chain mwan3_policy_wan5_only (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* unreachable */ MARK xset 0xfe00/0xff00

Chain mwan3_policy_wan6_only (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* unreachable */ MARK xset 0xfe00/0xff00

Chain mwan3_policy_wan_only (5 references)
 pkts bytes target     prot opt in     out     source               destination         
   15   900 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan 6 6 */ MARK xset 0x100/0xff00

Chain mwan3_policy_wn2_wn (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan2 5 5 */ MARK xset 0x200/0xff00

Chain mwan3_policy_wn3_wn (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan3 4 4 */ MARK xset 0x300/0xff00

Chain mwan3_policy_wn3_wn2_wn (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan3 4 4 */ MARK xset 0x300/0xff00

Chain mwan3_policy_wn4_wn (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan 6 6 */ MARK xset 0x100/0xff00

Chain mwan3_policy_wn4_wn2 (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan2 5 5 */ MARK xset 0x200/0xff00

Chain mwan3_policy_wn4_wn3 (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan3 4 4 */ MARK xset 0x300/0xff00

Chain mwan3_policy_wn4_wn_wn2_wn3 (1 references)
 pkts bytes target     prot opt in     out     source               destination         
  300 17211 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan 6 6 */ MARK xset 0x100/0xff00

Chain mwan3_policy_wn5_wn (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan 6 6 */ MARK xset 0x100/0xff00

Chain mwan3_policy_wn5_wn2 (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan2 5 5 */ MARK xset 0x200/0xff00

Chain mwan3_policy_wn5_wn3 (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan3 4 4 */ MARK xset 0x300/0xff00

Chain mwan3_policy_wn5_wn_wn2_wn3 (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan 6 6 */ MARK xset 0x100/0xff00

Chain mwan3_policy_wn6_wn (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan 6 6 */ MARK xset 0x100/0xff00

Chain mwan3_policy_wn6_wn2 (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan2 5 5 */ MARK xset 0x200/0xff00

Chain mwan3_policy_wn6_wn3 (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan3 4 4 */ MARK xset 0x300/0xff00

Chain mwan3_policy_wn6_wn_wn2_wn3 (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan 6 6 */ MARK xset 0x100/0xff00

Chain mwan3_policy_wn_wn2 (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan 6 6 */ MARK xset 0x100/0xff00

Chain mwan3_policy_wn_wn2_wn3 (2 references)
 pkts bytes target     prot opt in     out     source               destination         
  368 24172 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan 6 6 */ MARK xset 0x100/0xff00

Chain mwan3_policy_wn_wn3 (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan 6 6 */ MARK xset 0x100/0xff00

Chain mwan3_policy_wn_wn4 (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan 6 6 */ MARK xset 0x100/0xff00

Chain mwan3_policy_wn_wn5 (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan 6 6 */ MARK xset 0x100/0xff00

Chain mwan3_policy_wn_wn6 (0 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* wan 6 6 */ MARK xset 0x100/0xff00

Chain mwan3_rules (1 references)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 mwan3_policy_wan2_only  all  --  *      *       0.0.0.0/0            192.168.4.1          mark match 0x0/0xff00 /* Vodafone */
    0     0 mwan3_policy_wan_only  all  --  *      *       0.0.0.0/0            216.146.38.70        mark match 0x0/0xff00 /* DNS_WAN1 */
    0     0 mwan3_policy_wan_only  all  --  *      *       0.0.0.0/0            91.198.22.70         mark match 0x0/0xff00 /* DNS_WAN1 */
    0     0 mwan3_policy_wan_only  all  --  *      *       0.0.0.0/0            216.146.43.70        mark match 0x0/0xff00 /* DNS_WAN1 */
    0     0 mwan3_policy_wan2_only  all  --  *      *       0.0.0.0/0            176.58.123.25        mark match 0x0/0xff00 /* DNS_WAN2 */
    0     0 mwan3_policy_wan3_only  all  --  *      *       0.0.0.0/0            64.182.208.182       mark match 0x0/0xff00 /* DNS_WAN3 */
    0     0 mwan3_policy_wan3_only  all  --  *      *       0.0.0.0/0            64.182.208.183       mark match 0x0/0xff00 /* DNS_WAN3 */
    0     0 mwan3_policy_wan3_only  all  --  *      *       0.0.0.0/0            64.182.208.181       mark match 0x0/0xff00 /* DNS_WAN3 */
    0     0 mwan3_policy_wan3_only  all  --  *      *       0.0.0.0/0            64.182.208.184       mark match 0x0/0xff00 /* DNS_WAN3 */
    7   420 mwan3_policy_wan_only  all  --  *      *       192.168.2.24         0.0.0.0/0            mark match 0x0/0xff00 /* SKY_Francesco */
    8   480 mwan3_policy_wan_only  all  --  *      *       192.168.2.25         0.0.0.0/0            mark match 0x0/0xff00 /* SKY_Soggiorno */
    0     0 mwan3_policy_wan4_only  all  --  *      *       192.168.2.22         0.0.0.0/0            mark match 0x0/0xff00 /* TV_Francesco */
  664 39840 mwan3_policy_wan4_only  all  --  *      *       192.168.2.23         0.0.0.0/0            mark match 0x0/0xff00 /* TV_Emanuele */
    0     0 mwan3_policy_wan4_only  all  --  *      *       192.168.2.44         0.0.0.0/0            mark match 0x0/0xff00 /* NAS */
    0     0 mwan3_policy_wan4_only  all  --  *      *       0.0.0.0/0            192.168.2.44         mark match 0x0/0xff00 /* NAS_2 */
    0     0 mwan3_policy_wan3_only  all  --  *      *       0.0.0.0/0            195.232.255.190      mark match 0x0/0xff00 /* Vodafone_3G */
  368 24172 mwan3_policy_wn_wn2_wn3  all  --  *      *       192.168.2.2          0.0.0.0/0            mark match 0x0/0xff00 /* Francesco */
    0     0 mwan3_policy_wn_wn2_wn3  all  --  *      *       192.168.2.4          0.0.0.0/0            mark match 0x0/0xff00 /* Francesco_2 */
  300 17211 mwan3_policy_wn4_wn_wn2_wn3  all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xff00 /* default_rule */

Chain qos_Default (2 references)
 pkts bytes target     prot opt in     out     source               destination         
19980 5651K CONNMARK   all  --  *      *       0.0.0.0/0            0.0.0.0/0            CONNMARK restore mask 0xf
 3386  259K qos_Default_ct  all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xf
 2855  313K MARK       icmp --  *      *       0.0.0.0/0            0.0.0.0/0            MARK xset 0x11/0xff
19980 5651K CONNMARK   all  --  *      *       0.0.0.0/0            0.0.0.0/0            CONNMARK save mask 0xff

Chain qos_Default_ct (1 references)
 pkts bytes target     prot opt in     out     source               destination         
   16  3451 MARK       all  --  *      *       192.168.2.40         0.0.0.0/0            mark match 0x0/0xf /* VoIP */ MARK xset 0x33/0xff
    0     0 MARK       all  --  *      *       0.0.0.0/0            192.168.2.40         mark match 0x0/0xf /* VoIP_2 */ MARK xset 0x33/0xff
 3370  255K MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0x0/0xf /* TOTALE */ MARK xset 0x44/0xff
 3386  259K CONNMARK   all  --  *      *       0.0.0.0/0            0.0.0.0/0            CONNMARK save mask 0xff

tnx for your great work

1,602 (edited by junari 2017-03-15 10:26:49)

Re: mwan3; multi-wan policy routing (general topic)

Hi all, I have some problem with balancing and reserving wire internet and 3g via mwan3.
When eth channel down, 3g down too after some time.
I go to mwan3track script:

ping -I $2 -c $4 -W $5 -s 8 -q $track_ip &> /dev/null

and increase -s the parameter from 4 bytes to 8 bytes, because my mobile operator not skip so small packetsize.
After that 3g don't falling, but the eth is up and down arbitrarily (when eth must be down).
I added debug messages:       

for track_ip in $track_ips; do
    ping -I $2 -c $4 -W $5 -s 8 -q $track_ip &> /dev/null
    if [ $? -eq 0 ]; then
        let host_up_count++
        logger -t mwan3track -p notice "Up count $2: $host_up_c$
    else
        let lost++
        logger -t mwan3track -p notice "Lost $2: $lost"
    fi
done

Then noticed that sometimes the ping succesful on the down interface, if the second active.
My ping command provided by busybox, if it important.
I change ping to /bin/ping and after that I don't have the same problem.

/bin/ping -I $2 -c $4 -W $5 -s 8 -q $track_ip &> /dev/null

Perhaps this information will be useful for someone.
PS: Sorry for my English.

1,603 (edited by wazaa 2017-03-19 00:56:12)

Re: mwan3; multi-wan policy routing (general topic)

Hello and thank you very much for mwan3! I have a small problem, when an interface goes offline, I see the correct message in the log (via logread): "... mwan3track: Interface wan4g1 (wwan1) is offline" but when the interface comes back online, I don't see the message "... mwan3track: Interface wan4g1 (wwan1) is online" anymore. It happens with all interfaces. I used to see the "online" log entries but not anymore. Any idea of what happened? FYI, If I comment

if [ $score -gt $8 ]; then score=$(($7+$8)); fi
if [ $score -eq $8 ]; then

in /usr/sbin/mwan3track, I do see the online messages. So it must be a problem with the $score evaluation

Thanks for any help

1,604

Re: mwan3; multi-wan policy routing (general topic)

wazaa, you can try added debug messages like me

1,605

Re: mwan3; multi-wan policy routing (general topic)

@januari, can you explain more?

1,606

Re: mwan3; multi-wan policy routing (general topic)

Replace in mwan3track

for track_ip in $track_ips; do
    ping -I $2 -c $4 -W $5 -s 4 -q $track_ip &> /dev/null
    if [ $? -eq 0 ]; then
        let host_up_count++
    else
        let lost++
    fi
done

to

for track_ip in $track_ips; do
    ping -I $2 -c $4 -W $5 -s 8 -q $track_ip &> /dev/null
    if [ $? -eq 0 ]; then
        let host_up_count++
        logger -t mwan3track -p notice "Up count $2: $host_up_count"
    else
        let lost++
        logger -t mwan3track -p notice "Lost $2: $lost"
    fi
done

And you can see what is happening with the interface

1,607 (edited by wazaa 2017-03-20 17:04:32)

Re: mwan3; multi-wan policy routing (general topic)

@junari, thanks it's a great idea! So I replaced the code in /usr/sbin/mwan3track and I monitored the logs while I had the USB hub (where all 4x LTE dongles are attached) turned ON, then turned OFF (at 16:25:10), then turned ON again (at 16:25:55). Here is the result:

root@OpenWrt:/usr/sbin# logread -f -e mwan3track
Mon Mar 20 16:25:00 2017 user.notice mwan3track: Up count wwan2: 1
Mon Mar 20 16:25:00 2017 user.notice mwan3track: Up count wwan2: 2
Mon Mar 20 16:25:00 2017 user.notice mwan3track: Up count wwan2: 3
Mon Mar 20 16:25:00 2017 user.notice mwan3track: Up count wwan2: 4
Mon Mar 20 16:25:02 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:25:02 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:25:02 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:25:02 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:25:03 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:25:03 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:25:03 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:25:03 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:25:03 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:25:03 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:25:03 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:25:03 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:25:04 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:25:04 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:25:04 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:25:04 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:25:05 2017 user.notice mwan3track: Up count wwan2: 1
Mon Mar 20 16:25:05 2017 user.notice mwan3track: Up count wwan2: 2
Mon Mar 20 16:25:05 2017 user.notice mwan3track: Up count wwan2: 3
Mon Mar 20 16:25:05 2017 user.notice mwan3track: Up count wwan2: 4
Mon Mar 20 16:25:07 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:25:07 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:25:07 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:25:07 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:25:08 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:25:08 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:25:08 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:25:08 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:25:08 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:25:08 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:25:08 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:25:08 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:25:09 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:25:09 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:25:09 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:25:09 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:25:10 2017 user.notice mwan3track: Lost wwan2: 1
Mon Mar 20 16:25:10 2017 user.notice mwan3track: Lost wwan2: 2
Mon Mar 20 16:25:11 2017 user.notice mwan3track: Lost wwan2: 3
Mon Mar 20 16:25:11 2017 user.notice mwan3track: Lost wwan2: 4
Mon Mar 20 16:25:12 2017 user.notice mwan3track: Lost wwan3: 1
Mon Mar 20 16:25:12 2017 user.notice mwan3track: Lost wwan3: 2
Mon Mar 20 16:25:12 2017 user.notice mwan3track: Lost wwan3: 3
Mon Mar 20 16:25:12 2017 user.notice mwan3track: Lost wwan3: 4
Mon Mar 20 16:25:13 2017 user.notice mwan3track: Lost wwan0: 1
Mon Mar 20 16:25:13 2017 user.notice mwan3track: Lost wwan0: 2
Mon Mar 20 16:25:13 2017 user.notice mwan3track: Lost wwan0: 3
Mon Mar 20 16:25:13 2017 user.notice mwan3track: Lost wwan0: 4
Mon Mar 20 16:25:13 2017 user.notice mwan3track: Lost wwan1: 1
Mon Mar 20 16:25:13 2017 user.notice mwan3track: Lost wwan1: 2
Mon Mar 20 16:25:13 2017 user.notice mwan3track: Lost wwan1: 3
Mon Mar 20 16:25:13 2017 user.notice mwan3track: Lost wwan1: 4
Mon Mar 20 16:25:14 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:25:14 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:25:15 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:25:15 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:25:16 2017 user.notice mwan3track: Lost wwan2: 5
Mon Mar 20 16:25:16 2017 user.notice mwan3track: Lost wwan2: 6
Mon Mar 20 16:25:16 2017 user.notice mwan3track: Lost wwan2: 7
Mon Mar 20 16:25:16 2017 user.notice mwan3track: Lost wwan2: 8
Mon Mar 20 16:25:17 2017 user.notice mwan3track: Lost wwan3: 5
Mon Mar 20 16:25:17 2017 user.notice mwan3track: Lost wwan3: 6
Mon Mar 20 16:25:17 2017 user.notice mwan3track: Lost wwan3: 7
Mon Mar 20 16:25:17 2017 user.notice mwan3track: Lost wwan3: 8
Mon Mar 20 16:25:18 2017 user.notice mwan3track: Lost wwan0: 5
Mon Mar 20 16:25:18 2017 user.notice mwan3track: Lost wwan0: 6
Mon Mar 20 16:25:18 2017 user.notice mwan3track: Lost wwan0: 7
Mon Mar 20 16:25:18 2017 user.notice mwan3track: Lost wwan0: 8
Mon Mar 20 16:25:19 2017 user.notice mwan3track: Lost wwan1: 5
Mon Mar 20 16:25:19 2017 user.notice mwan3track: Lost wwan1: 6
Mon Mar 20 16:25:19 2017 user.notice mwan3track: Lost wwan1: 7
Mon Mar 20 16:25:19 2017 user.notice mwan3track: Lost wwan1: 8
Mon Mar 20 16:25:20 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:25:20 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:25:20 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:25:20 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:25:21 2017 user.notice mwan3track: Lost wwan2: 9
Mon Mar 20 16:25:21 2017 user.notice mwan3track: Lost wwan2: 10
Mon Mar 20 16:25:21 2017 user.notice mwan3track: Lost wwan2: 11
Mon Mar 20 16:25:21 2017 user.notice mwan3track: Lost wwan2: 12
Mon Mar 20 16:25:21 2017 user.notice mwan3track: Interface wan4g2 (wwan2) is offline
Mon Mar 20 16:25:23 2017 user.notice mwan3track: Lost wwan3: 9
Mon Mar 20 16:25:23 2017 user.notice mwan3track: Lost wwan3: 10
Mon Mar 20 16:25:23 2017 user.notice mwan3track: Lost wwan3: 11
Mon Mar 20 16:25:23 2017 user.notice mwan3track: Lost wwan3: 12
Mon Mar 20 16:25:23 2017 user.notice mwan3track: Interface wan4g3 (wwan3) is offline
Mon Mar 20 16:25:23 2017 user.notice mwan3track: Lost wwan0: 9
Mon Mar 20 16:25:24 2017 user.notice mwan3track: Lost wwan0: 10
Mon Mar 20 16:25:24 2017 user.notice mwan3track: Lost wwan0: 11
Mon Mar 20 16:25:24 2017 user.notice mwan3track: Lost wwan0: 12
Mon Mar 20 16:25:24 2017 user.notice mwan3track: Interface wan4g0 (wwan0) is offline
Mon Mar 20 16:25:24 2017 user.notice mwan3track: Lost wwan1: 9
Mon Mar 20 16:25:24 2017 user.notice mwan3track: Lost wwan1: 10
Mon Mar 20 16:25:24 2017 user.notice mwan3track: Lost wwan1: 11
Mon Mar 20 16:25:24 2017 user.notice mwan3track: Lost wwan1: 12
Mon Mar 20 16:25:24 2017 user.notice mwan3track: Interface wan4g1 (wwan1) is offline
Mon Mar 20 16:25:25 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:25:25 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:25:25 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:25:25 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:25:30 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:25:30 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:25:31 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:25:31 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:25:36 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:25:36 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:25:36 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:25:36 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:25:38 2017 user.notice mwan3track: Lost wwan2: 13
Mon Mar 20 16:25:38 2017 user.notice mwan3track: Lost wwan2: 14
Mon Mar 20 16:25:38 2017 user.notice mwan3track: Lost wwan2: 15
Mon Mar 20 16:25:38 2017 user.notice mwan3track: Lost wwan2: 16
Mon Mar 20 16:25:41 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:25:41 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:25:41 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:25:41 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:25:43 2017 user.notice mwan3track: Lost wwan2: 17
Mon Mar 20 16:25:43 2017 user.notice mwan3track: Lost wwan2: 18
Mon Mar 20 16:25:43 2017 user.notice mwan3track: Lost wwan2: 19
Mon Mar 20 16:25:43 2017 user.notice mwan3track: Lost wwan2: 20
Mon Mar 20 16:25:46 2017 user.notice mwan3track: Lost wwan3: 13
Mon Mar 20 16:25:46 2017 user.notice mwan3track: Lost wwan3: 14
Mon Mar 20 16:25:46 2017 user.notice mwan3track: Lost wwan3: 15
Mon Mar 20 16:25:46 2017 user.notice mwan3track: Lost wwan3: 16
Mon Mar 20 16:25:46 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:25:46 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:25:46 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:25:47 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:25:52 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:25:52 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:25:52 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:25:52 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:25:52 2017 user.notice mwan3track: Lost wwan2: 21
Mon Mar 20 16:25:53 2017 user.notice mwan3track: Lost wwan0: 13
Mon Mar 20 16:25:55 2017 user.notice mwan3track: Lost wwan3: 17
Mon Mar 20 16:25:55 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:25:55 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:25:55 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:25:55 2017 user.info mwan3track: Lost 17 ping(s) on interface wan4g3 (wwan3)
Mon Mar 20 16:25:56 2017 user.notice mwan3track: Lost wwan2: 22
Mon Mar 20 16:25:57 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:25:57 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:25:57 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:25:57 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:25:57 2017 user.notice mwan3track: Lost wwan0: 14
Mon Mar 20 16:26:00 2017 user.notice mwan3track: Lost wwan2: 23
Mon Mar 20 16:26:00 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:26:00 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:26:00 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:26:00 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:26:01 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:01 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:01 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:01 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:26:01 2017 user.info mwan3track: Lost 12 ping(s) on interface wan4g1 (wwan1)
Mon Mar 20 16:26:02 2017 user.notice mwan3track: Lost wwan0: 15
Mon Mar 20 16:26:02 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:26:02 2017 user.info mwan3track: Lost 15 ping(s) on interface wan4g0 (wwan0)
Mon Mar 20 16:26:03 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:26:03 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:26:03 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:26:03 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:26:03 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:26:03 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:26:03 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:26:03 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:26:04 2017 user.notice mwan3track: Lost wwan2: 24
Mon Mar 20 16:26:06 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:06 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:06 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:06 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:26:07 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:26:07 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:26:07 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:26:07 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:26:08 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:26:08 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:26:08 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:26:08 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:26:08 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:26:08 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:26:08 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:26:08 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:26:10 2017 user.notice mwan3track: Up count wwan2: 1
Mon Mar 20 16:26:10 2017 user.notice mwan3track: Up count wwan2: 2
Mon Mar 20 16:26:10 2017 user.notice mwan3track: Up count wwan2: 3
Mon Mar 20 16:26:10 2017 user.notice mwan3track: Up count wwan2: 4
Mon Mar 20 16:26:10 2017 user.info mwan3track: Lost 24 ping(s) on interface wan4g2 (wwan2)
Mon Mar 20 16:26:10 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:10 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:10 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:10 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:26:12 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:26:12 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:26:12 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:26:12 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:26:12 2017 user.notice mwan3track: Interface wan4g0 (wwan0) is online
Mon Mar 20 16:26:13 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:26:13 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:26:13 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:26:13 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:26:14 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:26:14 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:26:14 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:26:14 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:26:15 2017 user.notice mwan3track: Up count wwan2: 1
Mon Mar 20 16:26:15 2017 user.notice mwan3track: Up count wwan2: 2
Mon Mar 20 16:26:15 2017 user.notice mwan3track: Up count wwan2: 3
Mon Mar 20 16:26:15 2017 user.notice mwan3track: Up count wwan2: 4
Mon Mar 20 16:26:15 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:15 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:15 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:15 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:26:18 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:26:18 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:26:18 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:26:18 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:26:18 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:26:18 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:26:19 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:26:19 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:26:19 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:26:19 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:26:19 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:26:19 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:26:20 2017 user.notice mwan3track: Up count wwan2: 1
Mon Mar 20 16:26:20 2017 user.notice mwan3track: Up count wwan2: 2
Mon Mar 20 16:26:20 2017 user.notice mwan3track: Up count wwan2: 3
Mon Mar 20 16:26:20 2017 user.notice mwan3track: Up count wwan2: 4
Mon Mar 20 16:26:20 2017 user.notice mwan3track: Interface wan4g2 (wwan2) is online
Mon Mar 20 16:26:20 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:21 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:21 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:21 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:26:24 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:26:24 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:26:24 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:26:24 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:26:24 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:26:24 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:26:24 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:26:24 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:26:24 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:26:24 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:26:24 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:26:24 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:26:26 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:26 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:26 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:26 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:26:26 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:26:26 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:26:26 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:26:26 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:26:29 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:26:29 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:26:29 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:26:29 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:26:29 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:26:29 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:26:29 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:26:29 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:26:31 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:31 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:31 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:31 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:26:31 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:26:31 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:26:31 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:26:31 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:26:34 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:26:34 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:26:34 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:26:34 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:26:34 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:26:35 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:26:35 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:26:35 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:26:35 2017 user.notice mwan3track: Up count wwan2: 1
Mon Mar 20 16:26:35 2017 user.notice mwan3track: Up count wwan2: 2
Mon Mar 20 16:26:35 2017 user.notice mwan3track: Up count wwan2: 3
Mon Mar 20 16:26:35 2017 user.notice mwan3track: Up count wwan2: 4
Mon Mar 20 16:26:36 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:36 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:36 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:36 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:26:37 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:26:37 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:26:37 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:26:37 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:26:39 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:26:39 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:26:39 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:26:40 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:26:40 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:26:40 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:26:40 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:26:40 2017 user.notice mwan3track: Up count wwan2: 1
Mon Mar 20 16:26:40 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:26:40 2017 user.notice mwan3track: Up count wwan2: 2
Mon Mar 20 16:26:40 2017 user.notice mwan3track: Up count wwan2: 3
Mon Mar 20 16:26:40 2017 user.notice mwan3track: Up count wwan2: 4
Mon Mar 20 16:26:41 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:41 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:42 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:42 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:26:42 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:26:42 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:26:42 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:26:42 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:26:43 2017 user.notice mwan3track: Up count wwan2: 1
Mon Mar 20 16:26:43 2017 user.notice mwan3track: Up count wwan2: 2
Mon Mar 20 16:26:43 2017 user.notice mwan3track: Up count wwan2: 3
Mon Mar 20 16:26:43 2017 user.notice mwan3track: Up count wwan2: 4
Mon Mar 20 16:26:45 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:26:45 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:26:45 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:26:45 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:26:45 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:26:45 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:26:45 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:26:45 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:26:47 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:47 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:47 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:47 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:26:47 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:26:47 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:26:47 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:26:47 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:26:48 2017 user.notice mwan3track: Up count wwan2: 1
Mon Mar 20 16:26:48 2017 user.notice mwan3track: Up count wwan2: 2
Mon Mar 20 16:26:48 2017 user.notice mwan3track: Up count wwan2: 3
Mon Mar 20 16:26:48 2017 user.notice mwan3track: Up count wwan2: 4
Mon Mar 20 16:26:50 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:26:50 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:26:50 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:26:50 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:26:50 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:26:50 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:26:50 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:26:50 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:26:52 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:52 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:52 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:52 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:26:52 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:26:52 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:26:52 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:26:52 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:26:53 2017 user.notice mwan3track: Up count wwan2: 1
Mon Mar 20 16:26:53 2017 user.notice mwan3track: Up count wwan2: 2
Mon Mar 20 16:26:53 2017 user.notice mwan3track: Up count wwan2: 3
Mon Mar 20 16:26:53 2017 user.notice mwan3track: Up count wwan2: 4
Mon Mar 20 16:26:55 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:26:55 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:26:55 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:26:55 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:26:56 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:26:56 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:26:56 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:26:56 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:26:57 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:57 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:57 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:57 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:26:57 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:26:58 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:26:58 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:26:58 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:26:58 2017 user.notice mwan3track: Up count wwan2: 1
Mon Mar 20 16:26:58 2017 user.notice mwan3track: Up count wwan2: 2
Mon Mar 20 16:26:59 2017 user.notice mwan3track: Up count wwan2: 3
Mon Mar 20 16:26:59 2017 user.notice mwan3track: Up count wwan2: 4
Mon Mar 20 16:27:00 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:27:00 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:27:00 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:27:00 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:27:01 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:27:01 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:27:01 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:27:01 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:27:02 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:27:02 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:27:02 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:27:02 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:27:03 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:27:03 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:27:03 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:27:03 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:27:04 2017 user.notice mwan3track: Up count wwan2: 1
Mon Mar 20 16:27:04 2017 user.notice mwan3track: Up count wwan2: 2
Mon Mar 20 16:27:04 2017 user.notice mwan3track: Up count wwan2: 3
Mon Mar 20 16:27:04 2017 user.notice mwan3track: Up count wwan2: 4
Mon Mar 20 16:27:05 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:27:05 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:27:06 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:27:06 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:27:06 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:27:06 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:27:06 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:27:06 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:27:08 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:27:08 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:27:08 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:27:08 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:27:08 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:27:08 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:27:08 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:27:08 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:27:09 2017 user.notice mwan3track: Up count wwan2: 1
Mon Mar 20 16:27:09 2017 user.notice mwan3track: Up count wwan2: 2
Mon Mar 20 16:27:09 2017 user.notice mwan3track: Up count wwan2: 3
Mon Mar 20 16:27:09 2017 user.notice mwan3track: Up count wwan2: 4
Mon Mar 20 16:27:11 2017 user.notice mwan3track: Up count eth0.2: 1
Mon Mar 20 16:27:11 2017 user.notice mwan3track: Up count eth0.2: 2
Mon Mar 20 16:27:11 2017 user.notice mwan3track: Up count eth0.2: 3
Mon Mar 20 16:27:11 2017 user.notice mwan3track: Up count eth0.2: 4
Mon Mar 20 16:27:11 2017 user.notice mwan3track: Up count wwan3: 1
Mon Mar 20 16:27:11 2017 user.notice mwan3track: Up count wwan3: 2
Mon Mar 20 16:27:11 2017 user.notice mwan3track: Up count wwan3: 3
Mon Mar 20 16:27:11 2017 user.notice mwan3track: Up count wwan3: 4
Mon Mar 20 16:27:13 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:27:13 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:27:13 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:27:13 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:27:13 2017 user.notice mwan3track: Up count wwan0: 1
Mon Mar 20 16:27:13 2017 user.notice mwan3track: Up count wwan0: 2
Mon Mar 20 16:27:13 2017 user.notice mwan3track: Up count wwan0: 3
Mon Mar 20 16:27:13 2017 user.notice mwan3track: Up count wwan0: 4
Mon Mar 20 16:27:14 2017 user.notice mwan3track: Up count wwan2: 1
Mon Mar 20 16:27:14 2017 user.notice mwan3track: Up count wwan2: 2
Mon Mar 20 16:27:14 2017 user.notice mwan3track: Up count wwan2: 3
Mon Mar 20 16:27:14 2017 user.notice mwan3track: Up count wwan2: 4

as you can see it is weird because I see the "is online" message for wwan0 and wwan2 but not for wwan1 and wwan3 while they both came back to an "Up count" of 4. Here is a filter on the lines of wwan1:

Mon Mar 20 16:25:08 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:25:08 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:25:08 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:25:08 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:25:13 2017 user.notice mwan3track: Lost wwan1: 1
Mon Mar 20 16:25:13 2017 user.notice mwan3track: Lost wwan1: 2
Mon Mar 20 16:25:13 2017 user.notice mwan3track: Lost wwan1: 3
Mon Mar 20 16:25:13 2017 user.notice mwan3track: Lost wwan1: 4
Mon Mar 20 16:25:19 2017 user.notice mwan3track: Lost wwan1: 5
Mon Mar 20 16:25:19 2017 user.notice mwan3track: Lost wwan1: 6
Mon Mar 20 16:25:19 2017 user.notice mwan3track: Lost wwan1: 7
Mon Mar 20 16:25:19 2017 user.notice mwan3track: Lost wwan1: 8
Mon Mar 20 16:25:24 2017 user.notice mwan3track: Lost wwan1: 9
Mon Mar 20 16:25:24 2017 user.notice mwan3track: Lost wwan1: 10
Mon Mar 20 16:25:24 2017 user.notice mwan3track: Lost wwan1: 11
Mon Mar 20 16:25:24 2017 user.notice mwan3track: Lost wwan1: 12
Mon Mar 20 16:25:24 2017 user.notice mwan3track: Interface wan4g1 (wwan1) is offline
Mon Mar 20 16:26:01 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:01 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:01 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:01 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:26:01 2017 user.info mwan3track: Lost 12 ping(s) on interface wan4g1 (wwan1)
Mon Mar 20 16:26:06 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:06 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:06 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:06 2017 user.notice mwan3track: Up count wwan1: 4
Mon Mar 20 16:26:10 2017 user.notice mwan3track: Up count wwan1: 1
Mon Mar 20 16:26:10 2017 user.notice mwan3track: Up count wwan1: 2
Mon Mar 20 16:26:10 2017 user.notice mwan3track: Up count wwan1: 3
Mon Mar 20 16:26:10 2017 user.notice mwan3track: Up count wwan1: 4

Here is my config on these interfaces:

config interface 'wan'
    option enabled '1'
    option reliability '1'
    option count '1'
    option timeout '4'
    option interval '5'
    option down '3'
    option up '3'
    list track_ip '8.8.4.4'
    list track_ip '208.67.222.222'
    list track_ip '8.8.8.8'
    list track_ip '208.67.220.220'

config interface 'wan4g0'
    option enabled '1'
    option reliability '1'
        option count '1'
    option timeout '4'
    option interval '5'
    option down '3'
    option up '3'
    list track_ip '208.67.220.220'
    list track_ip '8.8.4.4'
    list track_ip '208.67.222.222'
    list track_ip '8.8.8.8'

config interface 'wan4g1'
    option enabled '1'
    option reliability '1'
        option count '1'
    option timeout '4'
    option interval '5'
    option down '3'
    option up '3'
    list track_ip '8.8.8.8'
    list track_ip '208.67.220.220'
    list track_ip '8.8.4.4'
    list track_ip '208.67.222.222'

config interface 'wan4g2'
    option enabled '1'
    option reliability '1'
        option count '1'
    option timeout '4'
    option interval '5'
    option down '3'
    option up '3'
    list track_ip '208.67.222.222'
    list track_ip '8.8.8.8'
    list track_ip '208.67.220.220'
    list track_ip '8.8.4.4'

config interface 'wan4g3'
    option enabled '1'
    option reliability '1'
        option count '1'
    option timeout '4'
    option interval '5'
    option down '3'
    option up '3'
    list track_ip '8.8.4.4'
    list track_ip '208.67.222.222'
    list track_ip '8.8.8.8'
    list track_ip '208.67.220.220'

FYI: wan4g0=wwan0, wan4g1=wwan1, wan4g2=wwan2, wan4g3=wwan3.
Do you understand why I don't see the "is online" message in the logs for wwan1 and wwan3??