Hi tcherenato,
I'd like to help you and get things running. But all in the right order. Your fiddling with mwan3, but your main routing table is still not correct. My advise to you is:
- Start over with reinstall of openwrt, but don't install mwan3 yet.
- Make sure your network config is OK and you have a default route for each wan interface in your default routing table.
- Don't continue with mwan3, until you extensively tested this setup. (reboot router, bring interfaces up and down and each time check if the main routing table is correct.)Only then continue installing mwan3. Your current setup is (as far as i can see) not an mwan3 problem.
Hi Tim, thanks for your reply!
I think that the routing tables are correct:
root-> ip route
default via 192.168.67.254 dev eth1 proto static metric 40
default via 10.64.64.64 dev 3g-wan1 proto static metric 50
10.64.64.64 dev 3g-wan1 proto kernel scope link src 187.119.141.115
192.168.66.0/24 dev eth0 proto kernel scope link src 192.168.66.254
192.168.67.0/24 dev eth1 proto static scope link metric 40
root-> ping -c 1 -I eth1 google.com
PING google.com (186.215.92.113): 56 data bytes
64 bytes from 186.215.92.113: seq=0 ttl=57 time=10.324 ms
root-> ping -c 1 -I 3g-wan1 google.com
PING google.com (186.215.92.93): 56 data bytes
64 bytes from 186.215.92.93: seq=0 ttl=52 time=407.254 ms
I can turn off and on the network interfaces without problems.
root-> ifconfig 3g-wan1 down
root-> ip route
default via 192.168.67.254 dev eth1 proto static metric 40
192.168.66.0/24 dev eth0 proto kernel scope link src 192.168.66.254
192.168.67.0/24 dev eth1 proto static scope link metric 40
MWAN3 Multi-WAN Interface Live Status
Aug 7 12:47:30 MR3220V12 user.info mwan3track: Interface wan1 (3g-wan1) is offline
root-> /etc/init.d/network restart
root-> ifconfig 3g-wan1
3g-wan1 Link encap:Point-to-Point Protocol
inet addr:189.96.231.203 P-t-P:10.64.64.64 Mask:255.255.255.255
UP POINTOPOINT RUNNING NOARP MULTICAST MTU:1500 Metric:1
RX packets:7 errors:0 dropped:0 overruns:0 frame:0
TX packets:7 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:3
RX bytes:124 (124.0 B) TX bytes:148 (148.0 B)
Aug 7 12:59:15 MR3220V12 user.info mwan3: ifup interface wan1 (3g-wan1)
root-> ip route
default via 192.168.67.254 dev eth1 proto static metric 40
default via 10.64.64.64 dev 3g-wan1 proto static metric 50
10.64.64.64 dev 3g-wan1 proto kernel scope link src 177.145.52.81
192.168.66.0/24 dev eth0 proto kernel scope link src 192.168.66.254
192.168.67.0/24 dev eth1 proto static scope link metric 40
root-> ifconfig eth1 down
root-> ip route
default via 10.64.64.64 dev 3g-wan1 proto static metric 50
10.64.64.64 dev 3g-wan1 proto kernel scope link src 189.96.231.203
192.168.66.0/24 dev eth0 proto kernel scope link src 192.168.66.254
Aug 7 13:03:13 MR3220V12 user.info mwan3track: Interface wan2 (eth1) is offline
root-> /etc/init.d/network restart
root-> ifconfig eth1
eth1 Link encap:Ethernet HWaddr 90:F6:52:B9:CF:29
inet addr:192.168.67.1 Bcast:192.168.67.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:6101 errors:0 dropped:0 overruns:0 frame:0
TX packets:6494 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:1915537 (1.8 MiB) TX bytes:939629 (917.6 KiB)
Interrupt:4
Aug 7 13:06:21 MR3220V12 user.info mwan3: ifup interface wan2 (eth1)
root-> ip route
default via 192.168.67.254 dev eth1 proto static metric 40
default via 10.64.64.64 dev 3g-wan1 proto static metric 50
10.64.64.64 dev 3g-wan1 proto kernel scope link src 177.145.52.81
192.168.66.0/24 dev eth0 proto kernel scope link src 192.168.66.254
192.168.67.0/24 dev eth1 proto static scope link metric 40
Testing a rule:
config rule 'whatsmyip'
option dest_ip 'whatsmyip.us' (67.228.228.244)
option proto 'all'
option use_policy 'wan1_only'
Ping inside the network:
renato@FX6100:~$ ping -c1 67.228.228.244
PING 67.228.228.244 (67.228.228.244) 56(84) bytes of data.
64 bytes from 67.228.228.244: icmp_req=1 ttl=47 time=409 ms
In OpenWRT:
root-> tcpdump -i eth1 dst host 67.228.228.244 -n -v
tcpdump: listening on eth1, link-type EN10MB (Ethernet), capture size 65535 bytes
NOTHING
root-> tcpdump -i 3g-wan1 dst host 67.228.228.244 -n -v
tcpdump: listening on 3g-wan1, link-type LINUX_SLL (Linux cooked), capture size 65535 bytes
13:30:24.569295 IP (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto ICMP (1), length 84)
177.145.52.81 > 67.228.228.244: ICMP echo request, id 3671, seq 1, length 64
MWAN is working perfectly!
Now 3g-wan is off:
root-> ifconfig 3g-wan1 down
Aug 7 13:36:00 MR3220V12 user.info mwan3track: Interface wan1 (3g-wan1) is offline
renato@FX6100:~$ ping -c1 67.228.228.244
PING 67.228.228.244 (67.228.228.244) 56(84) bytes of data.
64 bytes from 67.228.228.244: icmp_req=1 ttl=49 time=234 ms
root-> tcpdump -i eth1 dst host 67.228.228.244 -n -v
tcpdump: listening on eth1, link-type EN10MB (Ethernet), capture size 65535 bytes
13:36:55.054979 IP (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto ICMP (1), length 84)
192.168.67.1 > 67.228.228.244: ICMP echo request, id 3693, seq 1, length 64
By rule, should go out only by wan1, but is leaving by wan2 when wan1 is off.
root-> iptables -L -t mangle -n -v |grep 'mwan3_rules\|67.228.228.244'
391 24097 mwan3_rules all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x0/0xff00
Chain mwan3_rules (1 references)
4 240 MARK all -- * * 0.0.0.0/0 67.228.228.244 mark match 0x0/0xff00 MARK xset 0x1000/0xff00
root-> ip route list table 0
default via 10.64.64.64 dev 3g-wan1 table 1016 metric 1
default via 192.168.67.254 dev eth1 table 1022 metric 1
default via 10.64.64.64 dev 3g-wan1 table 1022 metric 2
default via 192.168.67.254 dev eth1 table 1020 metric 1
default via 10.64.64.64 dev 3g-wan1 table 1020 metric 2
default table 1018 metric 1
nexthop via 192.168.67.254 dev eth1 weight 2
nexthop via 10.64.64.64 dev 3g-wan1 weight 3
default via 192.168.67.254 dev eth1 table 1002
default via 192.168.67.254 dev eth1 proto static metric 40
default via 10.64.64.64 dev 3g-wan1 proto static metric 50
10.64.64.64 dev 3g-wan1 proto kernel scope link src 179.224.184.192
192.168.66.0/24 dev eth0 proto kernel scope link src 192.168.66.254
192.168.67.0/24 dev eth1 proto static scope link metric 40
default via 10.64.64.64 dev 3g-wan1 table 1001
default via 192.168.67.254 dev eth1 table 1021 metric 2
default via 10.64.64.64 dev 3g-wan1 table 1019 metric 1
default via 192.168.67.254 dev eth1 table 1019 metric 2
default via 192.168.67.254 dev eth1 table 1017 metric 1
broadcast 127.0.0.0 dev lo table local proto kernel scope link src 127.0.0.1
local 127.0.0.0/8 dev lo table local proto kernel scope host src 127.0.0.1
local 127.0.0.1 dev lo table local proto kernel scope host src 127.0.0.1
broadcast 127.255.255.255 dev lo table local proto kernel scope link src 127.0.0.1
local 179.224.184.192 dev 3g-wan1 table local proto kernel scope host src 179.224.184.192
broadcast 192.168.66.0 dev eth0 table local proto kernel scope link src 192.168.66.254
local 192.168.66.254 dev eth0 table local proto kernel scope host src 192.168.66.254
broadcast 192.168.66.255 dev eth0 table local proto kernel scope link src 192.168.66.254
broadcast 192.168.67.0 dev eth1 table local proto kernel scope link src 192.168.67.1
local 192.168.67.1 dev eth1 table local proto kernel scope host src 192.168.67.1
broadcast 192.168.67.255 dev eth1 table local proto kernel scope link src 192.168.67.1
[~]
Software versions :
OpenWrt - OpenWrt Attitude Adjustment 12.09 (r36088)
mwan3 - 1.2-17
luci-app-mwan3 - 1.1-13
Firewall default output policy (must be ACCEPT) :
ACCEPT
Output of "ip route show" :
default via 192.168.67.254 dev eth1 proto static metric 40
default via 10.64.64.64 dev 3g-wan1 proto static metric 50
10.64.64.64 dev 3g-wan1 proto kernel scope link src 179.224.184.192
192.168.66.0/24 dev eth0 proto kernel scope link src 192.168.66.254
192.168.67.0/24 dev eth1 proto static scope link metric 40
Output of "ip rule show" :
0: from all lookup local
1001: from all fwmark 0x100/0xff00 lookup 1001
1002: from all fwmark 0x200/0xff00 lookup 1002
1016: from all fwmark 0x1000/0xff00 lookup 1016
1017: from all fwmark 0x1100/0xff00 lookup 1017
1018: from all fwmark 0x1200/0xff00 lookup 1018
1019: from all fwmark 0x1300/0xff00 lookup 1019
1020: from all fwmark 0x1400/0xff00 lookup 1020
1021: from all fwmark 0x1500/0xff00 lookup 1021
1022: from all fwmark 0x1600/0xff00 lookup 1022
32766: from all lookup main
32767: from all lookup default
Output of "ip route list table 1001-1099" (1001-1015 = interface tables, 1016-1099 = policy tables) :
1001
default via 10.64.64.64 dev 3g-wan1
1002
default via 192.168.67.254 dev eth1
1016
default via 10.64.64.64 dev 3g-wan1 metric 1
1017
default via 192.168.67.254 dev eth1 metric 1
1018
default metric 1
nexthop via 192.168.67.254 dev eth1 weight 2
nexthop via 10.64.64.64 dev 3g-wan1 weight 3
1019
default via 10.64.64.64 dev 3g-wan1 metric 1
default via 192.168.67.254 dev eth1 metric 2
1020
default via 192.168.67.254 dev eth1 metric 1
default via 10.64.64.64 dev 3g-wan1 metric 2
1021
default via 192.168.67.254 dev eth1 metric 2
1022
default via 192.168.67.254 dev eth1 metric 1
default via 10.64.64.64 dev 3g-wan1 metric 2
Output of "iptables -L -t mangle -v -n | awk '/mwan3/' RS=" :
Chain PREROUTING (policy ACCEPT 2951 packets, 594K bytes)
pkts bytes target prot opt in out source destination
3032 601K mwan3_pre all -- * * 0.0.0.0/0 0.0.0.0/0
Chain INPUT (policy ACCEPT 1183 packets, 93953 bytes)
pkts bytes target prot opt in out source destination
1235 98251 mwan3_post all -- * * 0.0.0.0/0 0.0.0.0/0
Chain OUTPUT (policy ACCEPT 1214 packets, 418K bytes)
pkts bytes target prot opt in out source destination
1276 430K mwan3_pre all -- * * 0.0.0.0/0 0.0.0.0/0
Chain POSTROUTING (policy ACCEPT 2982 packets, 918K bytes)
pkts bytes target prot opt in out source destination
3066 932K mwan3_post all -- * * 0.0.0.0/0 0.0.0.0/0
Chain mwan3_default (1 references)
pkts bytes target prot opt in out source destination
1 142 MARK all -- * * 0.0.0.0/0 224.0.0.0/3 mark match 0x0/0xff00 MARK xset 0x7f00/0xff00
0 0 MARK all -- * * 0.0.0.0/0 127.0.0.0/8 mark match 0x0/0xff00 MARK xset 0x7f00/0xff00
0 0 MARK all -- * * 0.0.0.0/0 10.64.64.64 mark match 0x0/0xff00 MARK xset 0x7f00/0xff00
62 4787 MARK all -- * * 0.0.0.0/0 192.168.66.0/24 mark match 0x0/0xff00 MARK xset 0x7f00/0xff00
0 0 MARK all -- * * 0.0.0.0/0 192.168.67.0/24 mark match 0x0/0xff00 MARK xset 0x7f00/0xff00
Chain mwan3_post (2 references)
pkts bytes target prot opt in out source destination
742 89863 MARK all -- * eth1 0.0.0.0/0 0.0.0.0/0 mark match !0x7f00/0xff00 MARK xset 0x200/0xff00
487 70061 MARK all -- * 3g-wan1 0.0.0.0/0 0.0.0.0/0 mark match !0x7f00/0xff00 MARK xset 0x100/0xff00
1112 386K MARK all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x8000/0x8000 MARK and 0xffff7fff
4301 1030K CONNMARK all -- * * 0.0.0.0/0 0.0.0.0/0 CONNMARK save mask 0xff00
Chain mwan3_pre (2 references)
pkts bytes target prot opt in out source destination
4308 1031K CONNMARK all -- * * 0.0.0.0/0 0.0.0.0/0 CONNMARK restore mask 0xff00
689 278K MARK all -- eth1 * 0.0.0.0/0 0.0.0.0/0 MARK xset 0x8200/0xff00
423 108K MARK all -- 3g-wan1 * 0.0.0.0/0 0.0.0.0/0 MARK xset 0x8100/0xff00
486 47698 mwan3_default all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x0/0xff00
408 41149 mwan3_wan all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x0/0xff00
140 20184 mwan3_rules all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x0/0xff00
Chain mwan3_rules (1 references)
pkts bytes target prot opt in out source destination
8 480 MARK all -- * * 0.0.0.0/0 67.228.228.244 mark match 0x0/0xff00 MARK xset 0x1000/0xff00
29 1761 MARK tcp -- * * 0.0.0.0/0 0.0.0.0/0 multiport sports 0:65535 multiport dports 443 mark match 0x0/0xff00 MARK xset 0x1000/0xff00
74 14448 MARK all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x0/0xff00 MARK xset 0x1400/0xff00
Chain mwan3_wan (1 references)
pkts bytes target prot opt in out source destination
408 41149 mwan3_wan1 all -- * * 0.0.0.0/0 0.0.0.0/0
380 37828 mwan3_wan2 all -- * * 0.0.0.0/0 0.0.0.0/0
Chain mwan3_wan1 (1 references)
pkts bytes target prot opt in out source destination
62 5192 MARK all -- * * 179.224.184.192 0.0.0.0/0 MARK xset 0x100/0xff00
Chain mwan3_wan2 (1 references)
pkts bytes target prot opt in out source destination
206 15773 MARK all -- * * 192.168.67.1 0.0.0.0/0 MARK xset 0x200/0xff00
Output of "ifconfig" :
3g-wan1 Link encap:Point-to-Point Protocol
inet addr:179.224.184.192 P-t-P:10.64.64.64 Mask:255.255.255.255
UP POINTOPOINT RUNNING NOARP MULTICAST MTU:1500 Metric:1
RX packets:453 errors:0 dropped:0 overruns:0 frame:0
TX packets:522 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:3
RX bytes:110095 (107.5 KiB) TX bytes:72267 (70.5 KiB)
eth0 Link encap:Ethernet HWaddr 90:F6:52:B9:CF:27
inet addr:192.168.66.254 Bcast:192.168.66.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:12615 errors:0 dropped:0 overruns:0 frame:0
TX packets:11188 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:1593959 (1.5 MiB) TX bytes:4966966 (4.7 MiB)
Interrupt:5
eth1 Link encap:Ethernet HWaddr 90:F6:52:B9:CF:29
inet addr:192.168.67.1 Bcast:192.168.67.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:2137 errors:0 dropped:0 overruns:0 frame:0
TX packets:3146 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:732417 (715.2 KiB) TX bytes:408158 (398.5 KiB)
Interrupt:4
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:151 errors:0 dropped:0 overruns:0 frame:0
TX packets:151 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:11666 (11.3 KiB) TX bytes:11666 (11.3 KiB)
Output of "cat /etc/config/mwan3" :
config rule 'whatsmyip'
option dest_ip 'whatsmyip.us'
option proto 'all'
option use_policy 'wan1_only'
config rule 'Porta443'
option dest_port '443'
option proto 'tcp'
option use_policy 'wan1_only'
config rule 'DefaultGW'
option dest_ip '0.0.0.0/0'
option proto 'all'
option use_policy 'wan2_pri_wan1_sec'
config interface 'wan1'
option enabled '1'
option count '1'
option down '3'
option up '8'
option reroute '0'
list track_ip '8.8.8.8'
list track_ip '208.67.220.220'
option reliability '1'
option timeout '3'
option interval '10'
config interface 'wan2'
option enabled '1'
list track_ip '8.8.8.8'
list track_ip '208.67.220.220'
option reliability '1'
option count '1'
option timeout '2'
option interval '5'
option down '3'
option up '8'
option reroute '0'
config member 'wan1_m1_w3'
option interface 'wan1'
option metric '1'
option weight '3'
config member 'wan1_m2_w3'
option interface 'wan1'
option metric '2'
option weight '3'
config member 'wan2_m1_w2'
option interface 'wan2'
option metric '1'
option weight '2'
config member 'wan2_m2_w2'
option interface 'wan2'
option metric '2'
option weight '2'
config policy 'wan1_only'
list use_member 'wan1_m1_w3'
config policy 'wan2_only'
list use_member 'wan2_m1_w2'
config policy 'wan1_wan2_loadbalanced'
list use_member 'wan1_m1_w3'
list use_member 'wan2_m1_w2'
config policy 'wan1_pri_wan2_sec'
list use_member 'wan1_m1_w3'
list use_member 'wan2_m2_w2'
config policy 'wan2_pri_wan1_sec'
list use_member 'wan1_m2_w3'
list use_member 'wan2_m1_w2'
config interface 'wan3'
list track_ip '8.8.8.8'
list track_ip '208.67.220.220'
option reliability '1'
option count '1'
option timeout '2'
option interval '5'
option down '3'
option up '5'
option reroute '0'
option enabled '1'
config member 'wan3_m1_w2'
option interface 'wan3'
option metric '1'
option weight '2'
config member 'wan3_m2_w2'
option interface 'wan3'
option metric '2'
option weight '2'
config policy 'wan2_wan3_loadbalanced'
list use_member 'wan2_m2_w2'
list use_member 'wan3_m2_w2'
config policy 'wan2_wan3_Pri_wan1_sec'
list use_member 'wan1_m2_w3'
list use_member 'wan2_m1_w2'
list use_member 'wan3_m1_w2'
Output of "cat /etc/config/network" :
config interface 'loopback'
option ifname 'lo'
option proto 'static'
option ipaddr '127.0.0.1'
option netmask '255.0.0.0'
config interface 'lan'
option proto 'static'
option netmask '255.255.255.0'
option ipaddr '192.168.66.254'
option _orig_ifname 'eth0'
option _orig_bridge 'false'
option ifname 'eth0'
config switch
option name 'eth0'
option reset '1'
config interface 'wan1'
option proto '3g'
option device '/dev/ttyUSB2'
option apn 'zap.vivo.com.br'
USERNAME HIDDEN
PASSWORD HIDDEN
option metric '50'
config interface 'wan2'
option proto 'dhcp'
option ifname 'eth1'
option metric '40'
config interface 'wan3'
option proto 'dhcp'
option metric '20'
Sorry for my english and post size.
[]'s
Renato