Mwan3rtmon error - RTNETLINK answers: File exists

tsouza85 · December 21, 2021, 2:11pm

Hello guys!

logread | grep mwan3

Tue Dec 21 11:06:37 2021 user.warn mwan3rtmon[14488]: failed to add 172.16.0.0/24 dev br-lan scope link  src 172.16.0.1 to table 1 - error: ip: RTNETLINK answers: File exists
Tue Dec 21 11:06:37 2021 user.warn mwan3rtmon[14488]: failed to add 172.16.0.0/24 dev br-lan scope link  src 172.16.0.1 to table 2 - error: ip: RTNETLINK answers: File exists

Does anyone know how to fix issue?

Other infos:

Version: OpenWrt 21.02.1, r16325-88151b8303 (x86_64)

ip route show table 1

default via 189.107.28.1 dev eth2  src 189.107.29.41  metric 10 
172.16.0.0/24 dev br-lan scope link  src 172.16.0.1 
189.107.28.0/23 dev eth2 scope link  metric 10

ip route show table 2

default via 187.16.151.41 dev pppoe-West  metric 20 
172.16.0.0/24 dev br-lan scope link  src 172.16.0.1 
187.16.151.41 dev pppoe-West scope link  src 187.16.153.244

/etc/config/network

config interface 'loopback'
        option device 'lo'
        option proto 'static'
        option ipaddr '127.0.0.1'
        option netmask '255.0.0.0'

config globals 'globals'
        option packet_steering '1'

config device
        option name 'br-lan'
        option type 'bridge'
        option ipv6 '0'
        option mtu '1500'
        list ports 'eth1'

config interface 'lan'
        option device 'br-lan'
        option proto 'static'
        option netmask '255.255.255.0'
        option ipaddr '172.16.0.1'
        option delegate '0'

config device
        option name 'eth0'
        option ipv6 '0'
        option mtu '1500'
        option macaddr 'd4:5d:64:34:82:3c'

config device
        option name 'eth2'
        option macaddr '20:0D:B0:27:A2:E7'
        option ipv6 '0'

config interface 'West'
        option proto 'pppoe'
        option device 'eth0'
        option username '154631'
        option password '444338971'
        option ipv6 '0'
        option peerdns '0'
        option delegate '0'
        option metric '20'

config interface 'Oi'
        option proto 'dhcp'
        option device 'eth2'
        option peerdns '0'
        option delegate '0'
        option metric '10'

/etc/config/mwan3

config rule 'speedtest_west'
        option family 'ipv4'
        option src_ip '172.16.0.138'
        option proto 'all'
        option sticky '0'
        option use_policy 'West_link'
        option dest_ip '0.0.0.0/0'

config rule 'speedtest_oi'
        option family 'ipv4'
        option src_ip '172.16.0.135'
        option proto 'all'
        option sticky '0'
        option use_policy 'Oi_link'
        option dest_ip '0.0.0.0/0'

config rule 'Ping_west'
        option family 'ipv4'
        option src_ip '172.16.0.139'
        option proto 'all'
        option sticky '0'
        option use_policy 'West_link'
        option dest_ip '0.0.0.0/0'

config rule 'Ping_oi'
        option family 'ipv4'
        option src_ip '172.16.0.130'
        option proto 'all'
        option sticky '0'
        option use_policy 'Oi_link'
        option dest_ip '0.0.0.0/0'

config rule 'https'
        option family 'ipv4'
        option proto 'tcp'
        option sticky '1'
        option timeout '60'
        option use_policy 'load_balance'
        option dest_port '443'
        option dest_ip '0.0.0.0/0'
        option src_ip '0.0.0.0/0'

config rule 'default_rule_v4'
        option family 'ipv4'
        option proto 'all'
        option sticky '0'
        option use_policy 'load_balance'
        option dest_ip '0.0.0.0/0'
        option src_ip '0.0.0.0/0'

config globals 'globals'
        option mmx_mask '0x3F00'

config interface 'Oi'
        option initial_state 'online'
        option family 'ipv4'
        list track_ip '8.8.4.4'
        list track_ip '8.8.8.8'
        list track_ip '208.67.222.222'
        list track_ip '208.67.220.220'
        option track_method 'ping'
        option reliability '1'
        option count '1'
        option size '56'
        option max_ttl '60'
        option check_quality '1'
        option failure_latency '75'
        option failure_loss '20'
        option recovery_latency '25'
        option recovery_loss '10'
        option timeout '4'
        option interval '10'
        option failure_interval '5'
        option recovery_interval '5'
        option down '5'
        option up '5'
        option enabled '1'
        list flush_conntrack 'ifup'

config interface 'West'
        option initial_state 'online'
        option family 'ipv4'
        list track_ip '8.8.4.4'
        list track_ip '8.8.8.8'
        list track_ip '208.67.222.222'
        list track_ip '208.67.220.220'
        option track_method 'ping'
        option reliability '1'
        option count '5'
        option size '56'
        option max_ttl '60'
        option check_quality '1'
        option failure_latency '50'
        option failure_loss '20'
        option recovery_latency '25'
        option recovery_loss '10'
        option timeout '4'
        option interval '10'
        option failure_interval '5'
        option recovery_interval '5'
        option down '5'
        option up '5'
        option enabled '1'
        list flush_conntrack 'ifup'

config member 'Oi_member'
        option interface 'Oi'

config member 'West_member'
        option interface 'West'

config policy 'Oi_link'
        list use_member 'Oi_member'
        option last_resort 'unreachable'

config policy 'West_link'
        list use_member 'West_member'
        option last_resort 'unreachable'

config policy 'load_balance'
        list use_member 'Oi_member'
        list use_member 'West_member'
        option last_resort 'unreachable'

mwan3 status

Interface status:
 interface Oi is error (4) and tracking is active
 interface West is error (4) and tracking is active

Current ipv4 policies:
Oi_link:
 Oi (100%)
West_link:
 West (100%)
load_balance:
 West (50%)
 Oi (50%)

Current ipv6 policies:
Oi_link:
 unreachable
West_link:
 unreachable
load_balance:
 unreachable

Directly connected ipv4 networks:
189.107.28.0/23
127.0.0.0/8
172.16.0.0/24
100.93.117.117
187.16.153.244
224.0.0.0/3
187.16.151.41

Directly connected ipv6 networks:
fd7a:115c:a1e0::/48
fe80::/64
fd7a:115c:a1e0:ab12:4843:cd96:625d:7575

Active ipv4 user rules:
  375 24138 - West_link  all  --  *      *       172.16.0.138         0.0.0.0/0            
  375 23802 - Oi_link  all  --  *      *       172.16.0.135         0.0.0.0/0            
 4936  340K - West_link  all  --  *      *       172.16.0.139         0.0.0.0/0            
 4935  340K - Oi_link  all  --  *      *       172.16.0.130         0.0.0.0/0            
 3994  260K S https  tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            multiport dports 443 
 5085  703K - load_balance  all  --  *      *       0.0.0.0/0            0.0.0.0/0            

Active ipv6 user rules:

tuxbox · February 28, 2022, 3:34am

Hi tsouza85,

I had the same problem as you with mwan3 after upgrading to the latest busybox on your same version of openwrt but on tp-link ac2600 router (arm_cortex-a15_neon-vfpv4).

BTW, I miraculously found your "bug post" by googling "mwan3rtmon failed to add"

All my wan load balancing SEEMED to work except I got this on
mwan3 restart:

ip: invalid argument 'blackhole' to 'ip'
ip: invalid argument 'unreachable' to 'ip'
ip: invalid argument 'blackhole' to 'ip'
ip: invalid argument 'unreachable' to 'ip'
ip: invalid argument 'unreachable' to 'ip'
ip: invalid argument 'unreachable' to 'ip'
ip: invalid argument 'unreachable' to 'ip'
ip: invalid argument 'unreachable' to 'ip'

When I check log with logread -f | grep mwan3 I got:

Sun Feb 27 17:16:30 2022 user.warn mwan3rtmon[27440]: failed to add 192.168.96.0/22 dev br-lan scope link  src 192.168.99.1 to table 1 - error: ip: RTNETLINK answers: File exists
Sun Feb 27 17:16:30 2022 user.warn mwan3rtmon[27440]: failed to add 192.168.96.0/22 dev br-lan scope link  src 192.168.99.1 to table 2 - error: ip: RTNETLINK answers: File exists
Sun Feb 27 17:16:30 2022 user.warn mwan3rtmon[27440]: failed to add 192.168.96.0/22 dev br-lan scope link  src 192.168.99.1 to table 3 - error: ip: RTNETLINK answers: File exists
Sun Feb 27 17:16:30 2022 user.warn mwan3rtmon[27440]: failed to add 192.168.96.0/22 dev br-lan scope link  src 192.168.99.1 to table 4 - error: ip: RTNETLINK answers: File exists

On mwan3 status or mwan3 interfaces I got this:

Interface status:
 interface wan is error (4) and tracking is active
 interface wan_e1 is error (4) and tracking is active
 interface wan_e2 is error (4) and tracking is active
 interface wan_e3 is error (4) and tracking is active

I first thought of restoring old busybox from /rom/bin/busybox but I just did a force reinstall of ip-full like this:

opkg --force-reinstall install ip-full

and problem gone!

logread -f | grep mwan3

root@Wifreenitum_n10-GW64fc:~# logread -f | grep mwan3
Sun Feb 27 21:28:01 2022 user.notice mwan3track[544]: Stopping mwan3track for interface "wan_e2". Status was "online"
Sun Feb 27 21:28:01 2022 user.notice mwan3track[543]: Stopping mwan3track for interface "wan_e1". Status was "online"
Sun Feb 27 21:28:01 2022 user.notice mwan3track[545]: Stopping mwan3track for interface "wan_e3". Status was "online"
Sun Feb 27 21:28:01 2022 user.notice mwan3track[542]: Stopping mwan3track for interface "wan". Status was "online"
Sun Feb 27 21:28:02 2022 user.notice mwan3-hotplug[12257]: Execute ifup event on interface wan (eth0.2)
Sun Feb 27 21:28:02 2022 user.notice mwan3-hotplug[12266]: Execute ifup event on interface wan_e1 (eth0.3)
Sun Feb 27 21:28:02 2022 user.notice mwan3-hotplug[12277]: Execute ifup event on interface wan_e2 (eth0.4)
Sun Feb 27 21:28:03 2022 user.notice mwan3-hotplug[12292]: Execute ifup event on interface wan_e3 (eth0.5)
Sun Feb 27 21:28:05 2022 user.notice mwan3track[12994]: Interface wan (eth0.2) is online
Sun Feb 27 21:28:05 2022 user.notice mwan3track[12995]: Interface wan_e1 (eth0.3) is online
Sun Feb 27 21:28:05 2022 user.notice mwan3track[12997]: Interface wan_e3 (eth0.5) is online
Sun Feb 27 21:28:05 2022 user.notice mwan3track[12996]: Interface wan_e2 (eth0.4) is online

/etc/config/mwan3

config globals 'globals'
	option mmx_mask '0x3F00'
	option rtmon_interval '5'

config interface 'wan'
	option enabled '1'
	list track_ip '8.8.8.8'
#	list track_ip '8.8.4.4'
	list track_ip '208.67.222.222'
	list track_ip '208.67.220.220'
	option family 'ipv4'
	option count '1'
	option timeout '2'
	option interval '5'
	option down '3'
	option up '8'
	option initial_state 'online'
	option track_method 'ping'
	option reliability '1'
	option size '56'
	option max_ttl '60'
	option check_quality '0'
	option failure_interval '5'
	option recovery_interval '5'

config interface 'wan_e1'
	option enabled '1'
	list track_ip '8.8.8.8'
#	list track_ip '8.8.4.4'
	list track_ip '208.67.222.222'
	list track_ip '208.67.220.220'
	option family 'ipv4'
	option count '1'
	option timeout '2'
	option interval '5'
	option down '3'
	option up '8'
	option initial_state 'online'
	option track_method 'ping'
	option reliability '1'
	option size '56'
	option max_ttl '60'
	option check_quality '0'
	option failure_interval '5'
	option recovery_interval '5'

config interface 'wan_e2'
	option enabled '1'
	list track_ip '8.8.8.8'
#	list track_ip '8.8.4.4'
	list track_ip '208.67.222.222'
	list track_ip '208.67.220.220'
	option family 'ipv4'
	option count '1'
	option timeout '2'
	option interval '5'
	option down '3'
	option up '8'
	option initial_state 'online'
	option track_method 'ping'
	option reliability '1'
	option size '56'
	option max_ttl '60'
	option check_quality '0'
	option failure_interval '5'
	option recovery_interval '5'

config interface 'wan_e3'
	option enabled '1'
	list track_ip '8.8.8.8'
#	list track_ip '8.8.4.4'
	list track_ip '208.67.222.222'
	list track_ip '208.67.220.220'
	option family 'ipv4'
	option reliability '1'
	option count '1'
	option timeout '2'
	option interval '5'
	option down '3'
	option up '8'
	option initial_state 'online'
	option track_method 'ping'
	option size '56'
	option max_ttl '60'
	option check_quality '0'
	option failure_interval '5'
	option recovery_interval '5'

config interface 'wan6'
	option enabled '0'
	list track_ip '2001:4860:4860::8844'
	list track_ip '2001:4860:4860::8888'
	list track_ip '2620:0:ccd::2'
	list track_ip '2620:0:ccc::2'
	option family 'ipv6'
	option reliability '0'
	option count '1'
	option timeout '2'
	option interval '5'
	option down '3'
	option up '8'

config interface 'wan6_e1'
	option enabled '0'
	list track_ip '2001:4860:4860::8844'
	list track_ip '2001:4860:4860::8888'
	list track_ip '2620:0:ccd::2'
	list track_ip '2620:0:ccc::2'
	option family 'ipv6'
	option reliability '0'
	option count '1'
	option timeout '2'
	option interval '5'
	option down '3'
	option up '8'

config interface 'wan6_e2'
	option enabled '0'
	list track_ip '2001:4860:4860::8844'
	list track_ip '2001:4860:4860::8888'
	list track_ip '2620:0:ccd::2'
	list track_ip '2620:0:ccc::2'
	option family 'ipv6'
	option reliability '1'
	option count '1'
	option timeout '2'
	option interval '5'
	option down '3'
	option up '8'

config interface 'wan6_e3'
	option enabled '0'
	list track_ip '2001:4860:4860::8844'
	list track_ip '2001:4860:4860::8888'
	list track_ip '2620:0:ccd::2'
	list track_ip '2620:0:ccc::2'
	option family 'ipv6'
	option reliability '1'
	option count '1'
	option timeout '2'
	option interval '5'
	option down '3'
	option up '8'

config member '3wan1_m1_w4'
	option interface 'wan'
	option metric '1'
	option weight '4'

config member '3wan2_m1_w3'
	option interface 'wan_e1'
	option metric '1'
	option weight '3'

config member '3wan3_m1_w2'
	option interface 'wan_e2'
	option metric '1'
	option weight '2'

config member '3wan1_6_m1_w4'
	option interface 'wan6'
	option metric '1'
	option weight '4'

config member '3wan1_6_m1_w3'
	option interface 'wan6_e1'
	option metric '1'
	option weight '3'

config member '3wan1_6_m1_w2'
	option interface 'wan6_e2'
	option metric '1'
	option weight '2'

config member '4wan1_m1_w5'
	option interface 'wan'
	option metric '1'
	option weight '5'

config member '4wan2_m1_w4'
	option interface 'wan_e1'
	option metric '1'
	option weight '4'

config member '4wan3_m1_w3'
	option interface 'wan_e2'
	option metric '1'
	option weight '3'

config member '4wan4_m1_w2'
	option interface 'wan_e3'
	option metric '1'
	option weight '2'

config member '4wan1_6_m1_w5'
	option interface 'wan6'
	option metric '1'
	option weight '5'

config member '4wan2_6_m1_w4'
	option interface 'wan6_e1'
	option metric '1'
	option weight '4'

config member '4wan3_6_m1_w3'
	option interface 'wan6_e2'
	option metric '1'
	option weight '3'

config member '4wan4_6_m1_w2'
	option interface 'wan6_e3'
	option metric '1'
	option weight '2'

config member '2wan1_m1_w3'
	option interface 'wan'
	option metric '1'
	option weight '3'

config member '2wan1_m2_w3'
	option interface 'wan'
	option metric '2'
	option weight '3'

config member '2wan2_m1_w2'
	option interface 'wan_e1'
	option metric '1'
	option weight '2'

config member '2wan2_m2_w2'
	option interface 'wan_e1'
	option metric '2'
	option weight '2'

config member '2wan1_6_m1_w3'
	option interface 'wan6'
	option metric '1'
	option weight '3'

config member '2wan1_6_m2_w3'
	option interface 'wan6'
	option metric '2'
	option weight '3'

config member '2wan2_6_m1_w2'
	option interface 'wan6_e1'
	option metric '1'
	option weight '2'

config member '2wan2_6_m2_w2'
	option interface 'wan6_e1'
	option metric '2'
	option weight '2'

config member '2wan1_13_m1_w3'
	option interface 'wan'
	option metric '1'
	option weight '3'

config member '2wan1_13_m2_w3'
	option interface 'wan'
	option metric '2'
	option weight '3'

config member '2wan3_13_m1_w2'
	option interface 'wan_e2'
	option metric '1'
	option weight '2'

config member '2wan3_13_m2_w2'
	option interface 'wan_e2'
	option metric '2'
	option weight '2'

config member '2wan1_13_6_m1_w3'
	option interface 'wan6'
	option metric '1'
	option weight '3'

config member '2wan1_13_6_m2_w3'
	option interface 'wan6'
	option metric '2'
	option weight '3'

config member '2wan3_13_6_m1_w2'
	option interface 'wan6_e2'
	option metric '1'
	option weight '2'

config member '2wan3_13_6_m2_w2'
	option interface 'wan6_e2'
	option metric '2'
	option weight '2'

config member '2wan2_23_m1_w3'
	option interface 'wan_e1'
	option metric '1'
	option weight '3'

config member '2wan2_23_m2_w3'
	option interface 'wan_e1'
	option metric '2'
	option weight '3'

config member '2wan3_23_m1_w2'
	option interface 'wan_e2'
	option metric '1'
	option weight '2'

config member '2wan3_23_m2_w2'
	option interface 'wan_e2'
	option metric '2'
	option weight '2'

config member '2wan2_23_6_m1_w3'
	option interface 'wan6_e1'
	option metric '1'
	option weight '3'

config member '2wan2_23_6_m2_w3'
	option interface 'wan6_e1'
	option metric '2'
	option weight '3'

config member '2wan3_23_6_m1_w2'
	option interface 'wan6_e2'
	option metric '1'
	option weight '2'

config member '2wan3_23_6_m2_w2'
	option interface 'wan6_e2'
	option metric '2'
	option weight '2'

config policy 'wan_only'
	list use_member 'wan1_m1_w3'
	list use_member 'wan1_6_m1_w3'

config policy 'wan2_only'
	list use_member 'wan2_m1_w2'
	list use_member 'wan2_6_m1_w2'

config policy 'wan3_only'
	list use_member '4wan3_m1_w3'
	list use_member '4wan3_6_m1_w3'

config policy 'wan4_only'
	list use_member '4wan4_m1_w4'
	list use_member '4wan4_6_m1_w4'

config policy 'balanced2wan'
	list use_member '2wan1_m1_w3'
	list use_member '2wan2_m1_w2'
	list use_member '2wan1_6_m1_w3'
	list use_member '2wan2_6_m1_w2'

config policy 'balanced2wan13'
	list use_member '2wan1_13_m1_w3'
	list use_member '2wan3_13_m1_w2'
	list use_member '2wan1_13_6_m1_w3'
	list use_member '2wan3_13_6_m1_w2'

config policy 'balanced2wan23'
	list use_member '2wan2_23_m1_w3'
	list use_member '2wan3_23_m1_w2'
	list use_member '2wan2_23_6_m1_w3'
	list use_member '2wan3_23_6_m1_w2'

config policy 'balanced3wan'
	list use_member '3wan1_m1_w4'
	list use_member '3wan2_m1_w3'
	list use_member '3wan3_m1_w2'
	list use_member '3wan1_6_m1_w4'
	list use_member '3wan2_6_m1_w3'
	list use_member '3wan3_6_m1_w2'

config policy 'balanced4wan'
	list use_member '4wan1_m1_w5'
	list use_member '4wan2_m1_w4'
	list use_member '4wan3_m1_w3'
	list use_member '4wan4_m1_w2'
	list use_member '4wan1_6_m1_w5'
	list use_member '4wan2_6_m1_w4'
	list use_member '4wan3_6_m1_w3'
	list use_member '4wan4_6_m1_w2'

config policy 'wan_wan2'
	list use_member '2wan1_m1_w3'
	list use_member '2wan2_m2_w2'
	list use_member '2wan1_6_m1_w3'
	list use_member '2wan2_6_m2_w2'

config policy 'wan2_wan'
	list use_member '2wan1_m2_w3'
	list use_member '2wan2_m1_w2'
	list use_member '2wan1_6_m2_w3'
	list use_member '2wan2_6_m1_w2'

config rule 'https'
	option sticky '1'
	option dest_port '443'
	option proto 'tcp'
	option use_policy 'balanced4wan'
#added
	option family 'ipv4'

config rule 'default_rule_v4'
	option dest_ip '0.0.0.0/0'
	option use_policy 'balanced4wan'
	option family 'ipv4'

#config rule 'default_rule_v6'
#	option dest_ip '::/0'
#	option use_policy 'balanced4wan'
#	option family 'ipv6'

ip route show table 1

default via 192.168.1.1 dev eth0.2 proto static src 192.168.1.2 metric 10 
192.168.1.0/24 dev eth0.2 proto static scope link metric 10 
192.168.96.0/22 dev br-lan proto kernel scope link src 192.168.99.1

ip route show table 2

default via 192.168.2.1 dev eth0.3 proto static src 192.168.2.64 metric 20 
192.168.2.0/24 dev eth0.3 proto static scope link metric 20 
192.168.96.0/22 dev br-lan proto kernel scope link src 192.168.99.1

ip route show table 3

default via 192.168.3.1 dev eth0.4 proto static src 192.168.3.2 metric 30 
192.168.3.0/24 dev eth0.4 proto static scope link metric 30 
192.168.96.0/22 dev br-lan proto kernel scope link src 192.168.99.1

ip route show table 4

default via 192.168.4.1 dev eth0.5 proto static src 192.168.4.2 metric 40 
192.168.4.0/24 dev eth0.5 proto static scope link metric 40 
192.168.96.0/22 dev br-lan proto kernel scope link src 192.168.99.1

Khurram · March 2, 2022, 11:17am

I too was getting all sorts of errors from "ip: invalid argument ..." to "mwan3rtmon: failed to add ...". This did the trick. Thanks

tuxbox · March 3, 2022, 5:19pm

you're welcome

cahyo · September 9, 2022, 4:57am

nice work like a champ