That works! Thankyou, I will test it for a few hours and then mark it solved.
1 Like
Can you post the mwan3 configuration? uci export mwan3
There is more delicate way to do what you want.
1 Like
root@Node1-Turris:~# uci export mwan3
package mwan3
config policy 'failover'
list use_member 'wan_m1_w1'
list use_member 'lte_m2_w2'
list use_member 'wg0_m0_w0'
option last_resort 'default'
config globals 'globals'
option mmx_mask '0x3F00'
option logging '1'
option rtmon_interval '10'
option loglevel 'info'
config interface 'wan'
option enabled '1'
list track_ip '8.8.4.4'
list track_ip '8.8.8.8'
list track_ip '208.67.222.222'
list track_ip '208.67.220.220'
option family 'ipv4'
option reliability '2'
option count '1'
option timeout '2'
option interval '5'
option initial_state 'online'
option track_method 'ping'
option size '56'
option max_ttl '60'
option check_quality '0'
option keep_failure_interval '1'
option down '2'
option up '1'
option failure_interval '1'
option recovery_interval '1'
list flush_conntrack 'ifup'
list flush_conntrack 'ifdown'
list flush_conntrack 'connected'
list flush_conntrack 'disconnected'
config rule 'default_rule'
option dest_ip '0.0.0.0/0'
option proto 'all'
option use_policy 'failover'
option sticky '0'
option logging '1'
config member 'wan_m1_w1'
option interface 'wan'
option metric '2'
option weight '2'
config interface 'lte'
option enabled '1'
option initial_state 'online'
option family 'ipv4'
option track_method 'ping'
option reliability '1'
option count '2'
option size '56'
option max_ttl '60'
option check_quality '0'
option up '1'
list track_ip '8.8.8.8'
list track_ip '8.8.4.4'
option timeout '5'
option interval '3600'
option failure_interval '60'
option keep_failure_interval '1'
option recovery_interval '60'
option down '2'
config member 'lte_m2_w2'
option interface 'lte'
option metric '3'
option weight '3'
config interface 'wg0'
option enabled '1'
option initial_state 'online'
option family 'ipv4'
option track_method 'ping'
option reliability '1'
option size '56'
option max_ttl '60'
option check_quality '0'
option timeout '2'
option failure_interval '5'
option recovery_interval '5'
option down '3'
option up '3'
option count '2'
option interval '10'
list flush_conntrack 'ifup'
list flush_conntrack 'ifdown'
list flush_conntrack 'connected'
list flush_conntrack 'disconnected'
config member 'wg0_m0_w0'
option interface 'wg0'
option metric '1'
option weight '1'
1 Like
Start by flushing conntrack on lte interface.
If this doesn't work post the ip -4 ro li ta all ; mwan3 status after wan is back up.
1 Like
changed Flush conntrack on lte interface, disconnected wan, then;
root@Node1-Turris:/etc/config# ip -4 ro li ta all
default via 192.168.2.1 dev eth2 table 1 metric 10
10.0.10.0/24 dev wg0 table 1 proto static scope link metric 5
10.30.19.140/30 dev wwan0 table 1 proto static scope link metric 20
10.73.10.0/24 dev br-lan table 1 proto kernel scope link src 10.73.10.1
10.73.11.0/24 dev wlan2 table 1 proto kernel scope link src 10.73.11.1
185.xxx.xxx.xxx via 10.30.19.142 dev wwan0 table 1 proto static metric 20
192.168.2.0/24 dev eth2 table 1 proto static scope link metric 10
default via 10.30.19.142 dev wwan0 table 2 metric 20
10.0.10.0/24 dev wg0 table 2 proto static scope link metric 5
10.30.19.140/30 dev wwan0 table 2 proto static scope link metric 20
10.73.10.0/24 dev br-lan table 2 proto kernel scope link src 10.73.10.1
10.73.11.0/24 dev wlan2 table 2 proto kernel scope link src 10.73.11.1
185.xxx.xxx.xxx via 10.30.19.142 dev wwan0 table 2 proto static metric 20
192.168.2.0/24 dev eth2 table 2 proto static scope link metric 10
default dev wg0 table 3 scope link metric 5
10.0.10.0/24 dev wg0 table 3 proto static scope link metric 5
10.30.19.140/30 dev wwan0 table 3 proto static scope link metric 20
10.73.10.0/24 dev br-lan table 3 proto kernel scope link src 10.73.10.1
10.73.11.0/24 dev wlan2 table 3 proto kernel scope link src 10.73.11.1
185.xxx.xxx.xxx via 10.30.19.142 dev wwan0 table 3 proto static metric 20
192.168.2.0/24 dev eth2 table 3 proto static scope link metric 10
default via 192.168.2.1 dev eth2 table 201
10.73.11.0/24 dev wlan2 table 201 proto kernel scope link src 10.73.11.1
default via 10.0.10.2 dev wg0 table 202
10.73.11.0/24 dev wlan2 table 202 proto kernel scope link src 10.73.11.1
default via 10.30.19.142 dev wwan0 table 203
10.73.11.0/24 dev wlan2 table 203 proto kernel scope link src 10.73.11.1
default dev wg0 proto static scope link metric 5
default via 192.168.2.1 dev eth2 proto static metric 10
default via 10.30.19.142 dev wwan0 proto static src 10.30.19.141 metric 20
10.0.10.0/24 dev wg0 proto static scope link metric 5
10.30.19.140/30 dev wwan0 proto static scope link metric 20
10.73.10.0/24 dev br-lan proto kernel scope link src 10.73.10.1
10.73.11.0/24 dev wlan2 proto kernel scope link src 10.73.11.1
185.xxx.xxx.xxx via 10.30.19.142 dev wwan0 proto static metric 20
192.168.2.0/24 dev eth2 proto static scope link metric 10
broadcast 10.0.10.0 dev wg0 table local proto kernel scope link src 10.0.10.2
local 10.0.10.2 dev wg0 table local proto kernel scope host src 10.0.10.2
broadcast 10.0.10.255 dev wg0 table local proto kernel scope link src 10.0.10.2
broadcast 10.30.19.140 dev wwan0 table local proto kernel scope link src 10.30.19.141
local 10.30.19.141 dev wwan0 table local proto kernel scope host src 10.30.19.141
broadcast 10.30.19.143 dev wwan0 table local proto kernel scope link src 10.30.19.141
broadcast 10.73.10.0 dev br-lan table local proto kernel scope link src 10.73.10.1
local 10.73.10.1 dev br-lan table local proto kernel scope host src 10.73.10.1
broadcast 10.73.10.255 dev br-lan table local proto kernel scope link src 10.73.10.1
broadcast 10.73.11.0 dev wlan2 table local proto kernel scope link src 10.73.11.1
local 10.73.11.1 dev wlan2 table local proto kernel scope host src 10.73.11.1
broadcast 10.73.11.255 dev wlan2 table local proto kernel scope link src 10.73.11.1
broadcast 127.0.0.0 dev lo table local proto kernel scope link src 127.0.0.1
local 127.0.0.0/8 dev lo table local proto kernel scope host src 127.0.0.1
local 127.0.0.1 dev lo table local proto kernel scope host src 127.0.0.1
broadcast 127.255.255.255 dev lo table local proto kernel scope link src 127.0.0.1
broadcast 192.168.2.0 dev eth2 table local proto kernel scope link src 192.168.2.2
local 192.168.2.2 dev eth2 table local proto kernel scope host src 192.168.2.2
broadcast 192.168.2.255 dev eth2 table local proto kernel scope link src 192.168.2.2
*redacted address 185.xxx.xxx.xxx is my vps public ip
I asked you to reconnect the wan and post the routing table plus mwan3 status.
Apologies with wan reconnected;
root@Node1-Turris:/etc/config# ip -4 ro li ta all
default via 192.168.2.1 dev eth2 table 1 metric 10
10.0.10.0/24 dev wg0 table 1 proto static scope link metric 5
10.30.19.140/30 dev wwan0 table 1 proto static scope link metric 20
10.73.10.0/24 dev br-lan table 1 proto kernel scope link src 10.73.10.1
10.73.11.0/24 dev wlan2 table 1 proto kernel scope link src 10.73.11.1
185.xxx.xxx.xxx via 10.30.19.142 dev wwan0 table 1 proto static metric 20
192.168.2.0/24 dev eth2 table 1 proto static scope link metric 10
default via 10.30.19.142 dev wwan0 table 2 metric 20
10.0.10.0/24 dev wg0 table 2 proto static scope link metric 5
10.30.19.140/30 dev wwan0 table 2 proto static scope link metric 20
10.73.10.0/24 dev br-lan table 2 proto kernel scope link src 10.73.10.1
10.73.11.0/24 dev wlan2 table 2 proto kernel scope link src 10.73.11.1
185.xxx.xxx.xxx via 10.30.19.142 dev wwan0 table 2 proto static metric 20
192.168.2.0/24 dev eth2 table 2 proto static scope link metric 10
default dev wg0 table 3 scope link metric 5
10.0.10.0/24 dev wg0 table 3 proto static scope link metric 5
10.30.19.140/30 dev wwan0 table 3 proto static scope link metric 20
10.73.10.0/24 dev br-lan table 3 proto kernel scope link src 10.73.10.1
10.73.11.0/24 dev wlan2 table 3 proto kernel scope link src 10.73.11.1
185.xxx.xxx.xxx via 10.30.19.142 dev wwan0 table 3 proto static metric 20
192.168.2.0/24 dev eth2 table 3 proto static scope link metric 10
default via 192.168.2.1 dev eth2 table 201
10.73.11.0/24 dev wlan2 table 201 proto kernel scope link src 10.73.11.1
default via 10.0.10.2 dev wg0 table 202
10.73.11.0/24 dev wlan2 table 202 proto kernel scope link src 10.73.11.1
default via 10.30.19.142 dev wwan0 table 203
10.73.11.0/24 dev wlan2 table 203 proto kernel scope link src 10.73.11.1
default dev wg0 proto static scope link metric 5
default via 192.168.2.1 dev eth2 proto static metric 10
default via 10.30.19.142 dev wwan0 proto static src 10.30.19.141 metric 20
10.0.10.0/24 dev wg0 proto static scope link metric 5
10.30.19.140/30 dev wwan0 proto static scope link metric 20
10.73.10.0/24 dev br-lan proto kernel scope link src 10.73.10.1
10.73.11.0/24 dev wlan2 proto kernel scope link src 10.73.11.1
185.xxx.xxx.xxx via 10.30.19.142 dev wwan0 proto static metric 20
192.168.2.0/24 dev eth2 proto static scope link metric 10
broadcast 10.0.10.0 dev wg0 table local proto kernel scope link src 10.0.10.2
local 10.0.10.2 dev wg0 table local proto kernel scope host src 10.0.10.2
broadcast 10.0.10.255 dev wg0 table local proto kernel scope link src 10.0.10.2
broadcast 10.30.19.140 dev wwan0 table local proto kernel scope link src 10.30.19.141
local 10.30.19.141 dev wwan0 table local proto kernel scope host src 10.30.19.141
broadcast 10.30.19.143 dev wwan0 table local proto kernel scope link src 10.30.19.141
broadcast 10.73.10.0 dev br-lan table local proto kernel scope link src 10.73.10.1
local 10.73.10.1 dev br-lan table local proto kernel scope host src 10.73.10.1
broadcast 10.73.10.255 dev br-lan table local proto kernel scope link src 10.73.10.1
broadcast 10.73.11.0 dev wlan2 table local proto kernel scope link src 10.73.11.1
local 10.73.11.1 dev wlan2 table local proto kernel scope host src 10.73.11.1
broadcast 10.73.11.255 dev wlan2 table local proto kernel scope link src 10.73.11.1
broadcast 127.0.0.0 dev lo table local proto kernel scope link src 127.0.0.1
local 127.0.0.0/8 dev lo table local proto kernel scope host src 127.0.0.1
local 127.0.0.1 dev lo table local proto kernel scope host src 127.0.0.1
broadcast 127.255.255.255 dev lo table local proto kernel scope link src 127.0.0.1
broadcast 192.168.2.0 dev eth2 table local proto kernel scope link src 192.168.2.2
local 192.168.2.2 dev eth2 table local proto kernel scope host src 192.168.2.2
broadcast 192.168.2.255 dev eth2 table local proto kernel scope link src 192.168.2.2
and;
root@Node1-Turris:/etc/config# mwan3 status
Interface status:
interface wan is online 00h:01m:43s, uptime 00h:01m:45s and tracking is active
interface lte is online 00h:42m:56s, uptime 00h:43m:00s and tracking is active
interface wg0 is online 00h:00m:00s, uptime 00h:02m:10s and tracking is not enabled
Current ipv4 policies:
failover:
wg0 (100%)
Current ipv6 policies:
failover:
default
Directly connected ipv4 networks:
10.30.19.140/30
127.0.0.0/8
10.73.11.0/24
224.0.0.0/3
185.xxx.xxx.xxx
192.168.2.0/24
10.73.10.0/24
10.0.10.0/24
Directly connected ipv6 networks:
fde1:ef33:cef::/48
fd21:6a97:d2f0:10::/64
fe80::/64
fd21:6a97:d2f0::/64
fde1:ef33:cef::/64
fde1:ef33:cef:4::/64
Active ipv4 user rules:
265 27736 LOG all -- * * 0.0.0.0/0 0.0.0.0/0
265 27736 - failover all -- * * 0.0.0.0/0 0.0.0.0/0
Active ipv6 user rules:
2 272 LOG all * * ::/0 ::/0
2 272 - failover all * * ::/0 ::/0
Try the following.
Add a policy failover2 with members wan_m1_w1 and lte_m2_w2 unreachable last resort.
Then add a rule for vps IP 185... to use failover2 policy.
root@Node1-Turris:/etc/config# ip -4 ro li ta all
default via 192.168.2.1 dev eth2 table 1 metric 10
10.0.10.0/24 dev wg0 table 1 proto static scope link metric 5
10.30.19.140/30 dev wwan0 table 1 proto static scope link metric 20
10.73.10.0/24 dev br-lan table 1 proto kernel scope link src 10.73.10.1
10.73.11.0/24 dev wlan2 table 1 proto kernel scope link src 10.73.11.1
185.x.x.x via 10.30.19.142 dev wwan0 table 1 proto static metric 20
192.168.2.0/24 dev eth2 table 1 proto static scope link metric 10
default via 10.30.19.142 dev wwan0 table 2 metric 20
10.0.10.0/24 dev wg0 table 2 proto static scope link metric 5
10.30.19.140/30 dev wwan0 table 2 proto static scope link metric 20
10.73.10.0/24 dev br-lan table 2 proto kernel scope link src 10.73.10.1
10.73.11.0/24 dev wlan2 table 2 proto kernel scope link src 10.73.11.1
185.x.x.x via 10.30.19.142 dev wwan0 table 2 proto static metric 20
192.168.2.0/24 dev eth2 table 2 proto static scope link metric 10
default dev wg0 table 3 scope link metric 5
10.0.10.0/24 dev wg0 table 3 proto static scope link metric 5
10.30.19.140/30 dev wwan0 table 3 proto static scope link metric 20
10.73.10.0/24 dev br-lan table 3 proto kernel scope link src 10.73.10.1
10.73.11.0/24 dev wlan2 table 3 proto kernel scope link src 10.73.11.1
185.x.x.x via 10.30.19.142 dev wwan0 table 3 proto static metric 20
192.168.2.0/24 dev eth2 table 3 proto static scope link metric 10
default via 192.168.2.1 dev eth2 table 201
10.73.11.0/24 dev wlan2 table 201 proto kernel scope link src 10.73.11.1
default via 10.0.10.2 dev wg0 table 202
10.73.11.0/24 dev wlan2 table 202 proto kernel scope link src 10.73.11.1
default via 10.30.19.142 dev wwan0 table 203
10.73.11.0/24 dev wlan2 table 203 proto kernel scope link src 10.73.11.1
default dev wg0 proto static scope link metric 5
default via 192.168.2.1 dev eth2 proto static metric 10
default via 10.30.19.142 dev wwan0 proto static src 10.30.19.141 metric 20
10.0.10.0/24 dev wg0 proto static scope link metric 5
10.30.19.140/30 dev wwan0 proto static scope link metric 20
10.73.10.0/24 dev br-lan proto kernel scope link src 10.73.10.1
10.73.11.0/24 dev wlan2 proto kernel scope link src 10.73.11.1
185.x.x.x via 10.30.19.142 dev wwan0 proto static metric 20
192.168.2.0/24 dev eth2 proto static scope link metric 10
broadcast 10.0.10.0 dev wg0 table local proto kernel scope link src 10.0.10.2
local 10.0.10.2 dev wg0 table local proto kernel scope host src 10.0.10.2
broadcast 10.0.10.255 dev wg0 table local proto kernel scope link src 10.0.10.2
broadcast 10.30.19.140 dev wwan0 table local proto kernel scope link src 10.30.19.141
local 10.30.19.141 dev wwan0 table local proto kernel scope host src 10.30.19.141
broadcast 10.30.19.143 dev wwan0 table local proto kernel scope link src 10.30.19.141
broadcast 10.73.10.0 dev br-lan table local proto kernel scope link src 10.73.10.1
local 10.73.10.1 dev br-lan table local proto kernel scope host src 10.73.10.1
broadcast 10.73.10.255 dev br-lan table local proto kernel scope link src 10.73.10.1
broadcast 10.73.11.0 dev wlan2 table local proto kernel scope link src 10.73.11.1
local 10.73.11.1 dev wlan2 table local proto kernel scope host src 10.73.11.1
broadcast 10.73.11.255 dev wlan2 table local proto kernel scope link src 10.73.11.1
broadcast 127.0.0.0 dev lo table local proto kernel scope link src 127.0.0.1
local 127.0.0.0/8 dev lo table local proto kernel scope host src 127.0.0.1
local 127.0.0.1 dev lo table local proto kernel scope host src 127.0.0.1
broadcast 127.255.255.255 dev lo table local proto kernel scope link src 127.0.0.1
broadcast 192.168.2.0 dev eth2 table local proto kernel scope link src 192.168.2.2
local 192.168.2.2 dev eth2 table local proto kernel scope host src 192.168.2.2
broadcast 192.168.2.255 dev eth2 table local proto kernel scope link src 192.168.2.2
and
root@Node1-Turris:/etc/config# mwan3 status
Interface status:
interface wan is online 00h:02m:00s, uptime 00h:02m:04s and tracking is active
interface lte is online 00h:05m:11s, uptime 00h:43m:56s and tracking is active
interface wg0 is online 00h:00m:00s, uptime 00h:02m:54s and tracking is not enabled
Current ipv4 policies:
failover:
wg0 (100%)
failover2:
wan (100%)
Current ipv6 policies:
failover:
default
failover2:
unreachable
Directly connected ipv4 networks:
185.x.x.x
192.168.2.0/24
10.73.10.0/24
10.30.19.140/30
10.0.10.0/24
127.0.0.0/8
224.0.0.0/3
10.73.11.0/24
Directly connected ipv6 networks:
fde1:ef33:cef::/64
fe80::/64
fd21:6a97:d2f0::/64
fd21:6a97:d2f0:10::/64
fde1:ef33:cef:4::/64
fde1:ef33:cef::/48
Active ipv4 user rules:
0 0 LOG all -- * * 0.0.0.0/0 185.x.x.x
0 0 - failover2 all -- * * 0.0.0.0/0 185.x.x.x
Active ipv6 user rules:
The lte interface with these settings was still being used with wan back on line
I've had a look at that already but i am running
root@Node1-Turris:/etc/config# uname -r
4.14.171
I did a great deal of reading and researching prior to posting on this forum most of which brought me back to the 5.4 kernel issue but I figured that this was not my actual problem due to running the 4.14 kernel even though the issue seemed to correlate with the ticket you posted.
I will keep messing around and see if I can solve it, failing that I'll implement the hotplug sledge hammer to the lte interface as modemmanager seems rock solid for dealing with the modem.
I massively appreciate your input and time helping out with this, if I get it working before I run out of data I'll post the changes I made, nice 1 
EDIT: one more observation is that I cannot set up tracking on the wg0 interface, even if i manually edit the mwan3 conf file the tracking addresses get deleted when I implement the modified config, hence wg0 being online with tracking inactive, is this the expected behaviour?
1 Like
I don't use mwan3 on vpns, so I cannot be sure about that.
Likely not related to wackejohn's issue, as that was only a 5.4 kernel issue.
The problem is that once you have mwan3 turned on it routes all traffic through the wans rather the wireguard interface.
The option I would recommend would be to have a wireguard link set up for each WAN using the 'tunlink' option in wireguard, add these as interfaces in mwan3, and then add mwan3 traffic rules for the 10.0.10.0/24 ip address range to tell the traffic to use wireguard.
You might also be able to set it up with a single wireguard connection and a rule in mwan3 to route traffic to the 10.0.10.0/24 through the default routing table. However, I have not tested how wireguard handles the wan connection changing.
1 Like
Does wireguard recognise the tunlink option in the config file, I read this from the owrt docs and the tunlink option is applied to gre tunnels from what i can understand, and I'm also unsure how to set up a wireguard link for each wan as i thought that wireguard being a logical interface gets used on whichever interface can connect to the endpoint ip address, (i've probably massively over simplified it in my head).
I will investigate your suggestions and report back, probably not for a couple of days as I'm out of reach of the router.
Yeah, I added it in 1.0.20200513-2, so any version after that should work. Doc needs to be updated to reflect the change.
Your understanding is correct. Normally it will use whichever interface the routing rule specifies. I have my set up where each interface connects to a different wireguard endpoint. If both interfaces need to connect to the same endpoint, the tunlink option probably won't be enough.
If there is only one endpoint you need to connect to, you could try having them originate from different source ports and set up some additional mwan3 rules to make sure that the traffic goes out the right interface.
In order to achieve this ,as I only have the one endpoint which is a vps, would I setup the single wg_iface on the vps to listen on 2 ports, or would I create another wg_iface1 with an almost identical config with the exception of the port used?
You would want to create a second wg_iface with a different source port.
Unfortunately, you will need to create some custom firewall rules, because mwan3 won't handle rules for ip addresses that have a static route, which is added by the wireguard script.
You could add something like this to your /etc/firewall.user (warning, untested code)
#!/bin/sh
. /lib/functions.sh
. /lib/mwan3/mwan3.sh
. /lib/mwan3/common.sh
config_load mwan3
mwan3_init
function wireguard_ipt_rule ()
{
local id interface endpoint sport
interface="$1"
endpoint="$2"
sport="$3"
mwan3_get_iface_id id "$interface"
$IPT4 -I OUTPUT -d "$endpoint" -p udp --sport "$sport" \
-j MARK --set-xmark "$(mwan3_id2mask id MMX_MASK)/$MMX_MASK"
}
wireguard_ipt_rule wan 185.XXX.XXX.XXX 51820
wireguard_ipt_rule wwan 185.XXX.XXX.XXX 51821hello everybody, Ciao from Italy,
in this period of covid19 me and my wife are working from home everydays and both of us are software engineer from different company, but always in teleservice with some costumer / machine all around the world, so backup line is important.
In our house we have two linksys, WRT1200AC (primary) and WRT1900ACS (in another zone),
off course both with OpenWrt 19.07.4 r11208-ce6496d796.
I installed Mwan3 in the primary that have two WANs:
WAN: adsl modem 20/1 Mb
WANB: 4G iphone 5s as backup line
avoiding discussing of the issue of "trust this PC" at any router reboot, still an issue,
I have the same issue of beijjjj, we have always an VPN tunnel open using openwrt.
the behavior is the following:
- when WAN goes down => traffic reroute to WANB (and VPN disconnect and reconnect using WANB). it's OK
- when WAN goes up again => normal traffic reroute back to WAN, but VPN tunnel remain on WANB.
so I am planning to do the script that put in sleep for some second the WANB after WAN come back online.
This will disconnect the VPN tunnel again, that is OK, but when reconnect it will use the WAN.
I will wrote here if it working the script, unluckly I did't find a settings in Mwan3 as default "kill connection" .
thank you for you support, you are great team!
1 Like
ciao,
it works!!! great very much, now I am very happy!
my script:
cat << "EOF" > /etc/hotplug.d/iface/10-WAN
if [ "${ACTION}" = "ifup" -a "${INTERFACE}" = "WAN" ]
then sleep 10
ifdown WANB
sleep 30
ifup WANB
fi
EOF`
target reached, thank you for your support!
1 Like
This topic was automatically closed 10 days after the last reply. New replies are no longer allowed.