Mwan3 router itself and stunnel

Installing and Using OpenWrt
#1

hello, I have problems implementing mwan3 and stunnel so that it uses the interface that I need it to use only, I need the connection that stunnel makes to be for the wanb only but having a higher metric, I think it is directed by the default with a metric minor, the ipset had to be set to dhcp and not by dnsmasq.conf since the ipset list was not reflected, any help is appreciated.

/etc/config/dhcp

config dnsmasq
	option domainneeded '1'
	option localise_queries '1'
	option rebind_protection '1'
	option rebind_localhost '1'
	option local '/lan/'
	option domain 'lan'
	option expandhosts '1'
	option authoritative '1'
        list ipset '/.youtube.com/youtube'
	option readethers '1'
	option leasefile '/tmp/dhcp.leases'
	option resolvfile '/tmp/resolv.conf.auto'
	option localservice '1'

config dhcp 'lan'
	option interface 'lan'
	option start '100'
	option limit '150'
	option leasetime '12h'
	option dhcpv6 'server'
	option ra 'server'

config dhcp 'wan'
	option interface 'wan'
	option ignore '1'

config odhcpd 'odhcpd'
	option maindhcp '0'
	option leasefile '/tmp/hosts/odhcpd'
	option leasetrigger '/usr/sbin/odhcpd-update'
	option loglevel '4'

config dhcp 'lan2'
	option start '100'
	option leasetime '12h'
	option limit '150'
	option interface 'lan2'
	option ra 'server'
	option dhcpv6 'server'
	option ra_management '1'
	list dhcp_option '192.168.2.1'

config dhcp 'guest'
	option start '100'
	option leasetime '12h'
	option limit '150'
	option interface 'guest'

config dhcp 'external'
	option start '100'
	option leasetime '12h'
	option limit '150'
	option interface 'external'

/etc/config/firewall

config defaults
	option syn_flood '1'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'

config zone
	option name 'lan'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'
	option network 'lan'
	option masq '1'
	option masq_src '192.168.9.0/24'

config zone
	option name 'wan'
	list network 'wan'
	list network 'wan6'
	option input 'REJECT'
	option output 'ACCEPT'
	option forward 'REJECT'
	option masq '1'
	option mtu_fix '1'

config forwarding
	option src 'lan'
	option dest 'wan'

config rule
	option name 'Allow-DHCP-Renew'
	option src 'wan'
	option proto 'udp'
	option dest_port '68'
	option target 'ACCEPT'
	option family 'ipv4'

config rule
	option name 'Allow-Ping'
	option src 'wan'
	option proto 'icmp'
	option icmp_type 'echo-request'
	option family 'ipv4'
	option target 'ACCEPT'

config rule
	option name 'Allow-IGMP'
	option src 'wan'
	option proto 'igmp'
	option family 'ipv4'
	option target 'ACCEPT'

config rule
	option name 'Allow-DHCPv6'
	option src 'wan'
	option proto 'udp'
	option src_ip 'fc00::/6'
	option dest_ip 'fc00::/6'
	option dest_port '546'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-MLD'
	option src 'wan'
	option proto 'icmp'
	option src_ip 'fe80::/10'
	list icmp_type '130/0'
	list icmp_type '131/0'
	list icmp_type '132/0'
	list icmp_type '143/0'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-ICMPv6-Input'
	option src 'wan'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	list icmp_type 'router-solicitation'
	list icmp_type 'neighbour-solicitation'
	list icmp_type 'router-advertisement'
	list icmp_type 'neighbour-advertisement'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-ICMPv6-Forward'
	option src 'wan'
	option dest '*'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-IPSec-ESP'
	option src 'wan'
	option dest 'lan'
	option proto 'esp'
	option target 'ACCEPT'

config rule
	option name 'Allow-ISAKMP'
	option src 'wan'
	option dest 'lan'
	option dest_port '500'
	option proto 'udp'
	option target 'ACCEPT'

config include
	option path '/etc/firewall.user'

config zone
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'
	option network 'external'
	option name 'external'

config zone
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'
	option name 'guest'
	option network 'guest'

config forwarding
	option dest 'wan'
	option src 'guest'

config zone
	option forward 'REJECT'
	option output 'ACCEPT'
	option name 'wanb'
	option masq '1'
	option mtu_fix '1'
	option network 'wanb'
	option input 'REJECT'

config ipset
        option enabled '1'
        option name 'youtube'
        option match 'ip'
        option storage 'hash'

ipset --list youtube

Name: youtube
Type: hash:ip
Revision: 4
Header: family inet hashsize 1024 maxelem 65536
Size in memory: 96
References: 1
Number of entries: 1
Members:
172.217.30.238

ipset list

Name: youtube
Type: hash:ip
Revision: 4
Header: family inet hashsize 1024 maxelem 65536
Size in memory: 96
References: 1
Number of entries: 1
Members:
172.217.30.238

Name: mwan3_connected_v4
Type: hash:net
Revision: 6
Header: family inet hashsize 1024 maxelem 65536
Size in memory: 2216
References: 1
Number of entries: 34
Members:
10.8.0.5
192.168.2.0
192.168.9.2
192.168.3.1
192.168.1.1
127.255.255.255
190.*.*.*
10.8.0.1
192.168.9.1
192.168.9.0
192.168.1.255
127.0.0.1
127.0.0.0
192.168.9.3
192.168.2.0/24
192.168.2.255
192.168.30.0/24
190.*.*.*/24
192.168.1.0
192.168.1.0/24
192.168.9.4
186.*.*.*
224.0.0.0/3
190.*.*.255
192.168.9.0/24
192.168.2.1
127.0.0.0/8
192.168.3.0
192.168.9.255
10.8.0.6
190.*.*.0
192.168.3.255
192.168.3.0/24
181.*.*.*

Name: mwan3_connected_v6
Type: hash:net
Revision: 6
Header: family inet6 hashsize 1024 maxelem 65536
Size in memory: 1404
References: 1
Number of entries: 3
Members:

Name: mwan3_dynamic_v4
Type: hash:net
Revision: 6
Header: family inet hashsize 1024 maxelem 65536
Size in memory: 312
References: 1
Number of entries: 0
Members:

Name: mwan3_dynamic_v6
Type: hash:net
Revision: 6
Header: family inet6 hashsize 1024 maxelem 65536
Size in memory: 1092
References: 1
Number of entries: 0
Members:

Name: mwan3_custom_v4
Type: hash:net
Revision: 6
Header: family inet hashsize 1024 maxelem 65536
Size in memory: 312
References: 1
Number of entries: 0
Members:

Name: mwan3_custom_v6
Type: hash:net
Revision: 6
Header: family inet6 hashsize 1024 maxelem 65536
Size in memory: 1092
References: 1
Number of entries: 0
Members:

Name: mwan3_sticky_v4_youtube
Type: hash:ip,mark
Revision: 2
Header: family inet markmask 0x00003f00 hashsize 1024 maxelem 65536 timeout 600
Size in memory: 60
References: 1
Number of entries: 0
Members:

Name: mwan3_sticky_v6_youtube
Type: hash:ip,mark
Revision: 2
Header: family inet6 markmask 0x00003f00 hashsize 1024 maxelem 65536 timeout 600
Size in memory: 72
References: 1
Number of entries: 0
Members:

Name: mwan3_sticky_v4_https
Type: hash:ip,mark
Revision: 2
Header: family inet markmask 0x00003f00 hashsize 1024 maxelem 65536 timeout 600
Size in memory: 348
References: 1
Number of entries: 3
Members:

Name: mwan3_sticky_v6_https
Type: hash:ip,mark
Revision: 2
Header: family inet6 markmask 0x00003f00 hashsize 1024 maxelem 65536 timeout 600
Size in memory: 72
References: 1
Number of entries: 0
Members:

Name: mwan3_connected
Type: list:set
Revision: 3
Header: size 8
Size in memory: 184
References: 4
Number of entries: 6
Members:
mwan3_connected_v4
mwan3_connected_v6
mwan3_dynamic_v4
mwan3_dynamic_v6
mwan3_custom_v4
mwan3_custom_v6

Name: mwan3_sticky_youtube
Type: list:set
Revision: 3
Header: size 8
Size in memory: 88
References: 5
Number of entries: 2
Members:
mwan3_sticky_v4_youtube
mwan3_sticky_v6_youtube

Name: mwan3_sticky_https
Type: list:set
Revision: 3
Header: size 8
Size in memory: 88
References: 5
Number of entries: 2
Members:
mwan3_sticky_v4_https
mwan3_sticky_v6_https

/etc/config/mwan3

config rule 'youtube'
        option proto 'tcp'
        option dest_port '80,443'
        option sticky '1'
        option ipset 'youtube'
        option use_policy 'wanb_only'

config rule 'https'
	option dest_port '443'
	option proto 'tcp'
	option use_policy 'wan_wanb'
	option sticky '1'

config rule 'default_rule'
	option dest_ip '0.0.0.0/0'
	option proto 'all'
	option sticky '0'
	option use_policy 'wan_wanb'

config globals 'globals'
	option mmx_mask '0x3F00'
	option rtmon_interval '5'

config interface 'wan'
	option enabled '1'
	list track_ip '8.8.4.4'
	list track_ip '8.8.8.8'
	list track_ip '208.67.222.222'
	list track_ip '208.67.220.220'
	option family 'ipv4'
	option reliability '2'
	option count '1'
	option timeout '2'
	option failure_latency '1000'
	option recovery_latency '500'
	option failure_loss '20'
	option recovery_loss '5'
	option interval '5'
	option down '3'
	option up '8'

config interface 'wan6'
	option enabled '0'
	list track_ip '2001:4860:4860::8844'
	list track_ip '2001:4860:4860::8888'
	list track_ip '2620:0:ccd::2'
	list track_ip '2620:0:ccc::2'
	option family 'ipv6'
	option reliability '2'
	option count '1'
	option timeout '2'
	option interval '5'
	option down '3'
	option up '8'

config interface 'wanb'
	list track_ip '8.8.4.4'
	list track_ip '8.8.8.8'
	list track_ip '208.67.222.222'
	list track_ip '208.67.220.220'
	option family 'ipv4'
	option reliability '1'
	option count '1'
	option timeout '2'
	option interval '5'
	option down '3'
	option up '8'
	option enabled '1'
	option initial_state 'online'
	option track_method 'ping'
	option size '56'
	option max_ttl '60'
	option check_quality '0'
	option failure_interval '5'
	option recovery_interval '5'

config interface 'wanb6'
	option enabled '0'
	list track_ip '2001:4860:4860::8844'
	list track_ip '2001:4860:4860::8888'
	list track_ip '2620:0:ccd::2'
	list track_ip '2620:0:ccc::2'
	option family 'ipv6'
	option reliability '1'
	option count '1'
	option timeout '2'
	option interval '5'
	option down '3'
	option up '8'

config member 'wan_m1_w3'
	option interface 'wan'
	option metric '1'
	option weight '3'

config member 'wan_m2_w3'
	option interface 'wan'
	option metric '2'
	option weight '3'

config member 'wanb_m1_w2'
	option interface 'wanb'
	option metric '1'
	option weight '2'

config member 'wanb_m2_w2'
	option interface 'wanb'
	option metric '2'
	option weight '2'

config member 'wan6_m1_w3'
	option interface 'wan6'
	option metric '1'
	option weight '3'

config member 'wan6_m2_w3'
	option interface 'wan6'
	option metric '2'
	option weight '3'

config member 'wanb6_m1_w2'
	option interface 'wanb6'
	option metric '1'
	option weight '2'

config member 'wanb6_m2_w2'
	option interface 'wanb6'
	option metric '2'
	option weight '2'

config policy 'wan_only'
	list use_member 'wan_m1_w3'
	list use_member 'wan6_m1_w3'

config policy 'wanb_only'
	list use_member 'wanb_m1_w2'
	list use_member 'wanb6_m1_w2'

config policy 'balanced'
	list use_member 'wan_m1_w3'
	list use_member 'wanb_m1_w2'
	list use_member 'wan6_m1_w3'
	list use_member 'wanb6_m1_w2'

config policy 'wan_wanb'
	list use_member 'wan_m1_w3'
	list use_member 'wanb_m2_w2'
	list use_member 'wan6_m1_w3'
	list use_member 'wanb6_m2_w2'

config policy 'wanb_wan'
	list use_member 'wan_m2_w3'
	list use_member 'wanb_m1_w2'
	list use_member 'wan6_m2_w3'
	list use_member 'wanb6_m1_w2'

mwan3 status

Interface status:
 interface wan is online and tracking is active
 interface wan6 is offline and tracking is down
 interface wanb is online and tracking is active
 interface wanb6 is offline and tracking is down

Current ipv4 policies:
balanced:
 wanb (40%)
 wan (60%)
wan_only:
 wan (100%)
wan_wanb:
 wan (100%)
wanb_only:
 wanb (100%)
wanb_wan:
 wanb (100%)

Current ipv6 policies:
balanced:
 unreachable
wan_only:
 unreachable
wan_wanb:
 unreachable
wanb_only:
 unreachable
wanb_wan:
 unreachable

Directly connected ipv4 networks:
10.8.0.5
192.168.2.0
192.168.9.2
192.168.3.1
192.168.1.1
127.255.255.255
190.*.*.*
10.8.0.1
192.168.9.1
192.168.9.0
192.168.1.255
127.0.0.1
127.0.0.0
192.168.9.3
192.168.2.0/24
192.168.2.255
192.168.30.0/24
190.*.*.0/24
192.168.1.0
192.168.1.0/24
192.168.9.4
186.*.*.*
224.0.0.0/3
190.*.*.255
192.168.9.0/24
192.168.2.1
127.0.0.0/8
192.168.3.0
192.168.9.255
10.8.0.6
190.*.*.0
192.168.3.255
192.168.3.0/24
181.*.*.*

Directly connected ipv6 networks:


Active ipv4 user rules:
    0     0 S youtube  tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            match-set youtube dst multiport sports 0:65535 multiport dports 80,443 
   33  1912 S https  tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            multiport sports 0:65535 multiport dports 443 
  981 88865 - wan_wanb  all  --  *      *       0.0.0.0/0            0.0.0.0/0            

Active ipv6 user rules:
    0     0 - wan_only  udp      *      *       ::/0                 ::/0                 multiport sports 0:65535 multiport dports 51820 
    0     0 S https  tcp      *      *       ::/0                 ::/0                 multiport sports 0:65535 multiport dports 443 
  127 10388 - wan_wanb  all      *      *       ::/0                 ::/0

ip route

default via 190.*.*.1 dev eth0.2 proto static src 190.*.*.* metric 10 
default via 181.*.*.* dev pppoe-wanb proto static metric 20 
10.8.0.1 via 10.8.0.5 dev tun1 
10.8.0.5 dev tun1 proto kernel scope link src 10.8.0.6 
181.*.*.* dev pppoe-wanb proto kernel scope link src 186.*.*.* 
190.*.*.0/24 dev eth0.2 proto static scope link metric 10 
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 
192.168.2.0/24 dev eth0.3 proto kernel scope link src 192.168.2.1 
192.168.3.0/24 dev br-guest proto kernel scope link src 192.168.3.1 
192.168.30.0/24 via 10.8.0.5 dev tun1

ip rule

0:	from all lookup local 
1001:	from all iif eth0.2 lookup 1 
1003:	from all iif pppoe-wanb lookup 3 
2001:	from all fwmark 0x100/0x3f00 lookup 1 
2003:	from all fwmark 0x300/0x3f00 lookup 3 
2061:	from all fwmark 0x3d00/0x3f00 blackhole
2062:	from all fwmark 0x3e00/0x3f00 unreachable
32766:	from all lookup main 
32767:	from all lookup default

iptables

-P PREROUTING ACCEPT
-P INPUT ACCEPT
-P FORWARD ACCEPT
-P OUTPUT ACCEPT
-P POSTROUTING ACCEPT
-N mwan3_connected
-N mwan3_hook
-N mwan3_iface_in_wan
-N mwan3_iface_in_wanb
-N mwan3_ifaces_in
-N mwan3_policy_balanced
-N mwan3_policy_wan_only
-N mwan3_policy_wan_wanb
-N mwan3_policy_wanb_only
-N mwan3_policy_wanb_wan
-N mwan3_rule_https
-N mwan3_rule_youtube
-N mwan3_rules
-A PREROUTING -j mwan3_hook
-A FORWARD -o eth0.2 -p tcp -m tcp --tcp-flags SYN,RST SYN -m comment --comment "!fw3: Zone wan MTU fixing" -j TCPMSS --clamp-mss-to-pmtu
-A FORWARD -o pppoe-wanb -p tcp -m tcp --tcp-flags SYN,RST SYN -m comment --comment "!fw3: Zone wanb MTU fixing" -j TCPMSS --clamp-mss-to-pmtu
-A FORWARD -o tun0 -p tcp -m tcp --tcp-flags SYN,RST SYN -m comment --comment "!fw3: Zone zona_vpn0 MTU fixing" -j TCPMSS --clamp-mss-to-pmtu
-A FORWARD -o tun1 -p tcp -m tcp --tcp-flags SYN,RST SYN -m comment --comment "!fw3: Zone zona_vpn1 MTU fixing" -j TCPMSS --clamp-mss-to-pmtu
-A OUTPUT -j mwan3_hook
-A mwan3_connected -m set --match-set mwan3_connected dst -j MARK --set-xmark 0x3f00/0x3f00
-A mwan3_hook -j CONNMARK --restore-mark --nfmask 0x3f00 --ctmask 0x3f00
-A mwan3_hook -m mark --mark 0x0/0x3f00 -j mwan3_ifaces_in
-A mwan3_hook -m mark --mark 0x0/0x3f00 -j mwan3_connected
-A mwan3_hook -m mark --mark 0x0/0x3f00 -j mwan3_rules
-A mwan3_hook -j CONNMARK --save-mark --nfmask 0x3f00 --ctmask 0x3f00
-A mwan3_hook -m mark ! --mark 0x3f00/0x3f00 -j mwan3_connected
-A mwan3_iface_in_wan -i eth0.2 -m set --match-set mwan3_connected src -m mark --mark 0x0/0x3f00 -m comment --comment default -j MARK --set-xmark 0x3f00/0x3f00
-A mwan3_iface_in_wan -i eth0.2 -m mark --mark 0x0/0x3f00 -m comment --comment wan -j MARK --set-xmark 0x100/0x3f00
-A mwan3_iface_in_wanb -i pppoe-wanb -m set --match-set mwan3_connected src -m mark --mark 0x0/0x3f00 -m comment --comment default -j MARK --set-xmark 0x3f00/0x3f00
-A mwan3_iface_in_wanb -i pppoe-wanb -m mark --mark 0x0/0x3f00 -m comment --comment wanb -j MARK --set-xmark 0x300/0x3f00
-A mwan3_ifaces_in -m mark --mark 0x0/0x3f00 -j mwan3_iface_in_wan
-A mwan3_ifaces_in -m mark --mark 0x0/0x3f00 -j mwan3_iface_in_wanb
-A mwan3_policy_balanced -m mark --mark 0x0/0x3f00 -m statistic --mode random --probability 0.39999999991 -m comment --comment "wanb 2 5" -j MARK --set-xmark 0x300/0x3f00
-A mwan3_policy_balanced -m mark --mark 0x0/0x3f00 -m comment --comment "wan 3 3" -j MARK --set-xmark 0x100/0x3f00
-A mwan3_policy_wan_only -m mark --mark 0x0/0x3f00 -m comment --comment "wan 3 3" -j MARK --set-xmark 0x100/0x3f00
-A mwan3_policy_wan_wanb -m mark --mark 0x0/0x3f00 -m comment --comment "wan 3 3" -j MARK --set-xmark 0x100/0x3f00
-A mwan3_policy_wanb_only -m mark --mark 0x0/0x3f00 -m comment --comment "wanb 2 2" -j MARK --set-xmark 0x300/0x3f00
-A mwan3_policy_wanb_wan -m mark --mark 0x0/0x3f00 -m comment --comment "wanb 2 2" -j MARK --set-xmark 0x300/0x3f00
-A mwan3_rule_https -m mark --mark 0x0/0x3f00 -j MARK --set-xmark 0x100/0x3f00
-A mwan3_rule_https -m mark --mark 0x100/0x3f00 -m set ! --match-set mwan3_sticky_https src,src -j MARK --set-xmark 0x0/0x3f00
-A mwan3_rule_https -m mark --mark 0x0/0x3f00 -j mwan3_policy_wan_wanb
-A mwan3_rule_https -m mark ! --mark 0xfc00/0xfc00 -j SET --del-set mwan3_sticky_https src,src
-A mwan3_rule_https -m mark ! --mark 0xfc00/0xfc00 -j SET --add-set mwan3_sticky_https src,src
-A mwan3_rule_youtube -m mark --mark 0x0/0x3f00 -j MARK --set-xmark 0x300/0x3f00
-A mwan3_rule_youtube -m mark --mark 0x300/0x3f00 -m set ! --match-set mwan3_sticky_youtube src,src -j MARK --set-xmark 0x0/0x3f00
-A mwan3_rule_youtube -m mark --mark 0x0/0x3f00 -j mwan3_policy_wanb_only
-A mwan3_rule_youtube -m mark ! --mark 0xfc00/0xfc00 -j SET --del-set mwan3_sticky_youtube src,src
-A mwan3_rule_youtube -m mark ! --mark 0xfc00/0xfc00 -j SET --add-set mwan3_sticky_youtube src,src
-A mwan3_rules -p tcp -m set --match-set youtube dst -m multiport --sports 0:65535 -m multiport --dports 80,443 -m mark --mark 0x0/0x3f00 -m comment --comment youtube -j mwan3_rule_youtube
-A mwan3_rules -p tcp -m multiport --sports 0:65535 -m multiport --dports 443 -m mark --mark 0x0/0x3f00 -m comment --comment https -j mwan3_rule_https
-A mwan3_rules -m mark --mark 0x0/0x3f00 -m comment --comment default_rule -j mwan3_policy_wan_wanb
#2

Not connected, but which option is this?

Other than that it looks fine, but there are no hits.
The IP you are using in ipset for youtube is stunnel server or youtube?

#3

lan2 is another network that I use for external services so it does not use the lan network, for security. youtube is only to test since neither the stunnel server nor youtube is connected by wanb, they always use wan. traceroute looks good, always use wan. Does it help you to see the traceroute?

#4

I didn't ask exactly that. You have one dhcp_option without any number, just an IP address. DHCP options are usually a number and then it can be an IP or something else.

youtube ipset has one IP. If you are trying to connect to that IP it will use the wanb interface, but only tcp and udp ports 80 and 443. So anything else will use the wan if it is up or wanb if wan is down.

#5

Thanks for the information, I already modified dhcp_option correctly, I did tests with youtube and it works correctly going by wanb but when I use stunnel it connects to the stunnel server by wan and not by wanb, can be set stunnel to use a default port so I can set the mwan3 rule on source port to that port or something like that? any idea is appreciated.

root@router:/etc/stunnel# logread -e stunnel
Tue Jul 21 14:45:38 2020 daemon.notice stunnel: LOG5[ui]: stunnel 5.55 on mips-openwrt-linux-gnu platform
Tue Jul 21 14:45:38 2020 daemon.notice stunnel: LOG5[ui]: Compiled/running with OpenSSL 1.1.1g  21 Apr 2020
Tue Jul 21 14:45:38 2020 daemon.notice stunnel: LOG5[ui]: Threading:PTHREAD Sockets:POLL,IPv6 TLS:ENGINE,FIPS,OCSP,PSK,SNI
Tue Jul 21 14:45:38 2020 daemon.notice stunnel: LOG5[ui]: Reading configuration from file /etc/stunnel/stunnel.conf
Tue Jul 21 14:45:38 2020 daemon.notice stunnel: LOG5[ui]: UTF-8 byte order mark not detected
Tue Jul 21 14:45:38 2020 daemon.notice stunnel: LOG5[ui]: FIPS mode disabled
Tue Jul 21 14:45:38 2020 daemon.warn stunnel: LOG4[ui]: Service [dummy] needs authentication to prevent MITM attacks
Tue Jul 21 14:45:38 2020 daemon.warn stunnel: LOG4[ui]: Service [openvpn] needs authentication to prevent MITM attacks
Tue Jul 21 14:45:38 2020 daemon.notice stunnel: LOG5[ui]: Configuration successful
Tue Jul 21 14:45:44 2020 daemon.notice stunnel: LOG5[0]: Service [openvpn] accepted connection from 127.0.0.1:50570
Tue Jul 21 14:45:44 2020 daemon.notice stunnel: LOG5[0]: s_connect: connected 190.40.*.*:443
Tue Jul 21 14:45:44 2020 daemon.notice stunnel: LOG5[0]: Service [openvpn] connected remote server from 190.50.*.*:50474

It should be 186. *. *. * and not 190.50. *. *

root@router:/etc/stunnel# traceroute *.duckdns.org
traceroute to *.duckdns.org (190.40.*.*), 30 hops max, 38 byte packets
 1  host42.181-89-4.telecom.net.ar (181.89.4.42)  18.445 ms  18.374 ms  18.998 ms
 2  host194.181-96-113.telecom.net.ar (181.96.113.194)  42.061 ms  host52.181-89-3.telecom.net.ar (181.89.3.52)  42.780 ms  host194.181-96-113.telecom.net.ar (181.96.113.194)  42.659 ms
 3  *  *  *
 
root@router:/etc/stunnel# traceroute google.com
traceroute to google.com (172.217.173.14), 30 hops max, 38 byte packets
 1  *  *  *
 2  *  *  *
 3  *  *  *
 4  209-165-89-200.fibertel.com.ar (200.89.165.209)  21.298 ms  *  20.636 ms
 5  host119.181-89-2.telecom.net.ar (181.89.2.119)  20.939 ms  22.727 ms  *
#6

If you need the traffic to stunnel server to use wanb, then make a rule for the IP of stunnel in mwan3.

#7

Still not working, everything works correctly when I want to communicate to the ip where the stunnel server is by wanb but when I use stunnel client it is directed by wan. I have tried all the ports and all the protocols and neither.
mwan3 rule does not get hit when running stunnel, only if I do a traceroute there start to be hits.
Not having an ipv6 rule set in mwan3 can affect stunnel?

#8

It would be better to explain the flow (source IP/protocol/port -> destination IP/protocol/port) to reach this stunnel server.

#9

I was testing and stunnel works correctly, but I had to wait a while for it to use wanb, I estimate that the problem is that wanb takes longer to get active (pppoe), so mwan3 and stunnel use wan, after a while (several minutes) surely determined by mwan3 verifies that wanb is active and starts applying the rule sent by wanb. Thanks trendy for the help!

#10

pppoe doesn't take too long to come online. However if you consider it as offline when mwan3 starts and you have configured many successful pings to restore it online, you can play a bit with the wanb settings (failure interval, recovery interval, interface up)

If your problem is solved, please consider marking this topic as [Solved]. See How to mark a topic as [Solved] for a short how-to.

#11

I couldn't fix the problem .. what I can see is that wanb (pppoe) takes a while to get active, but once it gets active it does very few hits on wanb and still uses wan, try changing the values ​​of up, failure interval, interface up, but does not solve the problem.

#12

We'll need some more information to help with troubleshooting.

Once the wanb is up I don't see why it won't be used for the rules you have set. You might want to remove the sticky from that rule as well.

#13

Disabling sticky does not solve the problem, I use openvpn over stunnel, try changing the protocol on both sides of openvpn so that it uses tcp4-client and tcp4-server in case it is an ipv6 problem but it does not solve the problem.

Stunnel on both sides

[openvpn]                                                     
client = yes
accept = 127.0.0.1:1194
connect = *.duckdns.org:443 

[openvpn]                                                     
accept = 443                                                  
connect = 127.0.0.1:1194                                      
cert = /etc/stunnel/stunnel.pem

The rule is normally applied when I ping or traceroute, the problem is with stunnel so i think.

#14

Why?

Still you are not describing the flow properly and I have to guess that both source and destination ports are tcp/443 between this router and some host with duckdns. Which rule do you have for that?

#15

Because the isp blocks the connection, both ports are tcp / 443

config rule 'stunnel'                                                                                                                  
        option sticky '0'                                                                                                              
        option use_policy 'wanb_only'                                                                                                  
        option dest_ip '*.duckdns.org'                                                                                           
        #option ipset 'duckdns'
        #option dest_port '80,443'                                                                                                        
        option proto 'all'

I tried changing the protocols, the ports ... but it doesn't solve the problem.

#16

Then why don't you change the OpenVPN port to 443?

This rule works fine for me using the backup link only:

config rule 'test'
        option dest_ip '1.0.0.1'
        option proto 'all'
        option sticky '0'
        option use_policy 'lte_only'

Make sure the order of the rules is correct. Try to float this rule to the top of the list.

#17

I use openvpn over stunnel because it is more stable when transmitting image sequences and I don't have to deal with mtu, but both openvpn with or without stunnel have the same problem. The connection is established by wan but over time mwan3 does not pull down the connection and creates a new one with wanb when it is active, it maintains the connection with wan but sends packets over wanb, since netstat shows me that the connection is established and in one At the ends is the address of wan, in the packets that wanb sends and receives, I can see that packets are transmitted by wanb, but that happens with time, not immediately when the interface becomes active. Try once the wanb is active and working restart the stunnel service together with openvpn or openvpn only and it also connects via wan. At both points there is a router with openwrt.