Hi Friends,
Everything was going smooth, and then I got a requirement to route Wireguard traffic with a specific WAN.
After some google searches and an expert's suggestions from Reddit, I came to the conclusion that we should use policy-based routing to achieve this requirement.
Here is my topology,
Setup Description
- DJ's_NS3_CR-1 ---> OpenWRT device in Gns3, DJ's_AWS_CR-1 ---> OpenWRT Virtual device in AWS.
- Created 3 Wireguard tunnels between DJ's_NS3_CR-1 and DJ's_AWS_CR-1.
- DJ's_NS3_CR-1 has three WANs (eth0, eth1, eth2).
- Each Wireguard interface is mapped with different WANs through VPN-policy-routing.
Observation
If I try to push wg8/wg9 traffic, it should go via wan1(eth1)/wan2(eth2). But, I could notice that the traffic is been going through wan(eth0) only.
Configurations and logs:
config interface 'lan'
option type 'bridge'
option proto 'static'
option netmask '255.255.255.0'
option ipaddr '192.168.10.1'
option ifname ' eth3 '
config interface 'wan'
option ifname 'eth0'
option proto 'dhcp'
list dns '8.8.4.4'
option metric '10'
config wanlist 'wan_entry'
list usemember 'wan'
list usemember 'wan1'
list usemember 'wan2'
config interface 'wan1'
option ifname 'eth1'
option metric '11'
option proto 'dhcp'
config interface 'wan2'
option proto 'dhcp'
option ifname 'eth2'
option metric '12'
config interface 'wg7'
option proto 'wireguard'
option private_key 'sAt0imdKfD9PPsmUXzzZ7keNXtPXdZjD11bkWJw72Uc='
list addresses '10.181.116.128'
option listen_port '51821'
option nohostroute '1'
option tunlink 'wan'
config wireguard_wg7 '6C0128D0D'
option public_key 'TwSNiuQfjoszS6FLWLPGFdabaQAbyEvAjsjYljYf0TY='
option route_allowed_ips '0'
option endpoint_port '51821'
option persistent_keepalive '25'
list allowed_ips '0.0.0.0/0'
option endpoint_host '34.214.222.30'
config route 'wg7static'
option interface 'wg7'
option target '10.181.209.51'
option netmask '255.255.255.255'
option gateway '0.0.0.0'
config interface 'wg8'
option proto 'wireguard'
option private_key 'sJpyZjlKkrRUog0auXEuofMWbP6LExQLZpqb9I63u3s='
list addresses '10.181.173.234'
option listen_port '51822'
option nohostroute '1'
option tunlink 'wan1'
config wireguard_wg8 '36472042B'
option public_key 'AUjZFsnwls3gD4Zc1ZcFHY+eLRzHU7epVDoMyM7XaUY='
option route_allowed_ips '0'
list allowed_ips '0.0.0.0/0'
option endpoint_host '34.214.222.30'
option endpoint_port '51822'
option persistent_keepalive '25'
config route 'wg8static'
option interface 'wg8'
option target '10.181.156.137'
option netmask '255.255.255.255'
option gateway '0.0.0.0'
config interface 'wg9'
option proto 'wireguard'
option private_key 'SF7/EvqNMXx48OYTiof0MS1SRIQ/M7WJdN3SLFA8h0E='
list addresses '10.181.77.77'
option listen_port '51823'
option nohostroute '1'
option tunlink 'wan2'
config wireguard_wg9 'B0C51E6DE'
option public_key 'gkOp7q9aP7OOkHb1GyyauEVjqKfQ5sB01NBoGQdnQgU='
option route_allowed_ips '0'
list allowed_ips '0.0.0.0/0'
option endpoint_host '34.214.222.30'
option endpoint_port '51823'
option persistent_keepalive '25'
config route 'wg9static'
option interface 'wg9'
option target '10.181.118.38'
option netmask '255.255.255.255'
option gateway '0.0.0.0'
vpn-policy-routing configuration:
config vpn-policy-routing 'config'
option enabled '1'
option verbosity '2'
option strict_enforcement '1'
option src_ipset '0'
option dest_ipset '0'
option resolver_ipset 'dnsmasq.ipset'
option ipv6_enabled '0'
list ignored_interface 'vpnserver wgserver'
option boot_timeout '30'
option iptables_rule_option 'insert'
option procd_reload_delay '1'
option webui_enable_column '0'
option webui_protocol_column '0'
option webui_chain_column '0'
option webui_show_ignore_target '0'
option webui_sorting '1'
list webui_supported_protocol 'tcp'
list webui_supported_protocol 'udp'
list webui_supported_protocol 'tcp udp'
list webui_supported_protocol 'icmp'
list webui_supported_protocol 'all'
config policy 'wg7'
option name 'wg7'
option proto 'udp'
option src_port '51821'
option chain 'POSTROUTING'
option interface 'wan'
config policy 'wg8'
option name 'wg8'
option proto 'udp'
option src_port '51822'
option chain 'POSTROUTING'
option interface 'wan1'
config policy 'wg9'
option name 'wg9'
option proto 'udp'
option src_port '51823'
option chain 'POSTROUTING'
option interface 'wan2'
Route table:
root@V2000-3762:/# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.122.1 0.0.0.0 UG 10 0 0 eth0
0.0.0.0 10.10.1.1 0.0.0.0 UG 11 0 0 eth1
0.0.0.0 192.168.42.129 0.0.0.0 UG 12 0 0 eth2
10.10.1.0 0.0.0.0 255.255.255.0 U 11 0 0 eth1
10.171.0.0 0.0.0.0 255.255.255.0 U 0 0 0 wg0
10.171.0.1 0.0.0.0 255.255.255.255 UH 0 0 0 wg0
10.181.118.38 0.0.0.0 255.255.255.255 UH 0 0 0 wg9
10.181.156.137 0.0.0.0 255.255.255.255 UH 0 0 0 wg8
10.181.209.51 0.0.0.0 255.255.255.255 UH 0 0 0 wg7
54.68.136.42 192.168.42.129 255.255.255.255 UGH 12 0 0 eth2
192.168.10.0 0.0.0.0 255.255.255.0 U 0 0 0 br-lan
192.168.42.0 0.0.0.0 255.255.255.0 U 12 0 0 eth2
192.168.122.0 0.0.0.0 255.255.255.0 U 10 0 0 eth0
Pushing wg8 traffic:
root@V2000-3762:/# ping 10.181.156.137
PING 10.181.156.137 (10.181.156.137): 56 data bytes
64 bytes from 10.181.156.137: seq=0 ttl=64 time=237.293 ms
64 bytes from 10.181.156.137: seq=1 ttl=64 time=237.360 ms
64 bytes from 10.181.156.137: seq=2 ttl=64 time=237.070 ms
^C
--- 10.181.156.137 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 237.070/237.241/237.360 ms
root@V2000-3762:/# ping 10.181.156.137 > /dev/null &
Verification of packet hitting with respective chain:
root@V2000-3762:/# iptables -t mangle -nvL VPR_POSTROUTING
Chain VPR_POSTROUTING (1 references)
pkts bytes target prot opt in out source destination
176 20228 VPR_MARK0x010000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51821 /* wg7 */
1946 301K VPR_MARK0x020000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51822 /* wg8 */
176 20116 VPR_MARK0x040000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51823 /* wg9 */
root@V2000-3762:/# iptables -t mangle -nvL VPR_POSTROUTING
Chain VPR_POSTROUTING (1 references)
pkts bytes target prot opt in out source destination
176 20228 VPR_MARK0x010000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51821 /* wg7 */
1950 302K VPR_MARK0x020000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51822 /* wg8 */
176 20116 VPR_MARK0x040000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51823 /* wg9 */
root@V2000-3762:/# iptables -t mangle -nvL VPR_POSTROUTING
Chain VPR_POSTROUTING (1 references)
pkts bytes target prot opt in out source destination
176 20228 VPR_MARK0x010000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51821 /* wg7 */
1955 302K VPR_MARK0x020000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51822 /* wg8 */
178 20428 VPR_MARK0x040000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51823 /* wg9 */
root@V2000-3762:/# iptables -t mangle -nvL VPR_POSTROUTING
Chain VPR_POSTROUTING (1 references)
pkts bytes target prot opt in out source destination
176 20228 VPR_MARK0x010000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51821 /* wg7 */
1957 303K VPR_MARK0x020000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51822 /* wg8 */
178 20428 VPR_MARK0x040000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51823 /* wg9 */
root@V2000-3762:/# iptables -t mangle -nvL VPR_POSTROUTING
Chain VPR_POSTROUTING (1 references)
pkts bytes target prot opt in out source destination
176 20228 VPR_MARK0x010000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51821 /* wg7 */
1959 303K VPR_MARK0x020000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51822 /* wg8 */
178 20428 VPR_MARK0x040000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51823 /* wg9 */
root@V2000-3762:/# iptables -t mangle -nvL VPR_POSTROUTING
Chain VPR_POSTROUTING (1 references)
pkts bytes target prot opt in out source destination
177 20288 VPR_MARK0x010000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51821 /* wg7 */
1961 303K VPR_MARK0x020000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51822 /* wg8 */
178 20428 VPR_MARK0x040000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51823 /* wg9 */
root@V2000-3762:/# iptables -t mangle -nvL VPR_POSTROUTING
Chain VPR_POSTROUTING (1 references)
pkts bytes target prot opt in out source destination
177 20288 VPR_MARK0x010000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51821 /* wg7 */
1963 304K VPR_MARK0x020000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51822 /* wg8 */
178 20428 VPR_MARK0x040000 udp -- * * 0.0.0.0/0 0.0.0.0/0 [goto] multiport sports 51823 /* wg9 */
root@V2000-3762:/#
Note: Since wg8 is mapped with wan1(eth1) the packets are hitting as expected in the VPR_MARK0x020000 chain
IP Rule:
root@V2000-3762:/# ip rule show
0: from all lookup local
994: from all fwmark 0x70000/0xff0000 lookup wg9
995: from all fwmark 0x60000/0xff0000 lookup wg8
996: from all fwmark 0x50000/0xff0000 lookup wg7
997: from all fwmark 0x40000/0xff0000 lookup wan2
998: from all fwmark 0x30000/0xff0000 lookup wg0
999: from all fwmark 0x20000/0xff0000 lookup wan1
1000: from all fwmark 0x10000/0xff0000 lookup wan
1001: from all iif eth0 lookup 1
1002: from all iif eth1 lookup 2
1003: from all iif eth2 lookup 3
1004: from all iif wg7 lookup 4
1005: from all iif wg8 lookup 5
1006: from all iif wg9 lookup 6
2001: from all fwmark 0x100/0x3f00 lookup 1
2002: from all fwmark 0x200/0x3f00 lookup 2
2003: from all fwmark 0x300/0x3f00 lookup 3
2004: from all fwmark 0x400/0x3f00 lookup 4
2005: from all fwmark 0x500/0x3f00 lookup 5
2006: from all fwmark 0x600/0x3f00 lookup 6
2061: from all fwmark 0x3d00/0x3f00 blackhole
2062: from all fwmark 0x3e00/0x3f00 unreachable
3001: from all fwmark 0x100/0x3f00 unreachable
3002: from all fwmark 0x200/0x3f00 unreachable
3003: from all fwmark 0x300/0x3f00 unreachable
3004: from all fwmark 0x400/0x3f00 unreachable
3005: from all fwmark 0x500/0x3f00 unreachable
3006: from all fwmark 0x600/0x3f00 unreachable
32766: from all lookup main
32767: from all lookup default
root@V2000-3762:/#
tcpdump packet capture:
root@V2000-3762:/# tcpdump -ni wg8 icmp
[ 7570.596442] device wg8 entered promiscuous mode
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on wg8, link-type RAW (Raw IP), capture size 262144 bytes
05:31:57.715285 IP 10.181.173.234 > 10.181.156.137: ICMP echo request, id 37676, seq 1799, length 64
05:31:57.952556 IP 10.181.156.137 > 10.181.173.234: ICMP echo reply, id 37676, seq 1799, length 64
05:31:58.122815 IP 10.181.173.234 > 10.181.156.137: ICMP echo request, id 35074, seq 70, length 64
05:31:58.359943 IP 10.181.156.137 > 10.181.173.234: ICMP echo reply, id 35074, seq 70, length 64
05:31:58.715418 IP 10.181.173.234 > 10.181.156.137: ICMP echo request, id 37676, seq 1800, length 64
^C
5 packets captured
5 packets received by filter
0 packets dropped by kernel
[ 7571.874922] device wg8 left promiscuous mode
root@V2000-3762:/# tcpdump -ni eth1 port 51822
[ 7597.684178] device eth1 entered promiscuous mode
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth1, link-type EN10MB (Ethernet), capture size 262144 bytes
^C
0 packets captured
0 packets received by filter
0 packets dropped by kernel
[ 7600.158897] device eth1 left promiscuous mode
root@V2000-3762:/# tcpdump -ni eth0 port 51822
[ 7603.550988] device eth0 entered promiscuous mode
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes
05:32:30.720388 IP 192.168.122.251.51822 > 34.214.222.30.51822: UDP, length 128
05:32:30.957341 IP 34.214.222.30.51822 > 192.168.122.251.51822: UDP, length 128
05:32:31.127030 IP 192.168.122.251.51822 > 34.214.222.30.51822: UDP, length 128
05:32:31.364187 IP 34.214.222.30.51822 > 192.168.122.251.51822: UDP, length 128
05:32:31.720515 IP 192.168.122.251.51822 > 34.214.222.30.51822: UDP, length 128
05:32:31.957478 IP 34.214.222.30.51822 > 192.168.122.251.51822: UDP, length 128
05:32:32.127163 IP 192.168.122.251.51822 > 34.214.222.30.51822: UDP, length 128
05:32:32.364116 IP 34.214.222.30.51822 > 192.168.122.251.51822: UDP, length 128
^C
8 packets captured
8 packets received by filter
0 packets dropped by kernel
[ 7605.529658] device eth0 left promiscuous mode
root@V2000-3762:/#
Note: But when analysing through packet capture, the traffic is going via eth0(wan) interface.
Could anyone extend your support with this issue?Thanks in Advance.