PBR setup to bypass VPN

pluffmud · February 3, 2025, 8:59pm

I setup my router to use a vpn via wireguard .
It works .
So I soon discovered that Amazon Prime Video is unhappy with my vpn . This is not unusual it seems .
So research indicates I can setup PBR to route specific ips out the wan interface instead of the wg interface .
I decided to use fast.com(23.79.185.91) as a test since its easy to see if its working(getting full fiber speed) .
After installing the packages and setting up my policies pbr comes up with no complaints .
But my test to fast.com shows I'm still going out the vpn interface .

Any ideas ?

{
"kernel": "5.15.167",
"hostname": "shadow",
"system": "MediaTek MT7621 ver:1 eco:3",
"model": "Ubiquiti EdgeRouter X",
"board_name": "ubnt,edgerouter-x",
"rootfs_type": "squashfs",
"release": {
"distribution": "OpenWrt",
"version": "23.05.5",
"revision": "r24106-10cc5fcd00",
"target": "ramips/mt7621",
"description": "OpenWrt 23.05.5 r24106-10cc5fcd00"
}
}

Setting up routing for 'wan/eth0/67.146.202.1' [✓]
Setting up routing for 'WG/10.73.246.103' [✓]
Routing 'VPN No' via wan [✓]
Routing 'VPN Yes' via WG [✓]
Installing fw4 nft file [✓]
pbr 1.1.6-22 monitoring interfaces: wan WG
pbr 1.1.6-22 (fw4 nft file mode) started with gateways:
wan/eth0/67.146.202.1 [✓]
WG/10.73.246.103

pbr config
config pbr 'config'
option enabled '1'
option verbosity '2'
option strict_enforcement '1'
option resolver_set 'none'
list resolver_instance '*'
option ipv6_enabled '0'
option boot_timeout '30'
option rule_create_option 'add'
option procd_boot_delay '0'
option procd_reload_delay '1'
option webui_show_ignore_target '0'
option nft_rule_counter '0'
option nft_set_auto_merge '1'
option nft_set_counter '0'
option nft_set_flags_interval '1'
option nft_set_flags_timeout '0'
option nft_set_policy 'performance'
list webui_supported_protocol 'all'
list webui_supported_protocol 'tcp'
list webui_supported_protocol 'udp'
list webui_supported_protocol 'tcp udp'
list webui_supported_protocol 'icmp'
list supported_interface 'br-lan.1'
list supported_interface 'br-lan.3'

config policy
option name 'VPN No'
option dest_addr '23.79.185.91'
option interface 'wan'
option src_addr '10.10.10.0/24 10.10.20.0/24'
option src_port ' 0-65535'

config policy
option name 'VPN Yes'
option interface 'WG'
option src_addr '10.10.10.0/24 10.10.20.0/24'

network config
config interface 'loopback'
option device 'lo'
option proto 'static'
option ipaddr '127.0.0.1'
option netmask '255.0.0.0'

config globals 'globals'
option ula_prefix 'fd97:0191:ac00::/48'
option packet_steering '1'

config device
option name 'br-lan'
option type 'bridge'
option acceptlocal '1'
list ports 'eth1'
list ports 'eth2'
list ports 'eth3'
list ports 'eth4'

config interface 'lan'
option device 'br-lan.1'
option proto 'static'
option netmask '255.255.255.0'
option ipaddr '10.10.10.1'
option igmp_snooping '0'
option delegate '0'

config interface 'wan'
option device 'eth0'
option proto 'dhcp'
option peerdns '0'
list dns '1.1.1.1'
list dns '1.0.0.1'
option hostname 'DTES'

config device
option name 'eth1'
option acceptlocal '1'

config device
option name 'eth2'
option acceptlocal '1'

config device
option name 'eth3'
option acceptlocal '1'

config device
option name 'eth4'
option acceptlocal '1'

config bridge-vlan
option device 'br-lan'
option vlan '1'
list ports 'eth1:t'
list ports 'eth2:t'
list ports 'eth3:t'
list ports 'eth4:t'

config bridge-vlan
option device 'br-lan'
option vlan '3'
list ports 'eth1:u*'
list ports 'eth2:t'
list ports 'eth3:t'
list ports 'eth4:t'

config interface 'lan3'
option proto 'static'
option device 'br-lan.3'
option ipaddr '10.10.20.1'
option netmask '255.255.255.0'
option ip6ifaceid '::3'
option igmp_snooping '0'

config bridge-vlan
option device 'br-lan'
option vlan '5'
list ports 'eth1:t'
list ports 'eth4:t'

config interface 'lan5'
option proto 'static'
option device 'br-lan.5'
option ipaddr '10.10.30.1'
option netmask '255.255.255.0'

config interface 'WG'
option proto 'wireguard'
option private_key 'INrh+oBqi7zpL5sKDH7iBdYCKZEJKpp82kJk6ZQ9eng='
list addresses '10.73.246.103/32'
option force_link '1'

config wireguard_WG
option description 'mullvard-atlanta'
option public_key ''
list allowed_ips '0.0.0.0/0'
option route_allowed_ips '0'
option endpoint_host '104.223.91.18'

firewall config
config defaults
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'REJECT'
option synflood_protect '1'
option drop_invalid '1'
option flow_offloading '1'
option flow_offloading_hw '1'

config zone
option name 'vlan1'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
list network 'lan'

config zone
option name 'wan'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
option masq '1'
option mtu_fix '1'
list network 'wan'

config rule
option name 'Allow-DHCP-Renew'
option src 'wan'
option proto 'udp'
option dest_port '68'
option target 'ACCEPT'

config rule
option name 'Allow-Ping-Wan'
list proto 'icmp'
list icmp_type 'echo-request'
option src 'wan'
option target 'ACCEPT'

config rule
option name '3-to-dns-dhcp'
option src 'vlan3'
option dest_port '53 67'
option target 'ACCEPT'

config rule
option name 'media players to wan'
option dest 'wan'
option target 'DROP'
option direction 'out'
option device 'eth0'
option src 'vlan3'
list proto 'all'
list src_ip '10.10.20.34'
list src_ip '10.10.20.26'

config zone
option name 'vlan3'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
list network 'lan3'

config redirect
option target 'DNAT'
option name 'ntp-redirect-vlan1'
option src 'vlan1'
option src_dport '123'
list proto 'udp'
option reflection '0'
option enabled '0'

config redirect
option target 'DNAT'
option name 'ntp-redirect-vlan3'
option src 'vlan3'
option src_dport '123'
list proto 'udp'
option reflection '0'
option enabled '0'

config forwarding
option src 'vlan1'
option dest 'vlan3'

config rule
option name 'MDNS'
option src 'vlan3'
option src_port '5353'
option dest_port '5353'
option target 'ACCEPT'
list proto 'udp'
list dest_ip '224.0.0.251'
option family 'ipv4'

config include 'user'
option enabled '1'
option type 'script'
option path '/etc/config/firewall.user'
option fw4_compatible '1'

config rule
option name 'Chromecast Ports '
option src 'vlan3'
option target 'ACCEPT'
option dest_port '8008 8009 8010 32768-61000'
list proto 'tcp'
list proto 'udp'
option dest '*'
list src_mac 'E4:F0:42:A1:D2:36'
list src_mac '54:60:09:FD:A4:80'

config redirect 'dns_int_1'
option name 'Intercept-DNS vlan1'
option family 'ipv4'
option proto 'tcp udp'
option src 'vlan1'
option src_dport '53'
option target 'DNAT'
option enabled '0'

config redirect 'dns_int_3'
option name 'Intercept-DNS vlan3'
option family 'ipv4'
option proto 'tcp udp'
option src 'vlan3'
option src_dport '53'
option target 'DNAT'
option enabled '0'

config zone
option name 'vlan5'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'REJECT'
list network 'lan5'

config rule
option name 'Denon-to-1'
option src 'vlan3'
list src_ip '10.10.20.26'
option target 'ACCEPT'
option dest '*'
list proto 'tcp'
list proto 'udp'
option dest_port '80 443 3813 8080 5020 39000-60006'

config zone
option name 'WGVPN'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
option masq '1'
option mtu_fix '1'
list network 'WG'

config include 'pbr'
option fw4_compatible '1'
option type 'script'
option path '/usr/share/pbr/firewall.include'

config forwarding
option src 'vlan1'
option dest 'wan'

config forwarding
option src 'vlan3'
option dest 'wan'

config forwarding
option src 'vlan1'
option dest 'WGVPN'

config forwarding
option src 'vlan3'
option dest 'WGVPN'

bahtsiz_bedevi · February 3, 2025, 9:35pm

For destination addresses, use domains instead of IPs because they can change

pluffmud · February 4, 2025, 12:19pm

Thats true but I dont believe it has anything to do with my issue

egc · February 4, 2025, 12:30pm

As already noted use a domain unless you are sure that this domain only has this IP address.

As you already route default via the WAN no need to add a source, also remove the src_port

I tested with this and this works for me

config policy
	option name 'domain_via_wan'
	option dest_addr 'fast.com whatismyip.com'
	option interface 'wan'

You are using the IPv4 address of the interfaces I assume, this works OK unless you also have implemented IPv6 on your network in that case use the device e.g. @br-lan etc.

This is my rule:

config policy
	option name 'lan'
	option src_addr '@br-lan'
	option interface 'mullvad_se'

Important:

this only work for your LAN clients and not from the router , so you have to test from your LAN clients
The rule with the dest_address must become first! (which seems the case)
You are using an old version of PBR so if it that still does not work upgrade to latest 1.1.8 version
I do not know fast.com better test with ipleak.net of whatismyip.com
After you have applied it takes about 30-60 sec before the routes are operational because of restarting of pbr, dnsmasq, interfaces etc. so be patient while testing
When using domain names it can take a while for the DNS cache has been refreshed so reboot router and client for a proper test

pluffmud · February 4, 2025, 12:42pm

OK thats some good stuff . What about my firewall zones ? Do they come into play in this ?

egc · February 4, 2025, 12:47pm

Sure you must have a forwarding from your LAN zones both to the WAN and to the VPN

pluffmud · February 4, 2025, 1:01pm

Thats what I thought . I'll give all this a try when I get a 'downtime window'

bahtsiz_bedevi · February 4, 2025, 1:03pm

I really didn't look into your config soup because you have squeezed multiple configs and outputs into one code block.

pluffmud · February 5, 2025, 4:15pm

It works sort of .
I performed all the recommended actions including upgrading PBR .
It works for whatsmyip.com but not for fast.com .

Here is PBR status

pbr - environment
pbr 1.1.8-r6 running on OpenWrt 23.05.5.

Dnsmasq version 2.90  Copyright (c) 2000-2024 Simon Kelley
Compile time options: IPv6 GNU-getopt no-DBus UBus no-i18n no-IDN DHCP DHCPv6 no-Lua TFTP conntrack no-ipset nftset auth cryptohash DNSSEC no-ID loop-detect inotify dumpfile

pbr fw4 nft file: /usr/share/nftables.d/ruleset-post/30-pbr.nft
add chain inet fw4 pbr_mark_0x010000
add rule inet fw4 pbr_mark_0x010000  mark set mark and 0xff00ffff xor 0x010000
add rule inet fw4 pbr_mark_0x010000 return
add chain inet fw4 pbr_mark_0x020000
add rule inet fw4 pbr_mark_0x020000  mark set mark and 0xff00ffff xor 0x020000
add rule inet fw4 pbr_mark_0x020000 return
add rule inet fw4 pbr_prerouting ip daddr { 104.21.16.1,104.21.80.1,104.21.48.1,104.21.96.1,104.21.112.1,104.21.64.1,104.21.32.1, 23.79.185.91 }  goto pbr_mark_0x010000 comment "VPN No"

pbr chains - policies
        chain pbr_forward { # handle 53
        }
        chain pbr_input { # handle 54
        }
        chain pbr_output { # handle 55
        }
        chain pbr_postrouting { # handle 57
        }
        chain pbr_prerouting { # handle 56
                ip daddr { 23.79.185.91, 104.21.16.1, 104.21.32.1, 104.21.48.1, 104.21.64.1, 104.21.80.1, 104.21.96.1, 104.21.112.1 } goto pbr_mark_0x010000 comment "VPN No" # handle 2611
        }
        chain pbr_dstnat { # handle 52
        }

pbr chains - marking
        chain pbr_mark_0x010000 { # handle 2604
                meta mark set meta mark & 0xff01ffff | 0x00010000 # handle 2605
                return # handle 2606
        }
        chain pbr_mark_0x020000 { # handle 2607
                meta mark set meta mark & 0xff02ffff | 0x00020000 # handle 2608
                return # handle 2609
        }

pbr nft sets

dnsmasq sets
nftset=/whatsmyip.com/4#inet#fw4#pbr_wan_4_dst_ip_cfg056ff5 # VPN No
nftset=/fast.com/4#inet#fw4#pbr_wan_4_dst_ip_cfg056ff5 # VPN No

pbr tables & routing
IPv4 table 256 pbr_wan route:
default via 67.146.202.1 dev eth0 
IPv4 table 256 pbr_wan rule(s):
30000:  from all fwmark 0x10000/0xff0000 lookup pbr_wan

IPv4 table 257 pbr_WG route:
default via 10.73.246.103 dev WG 
IPv4 table 257 pbr_WG rule(s):
29998:  from all fwmark 0x20000/0xff0000 lookup pbr_WG

Here is PBR config

c

onfig pbr 'config'
        option enabled '1'
        option verbosity '2'
        option strict_enforcement '1'
        option resolver_set 'none'
        list resolver_instance '*'
        option ipv6_enabled '0'
        list ignored_interface 'vpnserver'
        option boot_timeout '30'
        option rule_create_option 'add'
        option procd_reload_delay '1'
        option webui_show_ignore_target '0'
        option nft_rule_counter '0'
        option nft_set_auto_merge '1'
        option nft_set_counter '0'
        option nft_set_flags_interval '1'
        option nft_set_flags_timeout '0'
        option nft_set_policy 'performance'
        list webui_supported_protocol 'all'
        list webui_supported_protocol 'tcp'
        list webui_supported_protocol 'udp'
        list webui_supported_protocol 'tcp udp'
        list webui_supported_protocol 'icmp'

config include
        option path '/usr/share/pbr/pbr.user.aws'
        option enabled '0'

config include
        option path '/usr/share/pbr/pbr.user.netflix'
        option enabled '0'

config policy
        option name 'Ignore Local Requests'
        option interface 'ignore'
        option dest_addr '10.0.0.0/24 10.0.1.0/24 192.168.100.0/24 192.168.1.0/24'
        option enabled '0'

config policy
        option name 'VPN No'
        option dest_addr 'whatsmyip.com fast.com'
        option interface 'wan'

egc · February 5, 2025, 4:33pm

Some domains use internal redirects in that case it will not work

Easy way to find out is do an nslookup of the domain and use the IP address to connect in your browser

pluffmud · February 5, 2025, 5:02pm

That appears to be the case for 'fast.com' . I'll have to find something else to test with . Thanks

egc · February 5, 2025, 5:09pm

ipleak.net , whatismyip.com are good

pluffmud · February 5, 2025, 5:51pm

Thanks again .
Of course I picked a 'simple' test case that cant work leading to much confusion

pluffmud · February 5, 2025, 6:19pm

OK. By using a couple of include scripts I have gotten my amazon video to bypass the vpn.
But this results in my 'VPN No' policy to no longer be in force . It has no 'elements' .

set pbr_wan_4_dst_ip_cfg046ff5 { # handle 4740
                type ipv4_addr
                flags interval
                auto-merge
                comment "VPN No"
        }

If I turn off the user includes and restart pbr the 'VPN No' policy start working again .
Is this by design ?

egc · February 5, 2025, 6:25pm

User include scripts are "as is".
Meaning no support, those are just some examples which need to be adapted to your use case.

pluffmud · February 5, 2025, 6:30pm

I'm marking this as 'fixed' .

couch-potato-dude · February 13, 2025, 8:33pm

So what was the "fix"? What you related in post 14?