Hello,
Essentially, I am unable to connect to internal hosts over a Wireguard VPN from the internet when using PBR routing. If PBR policies disabled, I can connect to internal hosts.
Steps to Reproduce:
- Connect to Wireguard Server on Openwrt from internet with PBR enabled (vpn ip 10.10.0.2) (wg0)
- Attempt to ping/connect to internal hosts, no response. (ping 10.0.0.11)
- The Internet PBR routing to (wg1, wg2, wg3) works fine.
My setup is a home router, with 1 ingress Wireguard server, and 3 egress Wireguard clients that I use PBR to route to. Internal hosts are connected over WIFI.
Did alot of setting changes, and forum posting looking, but no luck.
THANK YOU FOR YOUR HELP #WOW #FLOSS
ubus call system board
{
"kernel": "5.15.150",
"hostname": "router",
"system": "MediaTek MT7621 ver:1 eco:3",
"model": "Netgear R6260",
"board_name": "netgear,r6260",
"rootfs_type": "squashfs",
"release": {
"distribution": "OpenWrt",
"version": "23.05.3",
"revision": "r23809-234f1a2efa",
"target": "ramips/mt7621",
"description": "OpenWrt 23.05.3 r23809-234f1a2efa"
}
}
cat /etc/config/network
config interface 'loopback'
option device 'lo'
option proto 'static'
option ipaddr '127.0.0.1'
option netmask '255.0.0.0'
config globals 'globals'
option ula_prefix 'fd92:dac3:4423::/48'
option packet_steering '1'
config device
option name 'br-lan'
option type 'bridge'
list ports 'lan1'
list ports 'lan2'
list ports 'lan3'
list ports 'lan4'
option ipv6 '0'
config interface 'lan'
option device 'br-lan'
option proto 'static'
option ipaddr '10.0.0.1'
option netmask '255.255.255.0'
list dns '9.9.9.9'
list dns '149.112.112.112'
list dns_search '****DOMAIN*****'
option delegate '0'
config interface 'wan'
option device 'wan'
option proto 'dhcp'
option disabled '1'
option hostname '*'
option auto '0'
config interface 'wan6'
option device 'wan'
option proto 'dhcpv6'
option disabled '1'
option auto '0'
option reqaddress 'try'
option reqprefix 'auto'
config interface 'wwan'
option proto 'dhcp'
option device 'phy0-sta0'
option hostname '*'
config interface 'wg0'
option proto 'wireguard'
option private_key '************************************************'
option listen_port '51820'
list addresses '10.10.0.1/24'
option defaultroute '0'
config wireguard_wg0
option public_key '************************************************'
option private_key '************************************************'
option preshared_key '************************************************'
option route_allowed_ips '1'
option endpoint_host '*********'
option endpoint_port '51820'
option persistent_keepalive '25'
option description 'DEVICE1'
list allowed_ips '10.10.0.2/32'
config wireguard_wg0
option description 'DEVICE2'
option public_key '***********************************************''
option private_key '***********************************************''
option preshared_key '***********************************************''
list allowed_ips '10.10.0.3/32'
option route_allowed_ips '1'
option endpoint_host '************'
option persistent_keepalive '25'
option endpoint_port '51820'
config wireguard_wg0
option description 'DEVICE3'
option public_key '***********************************************''
option preshared_key '***********************************************''
list allowed_ips '10.10.0.4/32'
option route_allowed_ips '1'
option endpoint_host '*****************'
option endpoint_port '51820'
option persistent_keepalive '25'
config interface 'wg1'
option proto 'wireguard'
option private_key '*********************************************************'
list addresses '10.2.0.2/32'
option defaultroute '0'
list dns '9.9.9.9'
list dns '149.112.112.112'
config wireguard_wg1
option description 'INTERNET-VPN1'
option public_key '*********************************************************'
list allowed_ips '0.0.0.0/0'
option endpoint_host '143.*********'
option endpoint_port '51820'
option persistent_keepalive '25'
config interface 'wg2'
option proto 'wireguard'
option private_key '*********************************************************'
list addresses '10.200.200.2/32'
option defaultroute '0'
option delegate '0'
list dns '10.200.200.1'
config wireguard_wg2
option description 'INTERNET-VPN2'
option public_key '*********************************************************'
option preshared_key '*********************************************************'
list allowed_ips '0.0.0.0/0'
option endpoint_host '***********'
option endpoint_port '4500'
option persistent_keepalive '25'
config interface 'wg3'
option proto 'wireguard'
option private_key '*********************************************************'
list addresses '10.188.14.201/32'
option mtu '1320'
option defaultroute '0'
list dns '10.128.0.1'
option delegate '0'
config wireguard_wg3
option description 'INTERNET-VPN3'
option public_key '*********************************************************'
option preshared_key '*********************************************************'
list allowed_ips '0.0.0.0/0'
option endpoint_host '68.************'
option endpoint_port '1637'
option persistent_keepalive '15'
cat /etc/config/dhcp
config dnsmasq
option domainneeded '1'
option localise_queries '1'
option rebind_protection '1'
option rebind_localhost '1'
option local '//'
option domain '**DOMAIN**'
option expandhosts '1'
option cachesize '1000'
option authoritative '1'
option readethers '1'
option leasefile '/tmp/dhcp.leases'
option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
option localservice '1'
option ednspacket_max '1232'
option sequential_ip '1'
list interface 'lan'
list interface 'wg0'
config dhcp 'lan'
option interface 'lan'
option start '200'
option limit '249'
option leasetime '12h'
option dhcpv4 'server'
config dhcp 'wan'
option interface 'wan'
option ignore '1'
config odhcpd 'odhcpd'
option maindhcp '0'
option leasefile '/tmp/hosts/odhcpd'
option leasetrigger '/usr/sbin/odhcpd-update'
option loglevel '4'
config host
option name 'DEVICE1'
option dns '1'
option ip '10.0.0.100'
option leasetime '12h'
list mac '**************************'
config host
option name 'DEVICE5'
option dns '1'
option ip '10.0.0.11'
option leasetime '12h'
list mac '**************************'
config host
option name 'DEVICE4'
option dns '1'
option ip '10.0.0.10'
option leasetime '12h'
list mac '**************************'
config host
option name 'DEVICE2'
option dns '1'
option ip '10.0.0.101'
option leasetime '12h'
list mac '**************************'
config host
option name 'DEVICE3'
option dns '1'
option ip '10.0.0.102'
option leasetime '12h'
list mac '**************************'
cat /etc/config/firewall
config defaults
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
option synflood_protect '1'
option drop_invalid '1'
config zone
option name 'lan'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
list network 'lan'
config zone
option name 'wan'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
option masq '1'
option mtu_fix '1'
list network 'wan'
list network 'wan6'
list network 'wwan'
config forwarding
option src 'lan'
option dest 'wan'
config rule
option name 'Allow-DHCP-Renew'
option src 'wan'
option proto 'udp'
option dest_port '68'
option target 'ACCEPT'
option family 'ipv4'
config rule
option name 'Allow-Ping'
option src 'wan'
option proto 'icmp'
option icmp_type 'echo-request'
option family 'ipv4'
option target 'ACCEPT'
config zone
option name 'vpn'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
option family 'ipv4'
list network 'wg0'
config forwarding
option src 'vpn'
option dest 'wan'
config rule
option name 'Allow-Wireguard'
option src 'wan'
option dest_port '51820'
option target 'ACCEPT'
option family 'ipv4'
config forwarding
option src 'vpn'
option dest 'lan'
config zone
option name 'net'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
option family 'ipv4'
option masq '1'
list network 'wg1'
list network 'wg2'
list network 'wg3'
config forwarding
option src 'lan'
option dest 'net'
config forwarding
option src 'vpn'
option dest 'net'
config include 'pbr'
option fw4_compatible '1'
option type 'script'
option path '/usr/share/pbr/pbr.firewall.include'
ip ro
default via 192.168.0.1 dev phy0-sta0 proto static src 192.168.0.34
10.0.0.0/24 dev br-lan proto kernel scope link src 10.0.0.1
10.10.0.0/24 dev wg0 proto kernel scope link src 10.10.0.1
10.10.0.2 dev wg0 proto static scope link
68.********* via 192.168.0.1 dev phy0-sta0 proto static
129.********* via 192.168.0.1 dev phy0-sta0 proto static
143.********* via 192.168.0.1 dev phy0-sta0 proto static
192.168.0.0/24 dev phy0-sta0 proto kernel scope link src 192.168.0.34
ip route show table all
unreachable default table pbr_wan
10.0.0.0/24 dev br-lan table pbr_wan proto kernel scope link src 10.0.0.1
default via 192.168.0.1 dev phy0-sta0 table pbr_wwan
10.0.0.0/24 dev br-lan table pbr_wwan proto kernel scope link src 10.0.0.1
default via 10.10.0.1 dev wg0 table pbr_wg0
10.0.0.0/24 dev br-lan table pbr_wg0 proto kernel scope link src 10.0.0.1
default via 10.2.0.2 dev wg1 table pbr_wg1
10.0.0.0/24 dev br-lan table pbr_wg1 proto kernel scope link src 10.0.0.1
default via 10.200.200.2 dev wg2 table pbr_wg2
10.0.0.0/24 dev br-lan table pbr_wg2 proto kernel scope link src 10.0.0.1
default via 10.188.14.201 dev wg3 table pbr_wg3
10.0.0.0/24 dev br-lan table pbr_wg3 proto kernel scope link src 10.0.0.1
default via 192.168.0.1 dev phy0-sta0 proto static src 192.168.0.34
10.0.0.0/24 dev br-lan proto kernel scope link src 10.0.0.1
10.10.0.0/24 dev wg0 proto kernel scope link src 10.10.0.1
10.10.0.2 dev wg0 proto static scope link
68.********** via 192.168.0.1 dev phy0-sta0 proto static
129..********** via 192.168.0.1 dev phy0-sta0 proto static
143..********** via 192.168.0.1 dev phy0-sta0 proto static
192.168.0.0/24 dev phy0-sta0 proto kernel scope link src 192.168.0.34
local 10.0.0.1 dev br-lan table local proto kernel scope host src 10.0.0.1
broadcast 10.0.0.255 dev br-lan table local proto kernel scope link src 10.0.0.1
local 10.2.0.2 dev wg1 table local proto kernel scope host src 10.2.0.2
local 10.10.0.1 dev wg0 table local proto kernel scope host src 10.10.0.1
broadcast 10.10.0.255 dev wg0 table local proto kernel scope link src 10.10.0.1
local 10.188.14.201 dev wg3 table local proto kernel scope host src 10.188.14.201
local 10.200.200.2 dev wg2 table local proto kernel scope host src 10.200.200.2
local 127.0.0.0/8 dev lo table local proto kernel scope host src 127.0.0.1
local 127.0.0.1 dev lo table local proto kernel scope host src 127.0.0.1
broadcast 127.255.255.255 dev lo table local proto kernel scope link src 127.0.0.1
local 192.168.0.34 dev phy0-sta0 table local proto kernel scope host src 192.168.0.34
broadcast 192.168.0.255 dev phy0-sta0 table local proto kernel scope link src 192.168.0.34
unreachable fd92:dac3:4423::/48 dev lo proto static metric 2147483647 pref medium
fe80::/64 dev eth0 proto kernel metric 256 pref medium
fe80::/64 dev phy0-sta0 proto kernel metric 256 pref medium
fe80::/64 dev phy1-ap0 proto kernel metric 256 pref medium
fe80::/64 dev ifb4phy0-sta0 proto kernel metric 256 pref medium
local ::1 dev lo table local proto kernel metric 0 pref medium
anycast fe80:: dev eth0 table local proto kernel metric 0 pref medium
anycast fe80:: dev phy0-sta0 table local proto kernel metric 0 pref medium
anycast fe80:: dev phy1-ap0 table local proto kernel metric 0 pref medium
anycast fe80:: dev ifb4phy0-sta0 table local proto kernel metric 0 pref medium
local fe80::1659:c0ff:feb6:5ae2 dev eth0 table local proto kernel metric 0 pref medium
local fe80::1659:c0ff:feb6:5ae2 dev phy0-sta0 table local proto kernel metric 0 pref medium
local fe80::1659:c0ff:feb6:5ae3 dev phy1-ap0 table local proto kernel metric 0 pref medium
local fe80::fcc7:2dff:fe2e:eb50 dev ifb4phy0-sta0 table local proto kernel metric 0 pref medium
multicast ff00::/8 dev eth0 table local proto kernel metric 256 pref medium
multicast ff00::/8 dev phy0-sta0 table local proto kernel metric 256 pref medium
multicast ff00::/8 dev phy1-ap0 table local proto kernel metric 256 pref medium
multicast ff00::/8 dev wg0 table local proto kernel metric 256 pref medium
multicast ff00::/8 dev ifb4phy0-sta0 table local proto kernel metric 256 pref medium
multicast ff00::/8 dev wg1 table local proto kernel metric 256 pref medium
multicast ff00::/8 dev wg2 table local proto kernel metric 256 pref medium
multicast ff00::/8 dev wg3 table local proto kernel metric 256 pref medium
ip ru
0: from all lookup local
30000: from all fwmark 0x10000/0xff0000 lookup pbr_wan
30001: from all fwmark 0x20000/0xff0000 lookup pbr_wwan
30002: from all fwmark 0x30000/0xff0000 lookup pbr_wg0
30003: from all fwmark 0x40000/0xff0000 lookup pbr_wg1
30004: from all fwmark 0x50000/0xff0000 lookup pbr_wg2
30005: from all fwmark 0x60000/0xff0000 lookup pbr_wg3
32766: from all lookup main
32767: from all lookup default
wg show
interface: wg0
public key: **************************************************************
private key: (hidden)
listening port: 51820
peer: **************************************************************
preshared key: (hidden)
endpoint: 174.************8:2706
allowed ips: 10.10.0.2/32
latest handshake: 38 minutes, 21 seconds ago
transfer: 10.52 MiB received, 95.03 MiB sent
persistent keepalive: every 25 seconds
interface: wg1
public key: **************************************************************
private key: (hidden)
listening port: 47488
peer: **************************************************************
endpoint: 143.*************:51820
allowed ips: 0.0.0.0/0
latest handshake: 1 minute, 38 seconds ago
transfer: 2.88 GiB received, 285.13 MiB sent
persistent keepalive: every 25 seconds
interface: wg2
public key: **************************************************************
private key: (hidden)
listening port: 46914
peer:**************************************************************
preshared key: (hidden)
endpoint: 129.**************:4500
allowed ips: 0.0.0.0/0
latest handshake: 24 seconds ago
transfer: 731.90 MiB received, 100.29 MiB sent
persistent keepalive: every 25 seconds
interface: wg3
public key: **************************************************************
private key: (hidden)
listening port: 37694
peer: **************************************************************
preshared key: (hidden)
endpoint: 68.*************:1637
allowed ips: 0.0.0.0/0
latest handshake: 1 minute, 18 seconds ago
transfer: 18.56 GiB received, 825.48 MiB sent
persistent keepalive: every 15 seconds
cat /etc/config/pbr
config pbr 'config'
option enabled '1'
option verbosity '0'
option strict_enforcement '1'
option resolver_set 'none'
option ipv6_enabled '0'
list ignored_interface 'vpnserver'
list ignored_interface 'wgserver'
option boot_timeout '30'
option rule_create_option 'add'
option procd_reload_delay '1'
option webui_show_ignore_target '0'
list webui_supported_protocol 'all'
list webui_supported_protocol 'tcp'
list webui_supported_protocol 'udp'
list webui_supported_protocol 'tcp udp'
list webui_supported_protocol 'icmp'
option wg1_dscp '2'
option wg2_dscp '3'
config include
option path '/usr/share/pbr/pbr.user.aws'
option enabled '0'
config include
option path '/usr/share/pbr/pbr.user.netflix'
option enabled '0'
config policy
option name 'DEVICE1'
option src_addr '10.0.0.100'
option interface 'wg1'
config policy
option name 'DEVICE2'
option src_addr '10.0.0.101'
option interface 'wg1'
config policy
option name 'DEVICE3'
option src_addr '10.0.0.102'
option interface 'wg1'
config policy
option name 'DEVICE4'
option src_addr '10.0.0.10'
option interface 'wg3'
config policy
option name 'DEVICE5'
option src_addr '10.0.0.11'
option interface 'wg1'
/etc/init.d/pbr status
============================================================
pbr - environment
pbr 1.1.1-7 running on OpenWrt 23.05.3. WAN (IPv4): wwan/phy0-sta0/192.168.0.1.
============================================================
Dnsmasq version 2.90 Copyright (c) 2000-2024 Simon Kelley
Compile time options: IPv6 GNU-getopt no-DBus UBus no-i18n no-IDN DHCP no-DHCPv6 no-Lua TFTP no-conntrack no-ipset no-nftset no-auth no-cryptohash no-DNSSEC no-ID loop-detect inotify dumpfile
============================================================
pbr chains - policies
chain pbr_forward { # handle 2340
}
chain pbr_input { # handle 2341
}
chain pbr_output { # handle 2342
}
chain pbr_prerouting { # handle 2343
ip dscp 0x02 goto pbr_mark_0x040000 # handle 7574
ip dscp 0x03 goto pbr_mark_0x050000 # handle 7578
ip saddr @pbr_wg1_4_src_ip_cfg046ff5 goto pbr_mark_0x040000 comment "DEVICE1" # handle 7583
ip saddr @pbr_wg1_4_src_ip_cfg056ff5 goto pbr_mark_0x040000 comment "DEVICE2" # handle 7585
ip saddr @pbr_wg1_4_src_ip_cfg066ff5 goto pbr_mark_0x040000 comment "DEVICE3" # handle 7587
ip saddr @pbr_wg3_4_src_ip_cfg076ff5 goto pbr_mark_0x060000 comment "DEVICE4" # handle 7589
ip saddr @pbr_wg1_4_src_ip_cfg086ff5 goto pbr_mark_0x040000 comment "DEVICE5" # handle 7591
ip saddr @pbr_wg1_4_src_ip_cfg096ff5 goto pbr_mark_0x040000 comment "DEVICE6" # handle 7593
ip saddr @pbr_wg1_4_src_ip_cfg0a6ff5 goto pbr_mark_0x040000 comment "DEVICE7" # handle 7595
ip saddr @pbr_wg1_4_src_ip_cfg0b6ff5 goto pbr_mark_0x040000 comment "DEVICE8" # handle 7597
}
chain pbr_postrouting { # handle 2344
}
============================================================
pbr chains - marking
chain pbr_mark_0x010000 { # handle 7562
counter packets 0 bytes 0 meta mark set meta mark & 0xff01ffff | 0x00010000 # handle 7563
return # handle 7564
}
chain pbr_mark_0x020000 { # handle 7565
counter packets 0 bytes 0 meta mark set meta mark & 0xff02ffff | 0x00020000 # handle 7566
return # handle 7567
}
chain pbr_mark_0x030000 { # handle 7568
counter packets 0 bytes 0 meta mark set meta mark & 0xff03ffff | 0x00030000 # handle 7569
return # handle 7570
}
chain pbr_mark_0x040000 { # handle 7571
counter packets 33037 bytes 23832442 meta mark set meta mark & 0xff04ffff | 0x00040000 # handle 7572
return # handle 7573
}
chain pbr_mark_0x050000 { # handle 7575
counter packets 1153 bytes 353504 meta mark set meta mark & 0xff05ffff | 0x00050000 # handle 7576
return # handle 7577
}
chain pbr_mark_0x060000 { # handle 7579
counter packets 559252 bytes 30980900 meta mark set meta mark & 0xff06ffff | 0x00060000 # handle 7580
return # handle 7581
}
============================================================
pbr nft sets
set pbr_wg1_4_src_ip_cfg046ff5 { # handle 7582
type ipv4_addr
flags interval
counter
auto-merge
comment "DEVICE1"
elements = { 10.0.0.100 counter packets 9192 bytes 2523442 }
}
set pbr_wg1_4_src_ip_cfg056ff5 { # handle 7584
type ipv4_addr
flags interval
counter
auto-merge
comment "DEVICE2"
elements = { 10.0.0.101 counter packets 159 bytes 28048 }
}
set pbr_wg1_4_src_ip_cfg066ff5 { # handle 7586
type ipv4_addr
flags interval
counter
auto-merge
comment "DEVICE3"
elements = { 10.0.0.102 counter packets 0 bytes 0 }
}
set pbr_wg3_4_src_ip_cfg076ff5 { # handle 7588
type ipv4_addr
flags interval
counter
auto-merge
comment "DEVICE4"
elements = { 10.0.0.10 counter packets 559282 bytes 30982570 }
}
set pbr_wg1_4_src_ip_cfg086ff5 { # handle 7590
type ipv4_addr
flags interval
counter
auto-merge
comment "DEVICE5"
elements = { 10.0.0.11 counter packets 944 bytes 244938 }
}
set pbr_wg1_4_src_ip_cfg096ff5 { # handle 7592
type ipv4_addr
flags interval
counter
auto-merge
comment "DEVICE6"
elements = { 10.10.0.2 counter packets 593 bytes 73362 }
}
}
set pbr_wg1_4_src_ip_cfg0a6ff5 { # handle 7594
type ipv4_addr
flags interval
counter
auto-merge
comment "DEVICE7"
elements = { 10.10.0.3 counter packets 0 bytes 0 }
}
set pbr_wg1_4_src_ip_cfg0b6ff5 { # handle 7596
type ipv4_addr
flags interval
counter
auto-merge
comment "DEVICE8"
elements = { 10.10.0.4 counter packets 0 bytes 0 }
}
============================================================
IPv4 table 256 route: unreachable default
IPv4 table 256 rule(s):
30000: from all fwmark 0x10000/0xff0000 lookup pbr_wan
IPv4 table 257 route: default via 192.168.0.1 dev phy0-sta0
IPv4 table 257 rule(s):
30001: from all fwmark 0x20000/0xff0000 lookup pbr_wwan
IPv4 table 258 route: default via 10.10.0.1 dev wg0
IPv4 table 258 rule(s):
30002: from all fwmark 0x30000/0xff0000 lookup pbr_wg0
IPv4 table 259 route: default via 10.2.0.2 dev wg1
IPv4 table 259 rule(s):
30003: from all fwmark 0x40000/0xff0000 lookup pbr_wg1
IPv4 table 260 route: default via 10.200.200.2 dev wg2
IPv4 table 260 rule(s):
30004: from all fwmark 0x50000/0xff0000 lookup pbr_wg2
IPv4 table 261 route: default via 10.188.14.201 dev wg3
IPv4 table 261 rule(s):
30005: from all fwmark 0x60000/0xff0000 lookup pbr_wg3