Unable to forward traffic using PBR to WireGuard

Installing and Using OpenWrt Network and Wireless Configuration
1

Hey guys, I've been trying to get pbr to work for a while now with WireGuard, I have 3 zones "LAN", "WAN" and "WireGuard", I enabled forwarding from Wg to LAN and LAN to WG using manual traffic rules and it works well. Now what I want to achieve is to send traffic from one specific device on the network through the wireguard interface. I'm running the router as a DHCP client connected to another router.

Here are the contents of my config files:

/etc/config/dhcp :

config dnsmasq
        option domainneeded '1'
        option localise_queries '1'
        option rebind_protection '1'
        option rebind_localhost '1'
        option local '/lan/'
        option domain 'lan'
        option expandhosts '1'
        option authoritative '1'
        option readethers '1'
        option leasefile '/tmp/dhcp.leases'
        option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
        option localservice '1'
        option ednspacket_max '1232'
        option logdhcp '1'
        option logqueries '1'
        option logfacility '/tmp/dnsmasq.log'
        list address '/status.client/192.168.1.1'
        option serversfile '/var/run/simple-adblock/dnsmasq.servers'
        list server '1.1.1.1'
        list server '8.8.8.8'
        list server '2001:4860:4860::8888'
        list server '2001:4860:4860::8844'
        list server '2606:4700:4700::1111'
        list server '2606:4700:4700::1001'

config dhcp 'lan'
        option interface 'lan'
        option start '100'
        option limit '150'
        option leasetime '12h'
        option dhcpv4 'server'
        option dhcpv6 'server'
        option ra 'server'
        option ra_slaac '1'
        list ra_flags 'managed-config'
        list ra_flags 'other-config'
        list dhcp_option_force '114,http://status.client'

config dhcp 'wan'
        option interface 'wan'
        option ignore '1'

config odhcpd 'odhcpd'
        option maindhcp '0'
        option leasefile '/tmp/hosts/odhcpd'
        option leasetrigger '/usr/sbin/odhcpd-update'
        option loglevel '4'

/etc/config/firewall :

config defaults
        option input 'ACCEPT'
        option output 'ACCEPT'
        option forward 'REJECT'
        option synflood_protect '1'

config rule
        option target 'ACCEPT'
        option src 'lan'
        list proto 'all'
        option dest 'WireGuard'
        option name 'LanToWg'

config rule
        option dest 'lan'
        option target 'ACCEPT'
        option src 'WireGuard'
        list proto 'all'
        option name 'WgToLAN'

config zone
        option name 'lan'
        list network 'lan'
        option input 'ACCEPT'
        option output 'ACCEPT'
        option forward 'ACCEPT'

config zone
        option name 'wan'
        option input 'REJECT'
        option output 'ACCEPT'
        option forward 'REJECT'
        option masq '1'
        option mtu_fix '1'
        list network 'wan'
        list network 'wan6'
        list network '4G'

config zone
        option name 'WireGuard'
        option input 'ACCEPT'
        option output 'ACCEPT'
        option forward 'ACCEPT'
        list network 'wgc_piaJP'
        list network 'wg_server'
        option masq '1'

config rule
        option name 'Allow-DHCP-Renew'
        option src 'wan'
        option proto 'udp'
        option dest_port '68'
        option target 'ACCEPT'
        option family 'ipv4'

config rule
        option name 'Allow-Ping'
        option src 'wan'
        option proto 'icmp'
        option icmp_type 'echo-request'
        option family 'ipv4'
        option target 'ACCEPT'

config rule
        option name 'Allow-IGMP'
        option src 'wan'
        option proto 'igmp'
        option family 'ipv4'
        option target 'ACCEPT'

config rule
        option name 'Allow-DHCPv6'
        option src 'wan'
        option proto 'udp'
        option dest_port '546'
        option family 'ipv6'
        option target 'ACCEPT'

config rule
        option name 'Allow-MLD'
        option src 'wan'
        option proto 'icmp'
        option src_ip 'fe80::/10'
        list icmp_type '130/0'
        list icmp_type '131/0'
        list icmp_type '132/0'
        list icmp_type '143/0'
        option family 'ipv6'
        option target 'ACCEPT'

config rule
        option name 'Allow-ICMPv6-Input'
        option src 'wan'
        option proto 'icmp'
        list icmp_type 'echo-request'
        list icmp_type 'echo-reply'
        list icmp_type 'destination-unreachable'
        list icmp_type 'packet-too-big'
        list icmp_type 'time-exceeded'
        list icmp_type 'bad-header'
        list icmp_type 'unknown-header-type'
        list icmp_type 'router-solicitation'
        list icmp_type 'neighbour-solicitation'
        list icmp_type 'router-advertisement'
        list icmp_type 'neighbour-advertisement'
        option limit '1000/sec'
        option family 'ipv6'
        option target 'ACCEPT'

config rule
        option name 'Allow-ICMPv6-Forward'
        option src 'wan'
        option dest '*'
        option proto 'icmp'
        list icmp_type 'echo-request'
        list icmp_type 'echo-reply'
        list icmp_type 'destination-unreachable'
        list icmp_type 'packet-too-big'
        list icmp_type 'time-exceeded'
        list icmp_type 'bad-header'
        list icmp_type 'unknown-header-type'
        option limit '1000/sec'
        option family 'ipv6'
        option target 'ACCEPT'

config rule
        option name 'Allow-IPSec-ESP'
        option src 'wan'
        option dest 'lan'
        option proto 'esp'
        option target 'ACCEPT'

config rule
        option name 'Allow-ISAKMP'
        option src 'wan'
        option dest 'lan'
        option dest_port '500'
        option proto 'udp'
        option target 'ACCEPT'

config include 'opennds'
        option type 'script'
        option path '/usr/lib/opennds/restart.sh'

config include 'pbr'
        option fw4_compatible '1'
        option type 'script'
        option path '/usr/share/pbr/pbr.firewall.include'

config forwarding
        option src 'lan'
        option dest 'wan'

/etc/config/network :

config interface 'loopback'
        option device 'lo'
        option proto 'static'
        option ipaddr '127.0.0.1'
        option netmask '255.0.0.0'

config globals 'globals'
        option ula_prefix 'fd4c:1810:525a::/48'

config device
        option name 'br-lan'
        option type 'bridge'
        list ports 'lan1'
        list ports 'lan2'
        list ports 'lan3'
        list ports 'lan4'
        list ports 'sfp2'

config interface 'lan'
        option device 'br-lan'
        option proto 'static'
        option ipaddr '192.168.1.1'
        option netmask '255.255.255.0'
        option ip6assign '60'

config device
        option name 'br-wan'
        option type 'bridge'
        list ports 'eth1'
        list ports 'wan'

config device
        option name 'eth1'
        option macaddr '02:77:74:e2:18:12'

config device
        option name 'wan'
        option macaddr '02:77:74:e2:18:12'

config interface 'wan'
        option device 'br-wan'
        option proto 'dhcp'
        option metric '10'

config interface 'wan6'
        option device 'br-wan'
        option proto 'dhcpv6'
        option metric '10'

config interface '4G'
        option proto 'qmi'
        option auth 'none'
        option pdptype 'ipv4v6'
        option device '/dev/cdc-wdm0'
        option metric '50'

config interface 'wgc_piaJP'
        option proto 'wireguard'
        option private_key '<Redacted>'
        list addresses '10.5.147.18/32'

config wireguard_wgc_piaJP
        option description 'wgc_piaJP'
        option public_key '<Redacted>'
        list allowed_ips '0.0.0.0/0'
        option endpoint_host '<Redacted>'
        option endpoint_port '1337'
        option persistent_keepalive '25'
        option route_allowed_ips '1'

/etc/config/pbr :

config pbr 'config'
        option verbosity '2'
        option strict_enforcement '1'
        option resolver_set 'none'
        option ipv6_enabled '0'
        list ignored_interface 'vpnserver'
        list ignored_interface 'wgserver'
        option boot_timeout '30'
        option rule_create_option 'add'
        option procd_reload_delay '1'
        option webui_show_ignore_target '0'
        list webui_supported_protocol 'all'
        list webui_supported_protocol 'tcp'
        list webui_supported_protocol 'udp'
        list webui_supported_protocol 'tcp udp'
        list webui_supported_protocol 'icmp'
        option enabled '1'

config include
        option path '/usr/share/pbr/pbr.user.aws'

config include
        option path '/usr/share/pbr/pbr.user.netflix'

config policy
        option name 'PCtoVPN'
        option src_addr '192.168.1.142'
        option src_port '0-65535'
        option dest_addr '0.0.0.0/0'
        option dest_port '0-65535'
        option interface 'wan'

Output of /etc/init.d/pbr status

============================================================
pbr - environment
pbr 1.0.1-16 running on OpenWrt 1.3.3. WAN (IPv4): wan/br-wan/192.168.29.1.
============================================================
Dnsmasq version 2.88  Copyright (c) 2000-2022 Simon Kelley
Compile time options: IPv6 GNU-getopt no-RTC no-DBus UBus no-i18n no-IDN DHCP DHCPv6 no-Lua TFTP conntrack ipset nftset auth cryptohash DNSSEC no-ID loop-detect inotify dumpfile
============================================================
pbr chains - policies
        chain pbr_forward {
        }
        chain pbr_input {
        }
        chain pbr_output {
        }
        chain pbr_prerouting {
                ip saddr @pbr_wan_4_src_ip_cfg046ff5 ip daddr @pbr_wan_4_dst_ip_cfg046ff5 tcp sport 0-65535 tcp dport 0-65535 goto pbr_mark_0x010000 comment "PCtoVPN"
                ip daddr @pbr_wan_4_dst_ip_user goto pbr_mark_0x010000
                ip saddr @pbr_wan_4_src_ip_user goto pbr_mark_0x010000
                ether saddr @pbr_wan_4_src_mac_user goto pbr_mark_0x010000
                ip daddr @pbr_wgc_piaJP_4_dst_ip_user goto pbr_mark_0x020000
                ip saddr @pbr_wgc_piaJP_4_src_ip_user goto pbr_mark_0x020000
                ether saddr @pbr_wgc_piaJP_4_src_mac_user goto pbr_mark_0x020000
        }
        chain pbr_postrouting {
        }
============================================================
pbr chains - marking
        chain pbr_mark_0x010000 {
                counter packets 8126 bytes 1998280 meta mark set meta mark & 0xff01ffff | 0x00010000
                return
        }
        chain pbr_mark_0x020000 {
                counter packets 0 bytes 0 meta mark set meta mark & 0xff02ffff | 0x00020000
                return
        }
============================================================
pbr nft sets
        set pbr_wan_4_src_ip_cfg046ff5 {
                type ipv4_addr
                flags interval
                counter
                auto-merge
                comment "PCtoVPN"
                elements = { 192.168.1.142 counter packets 8141 bytes 2000762 }
        }
        set pbr_wan_4_dst_ip_cfg046ff5 {
                type ipv4_addr
                flags interval
                counter
                auto-merge
                comment "PCtoVPN"
                elements = { 0.0.0.0/0 counter packets 8142 bytes 2000802 }
        }
        set pbr_wan_4_dst_ip_user {
                type ipv4_addr
                policy memory
                flags interval
                auto-merge
                comment ""
        }
        set pbr_wan_4_src_ip_user {
                type ipv4_addr
                policy memory
                flags interval
                auto-merge
                comment ""
        }
        set pbr_wan_4_src_mac_user {
                type ether_addr
                policy memory
                flags interval
                auto-merge
                comment ""
        }
        set pbr_wgc_piaJP_4_dst_ip_user {
                type ipv4_addr
                policy memory
                flags interval
                auto-merge
                comment ""
        }
        set pbr_wgc_piaJP_4_src_ip_user {
                type ipv4_addr
                policy memory
                flags interval
                auto-merge
                comment ""
        }
        set pbr_wgc_piaJP_4_src_mac_user {
                type ether_addr
                policy memory
                flags interval
                auto-merge
                comment ""
        }
============================================================
IPv4 table 256 route: default via 192.168.29.1 dev br-wan
IPv4 table 256 rule(s):
30000:  from all fwmark 0x10000/0xff0000 lookup pbr_wan
IPv4 table 257 route: default via 10.5.147.18 dev wgc_piaJP
IPv4 table 257 rule(s):
30001:  from all fwmark 0x20000/0xff0000 lookup pbr_wgc_piaJP

Any pointers in the right direction would be greatly appreciated. Thanks!

EDIT: I use mwan3 as well, more details in posts below

2

Could you please explain what is not working because there is a contradiction.
The policy name is PCtoVPN but the selected interface is wan and not wgc_piaJP.

Also remove the port ranges because this way the rule only covers the tcp protocol.

3

Ah my bad, I had unintentionally set that to wan, it's now set to wgc_piaJP, the change is now reflected in the /etc/config/pbr config as well:

config policy
        option name 'PCtoVPN'
        option src_addr '192.168.1.142'
        option dest_addr '0.0.0.0/0'
        option interface 'wgc_piaJP'

I've also removed the port ranges as you can see, but when I try to visit a website (from the PC at say https://ifconfig.me/) it still shows me the WAN IP of original WAN connection not the VPN.

I also just want to add for posterity that I ran a curl request to the same domain using the following command (from the router) and it shows me the VPN IP

curl --interface wgc_piaJP ifconfig.me
4

If you disable the pbr service does all the traffic go through the wireguard interface?

This line bothers me. Default route via the IP address of the local wg interface...

Shouldn't be a problem, but change the suffix to /24 to see if it makes a difference.

5

With the pbr service disabled, I am still going through the WAN interface (from both the router and the PC)

I tried changing the address to 10.5.147.18/24 in the interface and it still doesn't route through the VPN

6

Are you sure there is a successful handshake?

Check the output of wg show.

Run ip route show.

The first two lines should look like this:

default dev wgc_piaJP scope link
default via 192.168.29.1 dev br-wan src <someIP> metric 10
7

Positive about the successful handshake, I am able to use the wgc_piaJP interface to make pings and curl requests sucessfully

Output from wg show:

interface: wgc_piaJP
  public key: <redacted>
  private key: (hidden)
  listening port: 44597

peer: <redacted>
  endpoint: x.x.x.x:1337
  allowed ips: 0.0.0.0/0
  latest handshake: 16 seconds ago
  transfer: 184 B received, 520 B sent
  persistent keepalive: every 25 seconds

Output of ip route show:

root@wrt:~# ip route show
default dev wgc_piaJP proto static scope link
default via 192.168.29.1 dev br-wan proto static src 192.168.29.21 metric 10
default via 100.75.31.57 dev wwan0 proto static src 100.75.31.58 metric 50
10.5.147.0/24 dev wgc_piaJP proto kernel scope link src 10.5.147.18
100.75.31.56/30 dev wwan0 proto static scope link metric 50
138.199.39.6 via 192.168.29.1 dev br-wan proto static metric 10
138.199.39.6 via 100.75.31.57 dev wwan0 proto static metric 50
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1
192.168.29.0/24 dev br-wan proto static scope link metric 10

EDIT: 138.199.39.6 -> is my VPN IP

8

The routing table looks correct to me.
Sorry, I ran out of ideas.
Maybe someone else can help.

@trendy @psherman @lleachii any clues?

1 Like
9

That's not how routes look when I make them. I should note, personally, I don't use the PBR package, I just make IP routes and rules.

I assisted someone the other day with routing a subnet, so there's no difference, instead you use /32 for the subnet. In fact, this is how I setup single IPs to use various VPNs - then I merely setup connection profiles on the computer and switch as needed - so the OP's use case is the same as mine.

# in /etc/config/network

config route                      
        option interface 'wgc_piaJP'
        option target '0.0.0.0/0'
        option table '1' 

config rule #<--- ensures everything routes to IP when DST
        option dest '192.168.1.142/32'
        option priority '1'
        option lookup 'main'

config rule
        option src '192.168.1.142/32'                       
        option dest '0.0.0.0/0'
        option priority '2' #order in IP_rules
        option lookup '1' #table_assigned

This is how I would make such a config.

10

The config works! But the problem is that now all the devices in my network goes through the VPN interface for some reason which is not desirable

11

Remove this, then.

13

Weird I tried doing that and even gave my device a reboot, it still routes all devices through the vpn

config interface 'wgc_piaJP'
        option proto 'wireguard'
        option private_key '<redacted>'
        list addresses '10.5.147.18/24'

config wireguard_wgc_piaJP
        option description 'wgc_piaJP'
        option public_key '<redacted>'
        list allowed_ips '0.0.0.0/0'
        option endpoint_host '<redacted>'
        option endpoint_port '1337'
        option persistent_keepalive '25'

config route
        option interface 'wgc_piaJP'
        option target '0.0.0.0/0'
        option table '1'

config rule
        option dest '192.168.1.137/32'
        option priority '1'
        option lookup 'main'

config rule
        option src '192.168.1.137/32'
        option dest '0.0.0.0/0'
        option priority '2'
        option lookup '1'
14

Provide the output of:

ip route show
ip route show table 1
ip rule
1 Like
15

Output of ip route show :

default via 192.168.29.1 dev br-wan proto static src 192.168.29.21 metric 10
default via 100.75.31.57 dev wwan0 proto static src 100.75.31.58 metric 50
10.5.147.0/24 dev wgc_piaJP proto kernel scope link src 10.5.147.18
100.75.31.56/30 dev wwan0 proto static scope link metric 50
138.199.39.6 via 192.168.29.1 dev br-wan proto static metric 10
138.199.39.6 via 100.75.31.57 dev wwan0 proto static metric 50
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1
192.168.29.0/24 dev br-wan proto static scope link metric 10

Output of ip route show table 1

default dev wgc_piaJP proto static scope link
default via 192.168.29.1 dev br-wan proto static src 192.168.29.21 metric 10
10.5.147.0/24 dev wgc_piaJP proto kernel scope link src 10.5.147.18
138.199.39.6 via 192.168.29.1 dev br-wan proto static metric 10
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1
192.168.29.0/24 dev br-wan proto static scope link metric 10

Output of ip rule

0:      from all lookup local
1:      from all to 192.168.1.137 lookup main
2:      from 192.168.1.137 lookup 1
1001:   from all iif br-wan lookup 1
1003:   from all iif wwan0 lookup 3
2001:   from all fwmark 0x100/0x3f00 lookup 1
2003:   from all fwmark 0x300/0x3f00 lookup 3
2061:   from all fwmark 0x3d00/0x3f00 blackhole
2062:   from all fwmark 0x3e00/0x3f00 unreachable
3001:   from all fwmark 0x100/0x3f00 unreachable
3003:   from all fwmark 0x300/0x3f00 unreachable
32766:  from all lookup main
32767:  from all lookup default
16

Wow...It seems your table 1 is already in use and populated. It should only contain one line (i.e. from the config I gave you):

It seems you already have a priories configured. Nonetheless, Priorities 1 and 2 are OK.

  • FYI, these settings cannot be duplicated - they must be unique
  • It seems you already have routes and rules setup

(There also seems to be a table No. 3.)

Wow...you already have policies setup. The No. 1001 rule is your issue. I donno how it got there; but it needs to be removed to solve your issue.

17

You have also installed mwan3!

/etc/init.d/mwan3 stop
/etc/init.d/mwan3 disable
1 Like
18

OK, that makes sense!

1 Like
19

Unfortunately uninstalling mwan3 isn't an option for me, but .. setting a different table number is and that seems to get the job done for me. My PC now goes through the VPN while all other devices goes through the wan. Is there any reason why this could be considered bad? (I'm not sure I'm pretty new to this stuff)

Thank you for all the guidance!

20

I'm not sure of your exact inquiry.

The only bad thing was you wanted to setup a single IP to use VPN; and some installed package prevented that - if that's what you're asking.

21

I was just wondering was there a reason why you both suggested to remove mwan3 instead of using a different table number?