Help! Policy Based Routing requiring multiple restarts per day

Magenta7602 · March 6, 2025, 3:18pm

I have PBR set to bypass my VPN (proton) for my guest network 192.168.10.0/24 but it to stops working and requires a restart every few hours or when the WAN interface renewes its IP.

I have tried using cron jobs to restart it every so oftern but that only fixes the issue temporarily.


]0;root@router: ~aroot@router:~# ubus call system board
{
"kernel": "6.6.73",
"hostname": "router",
"system": "ARMv8 Processor rev 4",
"model": "FriendlyElec NanoPi R4S",
"board_name": "friendlyarm,nanopi-r4s",
"rootfs_type": "squashfs",
"release": {
"distribution": "OpenWrt",
"version": "24.10.0",
"revision": "r28427-6df0e3d02a",
"target": "rockchip/armv8",
"description": "OpenWrt 24.10.0 r28427-6df0e3d02a",
"builddate": "1738624177"
}
}
e]0;root@router: ~aroot@router:~# uci export dhcp
package dhcp

config dnsmasq
option domainneeded '1'
option localise_queries '1'
option rebind_protection '1'
option rebind_localhost '1'
option local '/lan/'
option domain 'lan'
option expandhosts '1'
option cachesize '1000'
option authoritative '1'
option readethers '1'
option leasefile '/tmp/dhcp.leases'
option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
option localservice '1'
option ednspacket_max '1232'

config dhcp 'lan'
option interface 'lan'
option start '100'
option limit '150'
option leasetime '12h'
option dhcpv4 'server'
option force '1'

config dhcp 'wan'
option interface 'wan'
option ignore '1'

config odhcpd 'odhcpd'
option maindhcp '0'
option leasefile '/tmp/hosts/odhcpd'
option leasetrigger '/usr/sbin/odhcpd-update'
option loglevel '4'

config host
option name 'switch'
option ip '192.168.1.252'
option leasetime 'infinite'

config host
option name 'kitchen'
option ip '192.168.1.2'
option leasetime 'infinite'

config host
option name 'basement'
option ip '192.168.1.3'
option leasetime 'infinite'

config host
option name 'pihole'
option ip '192.168.1.254'
option leasetime 'infinite'

config dhcp 'guest'
option interface 'guest'
option start '100'
option limit '150'
option leasetime '12h'
list dhcp_option '6,192.168.1.254'

config host
option name 'plex'
option ip '192.168.1.253'
option leasetime 'infinite'

config host
option name 'work laptop'
option ip '192.168.10.196'
option leasetime 'infinite'

config host
option name 'home pc'
option ip '192.168.1.140'
option leasetime 'infinite'

e]0;root@router: ~aroot@router:~# uci export firewall
package firewall

config defaults
option syn_flood '1'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'

config zone
option name 'lan'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
list network 'lan'
option mtu_fix '1'

config zone
option name 'wan'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
option masq '1'
option mtu_fix '1'
list network 'wan'
list network 'wan6'

config rule
option name 'Allow-DHCP-Renew'
option src 'wan'
option proto 'udp'
option dest_port '68'
option target 'ACCEPT'
option family 'ipv4'

config rule
option name 'Allow-Ping'
option src 'wan'
option proto 'icmp'
option icmp_type 'echo-request'
option family 'ipv4'
option target 'ACCEPT'

config rule
option name 'Allow-IGMP'
option src 'wan'
option proto 'igmp'
option family 'ipv4'
option target 'ACCEPT'

config rule
option name 'Allow-DHCPv6'
option src 'wan'
option proto 'udp'
option dest_port '546'
option family 'ipv6'
option target 'ACCEPT'

config rule
option name 'Allow-MLD'
option src 'wan'
option proto 'icmp'
option src_ip 'fe80::/10'
list icmp_type '130/0'
list icmp_type '131/0'
list icmp_type '132/0'
list icmp_type '143/0'
option family 'ipv6'
option target 'ACCEPT'

config rule
option name 'Allow-ICMPv6-Input'
option src 'wan'
option proto 'icmp'
list icmp_type 'echo-request'
list icmp_type 'echo-reply'
list icmp_type 'destination-unreachable'
list icmp_type 'packet-too-big'
list icmp_type 'time-exceeded'
list icmp_type 'bad-header'
list icmp_type 'unknown-header-type'
list icmp_type 'router-solicitation'
list icmp_type 'neighbour-solicitation'
list icmp_type 'router-advertisement'
list icmp_type 'neighbour-advertisement'
option limit '1000/sec'
option family 'ipv6'
option target 'ACCEPT'

config rule
option name 'Allow-ICMPv6-Forward'
option src 'wan'
option dest '*'
option proto 'icmp'
list icmp_type 'echo-request'
list icmp_type 'echo-reply'
list icmp_type 'destination-unreachable'
list icmp_type 'packet-too-big'
list icmp_type 'time-exceeded'
list icmp_type 'bad-header'
list icmp_type 'unknown-header-type'
option limit '1000/sec'
option family 'ipv6'
option target 'ACCEPT'

config rule
option name 'Allow-IPSec-ESP'
option src 'wan'
option dest 'lan'
option proto 'esp'
option target 'ACCEPT'

config rule
option name 'Allow-ISAKMP'
option src 'wan'
option dest 'lan'
option dest_port '500'
option proto 'udp'
option target 'ACCEPT'

config zone
option name 'guest'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
list network 'guest'

config forwarding
option src 'guest'
option dest 'wan'

config rule
option name 'Allow-pihole-Guest'
option src 'guest'
option dest 'lan'
list dest_ip '192.168.1.254'
option dest_port '53'
option target 'ACCEPT'

config rule
option name 'Allow-DNS-Guest'
option src 'guest'
option dest_port '53'
option target 'ACCEPT'

config rule
option name 'Allow-Plex-Guest'
option src 'guest'
option dest 'lan'
list dest_ip '192.168.1.253'
option target 'ACCEPT'

config rule
option name 'Allow-DHCP-Guest'
list proto 'udp'
option src 'guest'
option dest_port '67'
option target 'ACCEPT'

config rule
option name 'Block-Guest-from-LAN'
list proto 'all'
option src 'guest'
option dest 'lan'
list dest_ip '192.168.1.0/24'
option target 'REJECT'

config include 'pbr'
option fw4_compatible '1'
option type 'script'
option path '/usr/share/pbr/firewall.include'

config zone
option name 'vpn'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
option masq '1'
list network 'proton0'

config forwarding
option src 'lan'
option dest 'vpn'

config redirect
option dest 'lan'
option target 'DNAT'
option name 'port-forward'
option src 'vpn'
option src_dport '63965'
option dest_ip '192.168.1.140'
option dest_port '63965'

e]0;root@router: ~aroot@router:~# uci export network
package network

config interface 'loopback'
option device 'lo'
option proto 'static'
option ipaddr '127.0.0.1'
option netmask '255.0.0.0'

config globals 'globals'
option ula_prefix 'fd6e:64de:12d7::/48'
option packet_steering '1'

config device
option name 'br-lan'
option type 'bridge'
list ports 'eth1'

config device
option name 'eth1'
option macaddr 'fe:0f:e7:16:19:58'

config interface 'lan'
option device 'br-lan'
option proto 'static'
option ipaddr '192.168.1.1'
option netmask '255.255.255.0'
option ip6assign '60'

config device
option name 'eth0'
option macaddr 'fc:0f:e7:16:19:58'

config interface 'wan'
option device 'eth0'
option proto 'dhcp'
option peerdns '0'
list dns '10.2.0.1'

config interface 'wan6'
option device 'eth0'
option proto 'dhcpv6'

config device
option type 'bridge'
option name 'br-guest'
option bridge_empty '1'

config interface 'guest'
option proto 'static'
option device 'eth1.10'
option ipaddr '192.168.10.1'
option netmask '255.255.255.0'

config interface 'proton0'
option proto 'wireguard'
list addresses '10.2.0.2/32'
list dns '10.2.0.1'

config wireguard_proton0
option description 'router-US-IL-252.conf'
list allowed_ips '0.0.0.0/0'
option endpoint_host '89.187.180.27'
option endpoint_port '51820'
option route_allowed_ips '1'

e]0;root@router: ~aroot@router:~# uci export pbr
package pbr

config pbr 'config'
option enabled '1'
option verbosity '2'
option strict_enforcement '1'
option resolver_set 'none'
list resolver_instance '*'
option ipv6_enabled '0'
list ignored_interface 'vpnserver'
option boot_timeout '30'
option rule_create_option 'add'
option procd_reload_delay '1'
option webui_show_ignore_target '0'
option nft_rule_counter '1'
option nft_set_auto_merge '1'
option nft_set_counter '1'
option nft_set_flags_interval '1'
option nft_set_flags_timeout '0'
option nft_set_policy 'performance'
list webui_supported_protocol 'all'
list webui_supported_protocol 'tcp'
list webui_supported_protocol 'udp'
list webui_supported_protocol 'tcp udp'
list webui_supported_protocol 'icmp'

config include
option path '/usr/share/pbr/pbr.user.aws'
option enabled '0'

config include
option path '/usr/share/pbr/pbr.user.netflix'
option enabled '0'

config dns_policy
option name 'Redirect Local IP DNS'
option src_addr '192.168.1.5'
option dest_dns '1.1.1.1'
option enabled '0'

config policy
option name 'Ignore Local Requests'
option interface 'ignore'
option dest_addr '10.0.0.0/24 10.0.1.0/24 192.168.100.0/24 192.168.1.0/24'
option enabled '0'

config policy
option name 'Plex/Emby Local Server'
option interface 'wan'
option src_port '8096 8920 32400'
option enabled '0'

config policy
option name 'Plex/Emby Remote Servers'
option interface 'wan'
option dest_addr 'plex.tv my.plexapp.com emby.media app.emby.media tv.emby.media'
option enabled '0'

config policy
option name 'Guest-Bypass-VPN'
option src_addr '192.168.10.0/24'
option interface 'wan'

config dns_policy
option name 'guest-dns'
option src_addr '192.168.10.0/24'
option dest_dns '192.168.1.254'

e]0;root@router: ~aroot@router:~# /etc/init.d/pbr status

pbr - environment
pbr 1.1.8-r10 running on OpenWrt 24.10.0.

Dnsmasq version 2.90 Copyright (c) 2000-2024 Simon Kelley
Compile time options: IPv6 GNU-getopt no-DBus UBus no-i18n no-IDN DHCP no-DHCPv6 no-Lua TFTP no-conntrack no-ipset no-nftset no-auth no-cryptohash no-DNSSEC no-ID loop-detect inotify dumpfile

pbr fw4 nft file: /usr/share/nftables.d/ruleset-post/30-pbr.nft
add chain inet fw4 pbr_mark_0x010000
add rule inet fw4 pbr_mark_0x010000 counter mark set mark and 0xff00ffff xor 0x010000
add rule inet fw4 pbr_mark_0x010000 return
add chain inet fw4 pbr_mark_0x020000
add rule inet fw4 pbr_mark_0x020000 counter mark set mark and 0xff00ffff xor 0x020000
add rule inet fw4 pbr_mark_0x020000 return
add rule inet fw4 pbr_prerouting ip saddr { 192.168.10.0/24 } counter goto pbr_mark_0x010000 comment "Guest-Bypass-VPN"
add rule inet fw4 pbr_dstnat ip saddr { 192.168.10.0/24 } counter meta nfproto ipv4 tcp dport 53 dnat ip to 192.168.1.254:53 comment "guest-dns"
add rule inet fw4 pbr_dstnat ip saddr { 192.168.10.0/24 } counter meta nfproto ipv4 udp dport 53 dnat ip to 192.168.1.254:53 comment "guest-dns"

pbr chains - policies
chain pbr_forward { # handle 48
}
chain pbr_input { # handle 49
}
chain pbr_output { # handle 50
}
chain pbr_postrouting { # handle 52
}
chain pbr_prerouting { # handle 51
ip saddr 192.168.10.0/24 counter packets 29038 bytes 2112068 goto pbr_mark_0x010000 comment "Guest-Bypass-VPN" # handle 1957
}
chain pbr_dstnat { # handle 47
ip saddr 192.168.10.0/24 counter packets 46 bytes 5001 meta nfproto ipv4 tcp dport 53 dnat ip to 192.168.1.254:53 comment "guest-dns" # handle 1958
ip saddr 192.168.10.0/24 counter packets 46 bytes 5001 meta nfproto ipv4 udp dport 53 dnat ip to 192.168.1.254:53 comment "guest-dns" # handle 1959
}

pbr chains - marking
chain pbr_mark_0x010000 { # handle 1951
counter packets 29038 bytes 2112068 meta mark set meta mark & 0xff01ffff | 0x00010000 # handle 1952
return # handle 1953
}
chain pbr_mark_0x020000 { # handle 1954
counter packets 0 bytes 0 meta mark set meta mark & 0xff02ffff | 0x00020000 # handle 1955
return # handle 1956
}

pbr nft sets

pbr tables & routing
IPv4 table 256 pbr_wan route:
default via 69.180.160.1 dev eth0
IPv4 table 256 pbr_wan rule(s):
30000:from all fwmark 0x10000/0xff0000 lookup pbr_wan

IPv4 table 257 pbr_proton0 route:
default via 10.2.0.2 dev proton0
IPv4 table 257 pbr_proton0 rule(s):
29998:from all fwmark 0x20000/0xff0000 lookup pbr_proton0

e]0;root@router: ~aroot@router:~# /etc/init.d/pbr reload
Using wan interface (on_start): wan e[0;32m[✓]e[0m
Found wan gateway (on_start): xx.xxx.xxx.x e[0;32m[✓]e[0m
Setting up routing for 'wan/eth0/xx.xxx.xxx.x' e[0;32m[✓]e[0m
Setting up routing for 'proton0/10.2.0.2' e[0;32m[✓]e[0m
Routing 'Guest-Bypass-VPN' via wan e[0;32m[✓]e[0m
Routing 'guest-dns' DNS to 192.168.1.254 e[0;32m[✓]e[0m
Installing fw4 nft file e[0;32m[✓]e[0m
Setting interface trigger for wan e[0;32m[✓]e[0m
Setting interface trigger for proton0 e[0;32m[✓]e[0m

pbr 1.1.8-r10 monitoring interfaces: wan proton0
pbr 1.1.8-r10 (fw4 nft file mode) started with gateways:
wan/eth0/xx.xxx.xxx.x
proton0/10.2.0.2 e[0;32m[✓]e[0m
e]0;root@router: ~aroot@router:~# /etc/init.d/pbr status

pbr - environment
pbr 1.1.8-r10 running on OpenWrt 24.10.0.

Dnsmasq version 2.90 Copyright (c) 2000-2024 Simon Kelley
Compile time options: IPv6 GNU-getopt no-DBus UBus no-i18n no-IDN DHCP no-DHCPv6 no-Lua TFTP no-conntrack no-ipset no-nftset no-auth no-cryptohash no-DNSSEC no-ID loop-detect inotify dumpfile

pbr fw4 nft file: /usr/share/nftables.d/ruleset-post/30-pbr.nft
add chain inet fw4 pbr_mark_0x010000
add rule inet fw4 pbr_mark_0x010000 counter mark set mark and 0xff00ffff xor 0x010000
add rule inet fw4 pbr_mark_0x010000 return
add chain inet fw4 pbr_mark_0x020000
add rule inet fw4 pbr_mark_0x020000 counter mark set mark and 0xff00ffff xor 0x020000
add rule inet fw4 pbr_mark_0x020000 return
add rule inet fw4 pbr_prerouting ip saddr { 192.168.10.0/24 } counter goto pbr_mark_0x010000 comment "Guest-Bypass-VPN"
add rule inet fw4 pbr_dstnat ip saddr { 192.168.10.0/24 } counter meta nfproto ipv4 tcp dport 53 dnat ip to 192.168.1.254:53 comment "guest-dns"
add rule inet fw4 pbr_dstnat ip saddr { 192.168.10.0/24 } counter meta nfproto ipv4 udp dport 53 dnat ip to 192.168.1.254:53 comment "guest-dns"

pbr chains - policies
chain pbr_forward { # handle 48
}
chain pbr_input { # handle 49
}
chain pbr_output { # handle 50
}
chain pbr_postrouting { # handle 52
}
chain pbr_prerouting { # handle 51
ip saddr 192.168.10.0/24 counter packets 204 bytes 17218 goto pbr_mark_0x010000 comment "Guest-Bypass-VPN" # handle 2085
}
chain pbr_dstnat { # handle 47
ip saddr 192.168.10.0/24 counter packets 1 bytes 60 meta nfproto ipv4 tcp dport 53 dnat ip to 192.168.1.254:53 comment "guest-dns" # handle 2086
ip saddr 192.168.10.0/24 counter packets 1 bytes 60 meta nfproto ipv4 udp dport 53 dnat ip to 192.168.1.254:53 comment "guest-dns" # handle 2087
}

pbr chains - marking
chain pbr_mark_0x010000 { # handle 2079
counter packets 274 bytes 20921 meta mark set meta mark & 0xff01ffff | 0x00010000 # handle 2080
return # handle 2081
}
chain pbr_mark_0x020000 { # handle 2082
counter packets 0 bytes 0 meta mark set meta mark & 0xff02ffff | 0x00020000 # handle 2083
return # handle 2084
}

pbr nft sets

pbr tables & routing
IPv4 table 256 pbr_wan route:
default via xx.xxx.xxx.x dev eth0
IPv4 table 256 pbr_wan rule(s):
30000:from all fwmark 0x10000/0xff0000 lookup pbr_wan
 
IPv4 table 257 pbr_proton0 route:
default via 10.2.0.2 dev proton0
IPv4 table 257 pbr_proton0 rule(s):
29998:from all fwmark 0x20000/0xff0000 lookup pbr_proton0

pavelgl · March 6, 2025, 6:15pm

So the situation has gone from bad to worse. In your previous thread you said you need to restart the service once a day.

Have you checked what exactly stops working? Routing over wan, DNS resolution?

Possible workarounds:

Add a command to /etc/udhcpc.user to reload/restart the pbr service on any udhcpc event.

Do not use the pbr package at all.

#/etc/config/network

config wireguard_proton0
	    ...
	    list allowed_ips '0.0.0.0/0'
	    option route_allowed_ips '0'

config rule
        option in 'lan'
        option src '192.168.1.0/24'
        option lookup '100'

config route
        option target '0.0.0.0/0'
        option interface 'proton0'
        option table '100'

And what is the upstream resolver used by the Pi-hole, because it looks like the router is set to use only Proton DNS.

Magenta7602 · March 6, 2025, 6:42pm

Yeah a few other issues have occured
-it does not always fully restart (ie I have to go in and start the service again)
-it requires a restart when the WAN ip is renewed from the ISP (this happens like every 3 days)
-it seems to give some devices (laptops for example) issues if they are trying to reconnect after being on a different network.

from what I am seeing its the routing over wan. Devices will not have connectivity at all until PBR restarts.

I think this is my preferred option if it still provides the same end goal. So i will try editing #/etc/config/network and report back.

pi-hole is configured to use cloudflared for DoH via Mullvad

I appreciate you responding. Felt bad about making a 2nd post regarding the issue but also didnt want to constantly bump a thread. Someone can feel free to let me know what the proper forum etiquette is.

Magenta7602 · March 6, 2025, 11:55pm

This did not work. The guest network bypassed the VPN but the home network stopped working.

I added the below to /etc/udhcpc.user and removed my scheduled task to restart pbr every 12 hours. I also reinstalled pbr with a fresh conffile.


#!/bin/sh

# Reload/restart PBR service
/etc/init.d/pbr reload

# Log the event
logger -t pbr "Reloading PBR due to udhcpc event: $1"

heres hoping

stangri · March 9, 2025, 12:58am

This should happen automatically thru a trigger the pbr sets for wan. Check the logs for pbr events after the wan ip is updated.

How are you testing this? Have you tried rebooting the devices (laptops for example) when they connect after being on a different network? Not saying this is a way to go, just asking if you have tried that.

Magenta7602 · March 9, 2025, 5:15pm

Ill look out for it, last time it happened the service did not fully restart (ie I had to manually start the service). The incident prompted me to make this post. This type of renewal only happens once every few days.

I never specifically tested for this use case, but I did have multiple times where laptops had to either be restarted or reset their adapters to connect to the guest network.

pbr has been working fine ever since adding a command to /etc/udhcpc.user to reload/restart the pbr service on any udhcpc event. Its only been 3 days, and while I can see from the logs the command works I can't say with certainty that it fixed any of these issues. It seems just as likely to me that all it needed was a fresh install.

system · March 19, 2025, 5:16pm

This topic was automatically closed 10 days after the last reply. New replies are no longer allowed.