Policy Base Routing with PBR not working

Installing and Using OpenWrt Network and Wireless Configuration
1

Hi,

I have set-up PBR but somehow it only seems to work for a few seconds after the router is up (I can see that because I curl ifconfig.me, I get the VPN IP for a few seconds). Then none of the PBR rules are applied even if I restart the PBR service.

I am thinking that maybe there is another package/config that is blocking PBR?
Could it be that using dnscrypt-proxy2 creates the issue?

System Board 
{
	"kernel": "5.15.162",
	"hostname": "hibiscus",
	"system": "ARMv7 Processor rev 0 (v7l)",
	"model": "Cisco Meraki MX65",
	"board_name": "meraki,mx65",
	"rootfs_type": "squashfs",
	"release": {
		"distribution": "OpenWrt",
		"version": "23.05.4",
		"revision": "r24012-d8dd03c46f",
		"target": "bcm53xx/generic",
		"description": "OpenWrt 23.05.4 r24012-d8dd03c46f"
	}
}
dhcp 
config dnsmasq
	option domainneeded '1'
	option boguspriv '1'
	option filterwin2k '0'
	option localise_queries '1'
	option rebind_protection '1'
	option rebind_localhost '1'
	option local '/home/'
	option domain 'home'
	option expandhosts '1'
	option nonegcache '0'
	option cachesize '0'
	option authoritative '1'
	option readethers '1'
	option leasefile '/tmp/dhcp.leases'
	option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
	option nonwildcard '1'
	option localservice '1'
	option ednspacket_max '1232'
	option filter_aaaa '0'
	option filter_a '0'
	option confdir '/tmp/dnsmasq.d'
	option noresolv '1'
	option localuse '1'
	list server '127.0.0.53'

config dhcp 'lan'
	option interface 'lan'
	option start '100'
	option limit '150'
	option leasetime '12h'
	option dhcpv4 'server'
	option dhcpv6 'server'
	option ra 'server'
	option ra_slaac '1'
	list ra_flags 'managed-config'
	list ra_flags 'other-config'

config dhcp 'wan'
	option interface 'wan'
	option ignore '1'

config odhcpd 'odhcpd'
	option maindhcp '0'
	option leasefile '/tmp/hosts/odhcpd'
	option leasetrigger '/usr/sbin/odhcpd-update'
	option loglevel '4'

config dhcp 'guest'
	option interface 'guest'
	option start '100'
	option limit '150'
	option leasetime '12h'
	option dhcpv4 'server'
	option dhcpv6 'server'
	option ra 'server'
	list ra_flags 'managed-config'
	list ra_flags 'other-config'
Network 
package network

config interface 'loopback'
	option device 'lo'
	option proto 'static'
	option ipaddr '127.0.0.1'
	option netmask '255.0.0.0'

config globals 'globals'
	option ula_prefix 'fda6:a8dc:257f::/48'
	option packet_steering '1'

config device
	option name 'br-lan'
	option type 'bridge'
	list ports 'lan3'
	list ports 'lan4'
	list ports 'lan5'
	list ports 'lan6'
	list ports 'lan7'
	list ports 'lan8'
	list ports 'lan9'
	list ports 'lan10'
	list ports 'lan11'
	list ports 'lan12'

config interface 'lan'
	option device 'br-lan.14'
	option proto 'static'
	option ipaddr '192.168.5.1'
	option netmask '255.255.255.0'
	option ip6assign '60'
	option vlan_filtering '1'

config device
	option name 'br-wan'
	option type 'bridge'
	list ports 'wan1'
	list ports 'wan2'

config device
	option name 'wan1'
	option macaddr 'xx:xx:xx:xx:xx:xx'

config device
	option name 'wan2'
	option macaddr 'xx:xx:xx:xx:xx:xx'

config interface 'wan'
	option device 'br-wan'
	option proto 'dhcp'
	list dns '8.8.8.8'
	list dns '8.8.4.4'
	option peerdns '0'

config interface 'wan6'
	option device 'br-wan'
	option proto 'dhcpv6'
	list dns '2001:4860:4860::8844'
	list dns '8.8.4.4'
	option peerdns '0'

config bridge-vlan
	option device 'br-lan'
	option vlan '14'
	list ports 'lan3:u*'
	list ports 'lan4:u*'
	list ports 'lan5:u*'
	list ports 'lan6:u*'
	list ports 'lan7:u*'
	list ports 'lan8:u*'
	list ports 'lan9:u*'
	list ports 'lan10:t*'
	list ports 'lan11:t*'
	list ports 'lan12:u*'

config bridge-vlan
	option device 'br-lan'
	option vlan '21'
	list ports 'lan10:t'
	list ports 'lan11:t'

config interface 'guest'
	option device 'br-lan.21'
	option proto 'static'
	option ipaddr '192.168.9.1'
	option netmask '255.255.255.0'
	option ip6assign '60'

config interface 'fr'
	option proto 'none'
	option delegate '0'
	option device 'tun0'

config interface 'hk'
	option proto 'none'
	option delegate '0'
	option device 'tun1'

config interface 'uk'
	option proto 'none'
	option delegate '0'
	option device 'tun2'
PBR 
package pbr

config pbr 'config'
	option enabled '1'
	option verbosity '2'
	option strict_enforcement '1'
	option resolver_set 'dnsmasq.nftset'
	option ipv6_enabled '0'
	list ignored_interface 'vpnserver'
	list ignored_interface 'wgserver'
	option boot_timeout '30'
	option rule_create_option 'add'
	option procd_reload_delay '1'
	option webui_show_ignore_target '0'
	list webui_supported_protocol 'all'
	list webui_supported_protocol 'tcp'
	list webui_supported_protocol 'udp'
	list webui_supported_protocol 'tcp udp'
	list webui_supported_protocol 'icmp'
	list supported_interface 'tun0'
	list supported_interface 'tun1'
	list supported_interface 'tun2'

config policy 'ignore'
	option name 'ignore'
	option interface 'ignore'
	option dest_addr '192.168.0.0/16'
	option enabled '1'

config policy 'fr_france'
	option name 'fr_france'
	option interface 'fr'
	option dest_addr 'ifconfig.me fr'
	option enabled '1'

config policy 'uk_default'
	option name 'uk_default'
	option proto 'tcp udp'
	option interface 'uk'
	option src_addr '0.0.0.0/0'
	option enabled '1'
firewall 
config defaults
	option syn_flood '1'
	option input 'REJECT'
	option output 'ACCEPT'
	option forward 'REJECT'

config zone
	option name 'lan'
	list network 'lan'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'

config zone
	option name 'wan'
	list network 'wan'
	list network 'wan6'
	option input 'REJECT'
	option output 'ACCEPT'
	option forward 'REJECT'
	option masq '1'
	option mtu_fix '1'
	list device 'tun0'
	list device 'tun1'
	list device 'tun2'
	list device 'tun3'

config forwarding
	option src 'lan'
	option dest 'wan'

config rule
	option name 'Allow-DHCP-Renew'
	option src 'wan'
	option proto 'udp'
	option dest_port '68'
	option target 'ACCEPT'
	option family 'ipv4'

config rule
	option name 'Allow-Ping'
	option src 'wan'
	option proto 'icmp'
	option icmp_type 'echo-request'
	option family 'ipv4'
	option target 'ACCEPT'

config rule
	option name 'Allow-IGMP'
	option src 'wan'
	option proto 'igmp'
	option family 'ipv4'
	option target 'ACCEPT'

config rule
	option name 'Allow-DHCPv6'
	option src 'wan'
	option proto 'udp'
	option dest_port '546'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-MLD'
	option src 'wan'
	option proto 'icmp'
	option src_ip 'fe80::/10'
	list icmp_type '130/0'
	list icmp_type '131/0'
	list icmp_type '132/0'
	list icmp_type '143/0'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-ICMPv6-Input'
	option src 'wan'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	list icmp_type 'router-solicitation'
	list icmp_type 'neighbour-solicitation'
	list icmp_type 'router-advertisement'
	list icmp_type 'neighbour-advertisement'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-ICMPv6-Forward'
	option src 'wan'
	option dest '*'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-IPSec-ESP'
	option src 'wan'
	option dest 'lan'
	option proto 'esp'
	option target 'ACCEPT'

config rule
	option name 'Allow-ISAKMP'
	option src 'wan'
	option dest 'lan'
	option dest_port '500'
	option proto 'udp'
	option target 'ACCEPT'

config include 'pbr'
	option fw4_compatible '1'
	option type 'script'
	option path '/usr/share/pbr/pbr.firewall.include'

config rule
	option name 'WAN SSH'
	option src 'wan'
	option dest_port '22'
	option target 'ACCEPT'
	option family 'ipv4'
	list src_ip '192.168.0.0/16'

config rule
	option name 'WAN HTTP'
	option src 'wan'
	option dest_port '80'
	option target 'ACCEPT'
	option family 'ipv4'
	list src_ip '192.168.0.0/16'

config zone 'guest'
	option name 'guest'
	option input 'REJECT'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'guest'

config rule 'guest_dns'
	option name 'Allow-DNS-guest'
	option src 'guest'
	option proto 'udp'
	option dest_port '53'
	option target 'ACCEPT'

config rule 'guest_dhcp'
	option name 'Allow-DHCP-guest'
	option src 'guest'
	option proto 'udp'
	option dest_port '67'
	option target 'ACCEPT'
	option family 'ipv4'

config rule 'guest_dhcpv6'
	option name 'Allow-DHCPv6-guest'
	option src 'guest'
	option proto 'udp'
	option dest_port '547'
	option target 'ACCEPT'
	option family 'ipv6'

config forwarding 'f_lan_guest'
	option src 'lan'
	option dest 'guest'

config forwarding 'f_guest_wan'
	option src 'guest'
	option dest 'wan'

config zone 'z_fr'
	option name 'FR'
	option input 'REJECT'
	option output 'REJECT'
	option forward 'ACCEPT'
	option masq '1'
	option mtu_fix '1'
	list network 'fr'

config forwarding 'f_lan_fr'
	option src 'lan'
	option dest 'FR'

config forwarding 'f_guest_fr'
	option src 'guest'
	option dest 'FR'

config zone 'z_hk'
	option name 'HK'
	option input 'REJECT'
	option output 'REJECT'
	option forward 'ACCEPT'
	option masq '1'
	option mtu_fix '1'
	list network 'hk'

config forwarding 'f_lan_hk'
	option src 'lan'
	option dest 'HK'

config forwarding 'f_guest_hk'
	option src 'guest'
	option dest 'HK'

config zone 'z_us'
	option name 'US'
	option input 'REJECT'
	option output 'REJECT'
	option forward 'ACCEPT'
	option masq '1'
	option mtu_fix '1'
	list network 'us'

config forwarding 'f_lan_us'
	option src 'lan'
	option dest 'US'

config forwarding 'f_guest_us'
	option src 'guest'
	option dest 'US'

config zone 'z_uk'
	option name 'UK'
	option input 'REJECT'
	option output 'REJECT'
	option forward 'ACCEPT'
	option masq '1'
	option mtu_fix '1'
	list network 'uk'

config forwarding 'f_lan_uk'
	option src 'lan'
	option dest 'UK'

config forwarding 'f_guest_uk'
	option src 'guest'
	option dest 'UK'
PBR Status 
/etc/init.d/pbr status
/etc/init.d/pbr reload
/etc/init.d/pbr status
============================================================
pbr - environment
pbr 1.1.1-7 running on OpenWrt 23.05.4. WAN (IPv4): wan/br-wan/192.168.1.254.
============================================================
Dnsmasq version 2.90  Copyright (c) 2000-2024 Simon Kelley
Compile time options: IPv6 GNU-getopt no-RTC no-DBus UBus no-i18n no-IDN DHCP DHCPv6 no-Lua TFTP conntrack ipset nftset auth cryptohash DNSSEC no-ID loop-detect inotify dumpfile
============================================================
pbr chains - policies
	chain pbr_forward { # handle 67
	}
	chain pbr_input { # handle 68
	}
	chain pbr_output { # handle 69
	}
	chain pbr_prerouting { # handle 70
		ip daddr @pbr_ignore_4_dst_ip_ignore return comment "ignore" # handle 1663
		ip daddr @pbr_fr_4_dst_ip_fr_france goto pbr_mark_0x020000 comment "fr_france" # handle 1665
		ip saddr @pbr_uk_4_src_ip_uk_default goto pbr_mark_0x040000 comment "uk_default" # handle 1667
	}
	chain pbr_postrouting { # handle 71
	}
============================================================
pbr chains - marking
	chain pbr_mark_0x010000 { # handle 1647
		counter packets 0 bytes 0 meta mark set meta mark & 0xff01ffff | 0x00010000 # handle 1648
		return # handle 1649
	}
	chain pbr_mark_0x020000 { # handle 1650
		counter packets 0 bytes 0 meta mark set meta mark & 0xff02ffff | 0x00020000 # handle 1651
		return # handle 1652
	}
	chain pbr_mark_0x030000 { # handle 1653
		counter packets 0 bytes 0 meta mark set meta mark & 0xff03ffff | 0x00030000 # handle 1654
		return # handle 1655
	}
	chain pbr_mark_0x040000 { # handle 1656
		counter packets 131 bytes 63194 meta mark set meta mark & 0xff04ffff | 0x00040000 # handle 1657
		return # handle 1658
	}
	chain pbr_mark_0x050000 { # handle 1659
		counter packets 0 bytes 0 meta mark set meta mark & 0xff05ffff | 0x00050000 # handle 1660
		return # handle 1661
	}
============================================================
pbr nft sets
	set pbr_ignore_4_dst_ip_ignore { # handle 1662
		type ipv4_addr
		flags interval
		counter
		auto-merge
		comment "ignore"
		elements = { 192.168.0.0/16 counter packets 994 bytes 1030943 }
	}
	set pbr_fr_4_dst_ip_fr_france { # handle 1664
		type ipv4_addr
		flags interval
		counter
		auto-merge
		comment "fr_france"
	}
	set pbr_uk_4_src_ip_uk_default { # handle 1666
		type ipv4_addr
		flags interval
		counter
		auto-merge
		comment "uk_default"
		elements = { 0.0.0.0/0 counter packets 142 bytes 73557 }
	}
============================================================
dnsmasq sets
nftset=/ifconfig.me/4#inet#fw4#pbr_fr_4_dst_ip_fr_france # fr_france
nftset=/fr/4#inet#fw4#pbr_fr_4_dst_ip_fr_france # fr_france
============================================================
IPv4 table 256 route: default via 192.168.1.254 dev br-wan
IPv4 table 256 rule(s):
30000:	from all fwmark 0x10000/0xff0000 lookup pbr_wan
IPv4 table 257 route: default via 10.100.0.2 dev tun0
IPv4 table 257 rule(s):
30001:	from all fwmark 0x20000/0xff0000 lookup pbr_fr
IPv4 table 258 route: default via 10.100.0.2 dev tun1
IPv4 table 258 rule(s):
30002:	from all fwmark 0x30000/0xff0000 lookup pbr_hk
IPv4 table 259 route: default via 10.100.0.2 dev tun2
IPv4 table 259 rule(s):
30003:	from all fwmark 0x40000/0xff0000 lookup pbr_uk
IPv4 table 260 route: unreachable default
IPv4 table 260 rule(s):
30004:	from all fwmark 0x50000/0xff0000 lookup pbr_us
Activating traffic killswitch [✓]
Setting up routing for 'wan/br-wan/192.168.1.254' [✓]
Setting up routing for 'fr/tun0/10.100.0.2' [✓]
Setting up routing for 'hk/tun1/10.100.0.2' [✓]
Setting up routing for 'uk/tun2/10.100.0.2' [✓]
Setting up routing for 'us/tun3/0.0.0.0' [✓]
Routing 'ignore' via ignore [✓]
Routing 'fr_france' via fr [✓]
Routing 'uk_default' via uk [✓]
Deactivating traffic killswitch [✓]
pbr 1.1.1-7 monitoring interfaces: wan fr hk uk us
pbr 1.1.1-7 (nft) started with gateways:
wan/br-wan/192.168.1.254 [✓]
fr/tun0/10.100.0.2
hk/tun1/10.100.0.2
uk/tun2/10.100.0.2
us/tun3/0.0.0.0
============================================================
pbr - environment
pbr 1.1.1-7 running on OpenWrt 23.05.4. WAN (IPv4): wan/br-wan/192.168.1.254.
============================================================
Dnsmasq version 2.90  Copyright (c) 2000-2024 Simon Kelley
Compile time options: IPv6 GNU-getopt no-RTC no-DBus UBus no-i18n no-IDN DHCP DHCPv6 no-Lua TFTP conntrack ipset nftset auth cryptohash DNSSEC no-ID loop-detect inotify dumpfile
============================================================
pbr chains - policies
	chain pbr_forward { # handle 67
	}
	chain pbr_input { # handle 68
	}
	chain pbr_output { # handle 69
	}
	chain pbr_prerouting { # handle 70
		ip daddr @pbr_ignore_4_dst_ip_ignore return comment "ignore" # handle 1686
		ip daddr @pbr_fr_4_dst_ip_fr_france goto pbr_mark_0x020000 comment "fr_france" # handle 1688
		ip saddr @pbr_uk_4_src_ip_uk_default goto pbr_mark_0x040000 comment "uk_default" # handle 1690
	}
	chain pbr_postrouting { # handle 71
	}
============================================================
pbr chains - marking
	chain pbr_mark_0x010000 { # handle 1670
		counter packets 0 bytes 0 meta mark set meta mark & 0xff01ffff | 0x00010000 # handle 1671
		return # handle 1672
	}
	chain pbr_mark_0x020000 { # handle 1673
		counter packets 0 bytes 0 meta mark set meta mark & 0xff02ffff | 0x00020000 # handle 1674
		return # handle 1675
	}
	chain pbr_mark_0x030000 { # handle 1676
		counter packets 0 bytes 0 meta mark set meta mark & 0xff03ffff | 0x00030000 # handle 1677
		return # handle 1678
	}
	chain pbr_mark_0x040000 { # handle 1679
		counter packets 24 bytes 14660 meta mark set meta mark & 0xff04ffff | 0x00040000 # handle 1680
		return # handle 1681
	}
	chain pbr_mark_0x050000 { # handle 1682
		counter packets 0 bytes 0 meta mark set meta mark & 0xff05ffff | 0x00050000 # handle 1683
		return # handle 1684
	}
============================================================
pbr nft sets
	set pbr_ignore_4_dst_ip_ignore { # handle 1685
		type ipv4_addr
		flags interval
		counter
		auto-merge
		comment "ignore"
		elements = { 192.168.0.0/16 counter packets 43 bytes 2884 }
	}
	set pbr_fr_4_dst_ip_fr_france { # handle 1687
		type ipv4_addr
		flags interval
		counter
		auto-merge
		comment "fr_france"
	}
	set pbr_uk_4_src_ip_uk_default { # handle 1689
		type ipv4_addr
		flags interval
		counter
		auto-merge
		comment "uk_default"
		elements = { 0.0.0.0/0 counter packets 25 bytes 14712 }
	}
============================================================
dnsmasq sets
nftset=/ifconfig.me/4#inet#fw4#pbr_fr_4_dst_ip_fr_france # fr_france
nftset=/fr/4#inet#fw4#pbr_fr_4_dst_ip_fr_france # fr_france
============================================================
IPv4 table 256 route: default via 192.168.1.254 dev br-wan
IPv4 table 256 rule(s):
30000:	from all fwmark 0x10000/0xff0000 lookup pbr_wan
IPv4 table 257 route: default via 10.100.0.2 dev tun0
IPv4 table 257 rule(s):
30001:	from all fwmark 0x20000/0xff0000 lookup pbr_fr
IPv4 table 258 route: default via 10.100.0.2 dev tun1
IPv4 table 258 rule(s):
30002:	from all fwmark 0x30000/0xff0000 lookup pbr_hk
IPv4 table 259 route: default via 10.100.0.2 dev tun2
IPv4 table 259 rule(s):
30003:	from all fwmark 0x40000/0xff0000 lookup pbr_uk
IPv4 table 260 route: unreachable default
IPv4 table 260 rule(s):
30004:	from all fwmark 0x50000/0xff0000 lookup pbr_us
nft list ruleset

Warning: table ip mangle is managed by iptables-nft, do not touch!

	xt match "set" counter packets 0 bytes 0 xt target "MARK"
	xt match "set" counter packets 5705400 bytes 6911320708 xt target "MARK"
	xt match "set" counter packets 0 bytes 0 xt target "MARK"
	meta l4proto tcp xt match "multiport" meta mark & 0x00003f00 == 0x00000000 counter packets 19342 bytes 1364569 jump mwan3_rule_https
	meta mark & 0x00003f00 == 0x00000000 counter packets 8758265 bytes 8004010026 xt target "CONNMARK"
	counter packets 8797917 bytes 8008067925 xt target "CONNMARK"
	meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
	meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
	meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 59201 bytes 10653662 xt target "MARK"
	meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
	meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
	meta mark & 0x0000fc00 != 0x0000fc00 counter packets 19342 bytes 1364569 xt target "SET"
	meta mark & 0x0000fc00 != 0x0000fc00 counter packets 19342 bytes 1364569 xt target "SET"
	iifname "br-wan" xt match "set" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
	iifname "br-wan" xt match "set" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
	iifname "br-wan" xt match "set" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"

Warning: table ip6 mangle is managed by iptables-nft, do not touch!

	iifname "br-wan" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 914 bytes 51599 xt target "MARK"
	xt match "set" counter packets 0 bytes 0 xt target "MARK"
	xt match "set" counter packets 78 bytes 7941 xt target "MARK"
	xt match "set" counter packets 0 bytes 0 xt target "MARK"
	meta l4proto tcp xt match "multiport" meta mark & 0x00003f00 == 0x00000000 counter packets 0 bytes 0 jump mwan3_rule_https
	meta l4proto ipv6-icmp xt match "icmp6" counter packets 56 bytes 2872 return
	meta l4proto ipv6-icmp xt match "icmp6" counter packets 221 bytes 35360 return
	meta l4proto ipv6-icmp xt match "icmp6" counter packets 1460 bytes 104840 return
	meta l4proto ipv6-icmp xt match "icmp6" counter packets 880 bytes 58784 return
	meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 return
	meta mark & 0x00003f00 == 0x00000000 counter packets 40300 bytes 11196724 xt target "CONNMARK"
	counter packets 40506 bytes 11219377 xt target "CONNMARK"
	meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
	meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
	meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 39852 bytes 11111988 xt target "MARK"
	meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
	meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
	meta mark & 0x0000fc00 != 0x0000fc00 counter packets 0 bytes 0 xt target "SET"
	meta mark & 0x0000fc00 != 0x0000fc00 counter packets 0 bytes 0 xt target "SET"
2

Please review README->Getting Help and amend with the missing config information.

From the service pbr status output tho, it's evident that necessary nft rules are created, as well as dnsmasq's nft sets for ifconfig.me and .fr domain names. Thoroughly peruse README as it may have additional information on how things do and do not work.

PS. 1.1.1-7 is quite old, the current version in 23.05 packages repo is 1.1.6-20, it will most likely not solve your problem (domain name-based policies failures are almost always seem to be DNS-related misconfigurations on the router or the client), but still be beneficial to update.

3

post output of

nft list ruleset | grep xt
4

Thanks @brada4 and @stangri , for looking into it.
Sorry for not putting all the info. I have added the output of nft list ruleset | grep xt

5

Based on this, it appears that the three tunnels share the same local IP...

1 Like
6

Can you also show the output of:

ip route show
ip route show table all
ip rule show

Did you disable the default routing of the OpenVPN tunnels?

I did not had time to look at everything but:
The option src_addr '0.0.0.0/0' actually is a from all probably not what you want

EDIT:
You can remove the devices here as you already use the networks which have the device in the interfaces but the US interface with tun3 seems missing in action. you have to add that too as interface in the network

Furthermore all tunnels have the same IP that is also problematic

1 Like
7

xt target mark and meta mark are two different places.
Ref:
https://wiki.nftables.org/wiki-nftables/index.php/Supported_features_compared_to_xtables
You need kmod-nft-core kmod-nf-conntrack and kmod-nf-conntrack6 , maybe something else. Reboot to change engine after installing those and verify again, only xt set match should remain.

8

Hi,

Yes, all tunnels are using the same IP, not sure how I can change that (I am connected to a VPN Provider)?

I disabled the default routing in the vpn with:

pull-filter ignore "route-gateway"

I updated pbr to from 1.1.1-7 to 1.1.6-20 but that didn't change anything (it's strange that I need to update because I build the image myself so I thought it would automatically get latest version). Still not luck.

ip route 
default via 192.168.1.254 dev br-wan proto static src 192.168.1.1
10.100.0.0/24 dev tun2 proto kernel scope link src 10.100.0.2
10.100.0.0/24 dev tun1 proto kernel scope link src 10.100.0.2
10.100.0.0/24 dev tun0 proto kernel scope link src 10.100.0.2
192.168.1.0/24 dev br-wan proto kernel scope link src 192.168.1.1
192.168.17.0/24 dev br-lan.14 proto kernel scope link src 192.168.17.1
192.168.27.0/24 dev br-lan.21 proto kernel scope link src 192.168.27.1
default via 192.168.1.254 dev br-wan table pbr_wan
192.168.17.0/24 dev br-lan.14 table pbr_wan proto kernel scope link src 192.168.17.1
192.168.27.0/24 dev br-lan.21 table pbr_wan proto kernel scope link src 192.168.27.1
default via 10.100.0.2 dev tun0 table pbr_fr
192.168.17.0/24 dev br-lan.14 table pbr_fr proto kernel scope link src 192.168.17.1
192.168.27.0/24 dev br-lan.21 table pbr_fr proto kernel scope link src 192.168.27.1
default via 192.168.1.254 dev br-wan table 1 proto static src 192.168.1.1
10.100.0.0/24 dev tun0 table 1 proto kernel scope link src 10.100.0.2
192.168.1.0/24 dev br-wan table 1 proto kernel scope link src 192.168.1.1
192.168.17.0/24 dev br-lan.14 table 1 proto kernel scope link src 192.168.17.1
192.168.27.0/24 dev br-lan.21 table 1 proto kernel scope link src 192.168.27.1
default via 10.100.0.2 dev tun1 table pbr_hk
192.168.17.0/24 dev br-lan.14 table pbr_hk proto kernel scope link src 192.168.17.1
192.168.27.0/24 dev br-lan.21 table pbr_hk proto kernel scope link src 192.168.27.1
default via 10.100.0.2 dev tun2 table pbr_uk
192.168.17.0/24 dev br-lan.14 table pbr_uk proto kernel scope link src 192.168.17.1
192.168.27.0/24 dev br-lan.21 table pbr_uk proto kernel scope link src 192.168.27.1
unreachable default table pbr_us
192.168.17.0/24 dev br-lan.14 table pbr_us proto kernel scope link src 192.168.17.1
192.168.27.0/24 dev br-lan.21 table pbr_us proto kernel scope link src 192.168.27.1
default via 192.168.1.254 dev br-wan proto static src 192.168.1.1
10.100.0.0/24 dev tun2 proto kernel scope link src 10.100.0.2
10.100.0.0/24 dev tun1 proto kernel scope link src 10.100.0.2
10.100.0.0/24 dev tun0 proto kernel scope link src 10.100.0.2
192.168.1.0/24 dev br-wan proto kernel scope link src 192.168.1.1
192.168.17.0/24 dev br-lan.14 proto kernel scope link src 192.168.17.1
192.168.27.0/24 dev br-lan.21 proto kernel scope link src 192.168.27.1
local 10.100.0.2 dev tun2 table local proto kernel scope host src 10.100.0.2
local 10.100.0.2 dev tun1 table local proto kernel scope host src 10.100.0.2
local 10.100.0.2 dev tun0 table local proto kernel scope host src 10.100.0.2
broadcast 10.100.0.255 dev tun2 table local proto kernel scope link src 10.100.0.2
broadcast 10.100.0.255 dev tun1 table local proto kernel scope link src 10.100.0.2
broadcast 10.100.0.255 dev tun0 table local proto kernel scope link src 10.100.0.2
local 127.0.0.0/8 dev lo table local proto kernel scope host src 127.0.0.1
local 127.0.0.1 dev lo table local proto kernel scope host src 127.0.0.1
broadcast 127.255.255.255 dev lo table local proto kernel scope link src 127.0.0.1
local 192.168.1.1 dev br-wan table local proto kernel scope host src 192.168.1.1
broadcast 192.168.1.255 dev br-wan table local proto kernel scope link src 192.168.1.1
local 192.168.17.1 dev br-lan.14 table local proto kernel scope host src 192.168.17.1
broadcast 192.168.17.255 dev br-lan.14 table local proto kernel scope link src 192.168.17.1
local 192.168.27.1 dev br-lan.21 table local proto kernel scope host src 192.168.27.1
broadcast 192.168.27.255 dev br-lan.21 table local proto kernel scope link src 192.168.27.1
fda6:a8dc:257f::/64 dev br-lan.21 proto static metric 1024 pref medium
fda6:a8dc:257f:10::/64 dev br-lan.14 proto static metric 1024 pref medium
unreachable fda6:a8dc:257f::/48 dev lo proto static metric 2147483647 pref medium
fe80::/64 dev eth0 proto kernel metric 256 pref medium
fe80::/64 dev sw0 proto kernel metric 256 pref medium
fe80::/64 dev sw1 proto kernel metric 256 pref medium
fe80::/64 dev br-lan proto kernel metric 256 pref medium
fe80::/64 dev br-lan.14 proto kernel metric 256 pref medium
fe80::/64 dev br-lan.21 proto kernel metric 256 pref medium
fe80::/64 dev br-wan proto kernel metric 256 pref medium
fe80::/64 dev tun2 proto kernel metric 256 pref medium
fe80::/64 dev tun1 proto kernel metric 256 pref medium
fe80::/64 dev tun0 proto kernel metric 256 pref medium
local ::1 dev lo table local proto kernel metric 0 pref medium
anycast fda6:a8dc:257f:: dev br-lan.21 table local proto kernel metric 0 pref medium
local fda6:a8dc:257f::1 dev br-lan.21 table local proto kernel metric 0 pref medium
anycast fda6:a8dc:257f:10:: dev br-lan.14 table local proto kernel metric 0 pref medium
local fda6:a8dc:257f:10::1 dev br-lan.14 table local proto kernel metric 0 pref medium
anycast fe80:: dev sw1 table local proto kernel metric 0 pref medium
anycast fe80:: dev eth0 table local proto kernel metric 0 pref medium
anycast fe80:: dev sw0 table local proto kernel metric 0 pref medium
anycast fe80:: dev br-lan.21 table local proto kernel metric 0 pref medium
anycast fe80:: dev br-lan.14 table local proto kernel metric 0 pref medium
anycast fe80:: dev br-lan table local proto kernel metric 0 pref medium
anycast fe80:: dev br-wan table local proto kernel metric 0 pref medium
anycast fe80:: dev tun2 table local proto kernel metric 0 pref medium
anycast fe80:: dev tun1 table local proto kernel metric 0 pref medium
anycast fe80:: dev tun0 table local proto kernel metric 0 pref medium
local fe80::4ec:d0ff:fef4:5fec dev sw1 table local proto kernel metric 0 pref medium
local fe80::4ec:d0ff:fef4:5fec dev eth0 table local proto kernel metric 0 pref medium
local fe80::4ec:d0ff:fef4:5fec dev sw0 table local proto kernel metric 0 pref medium
local fe80::4ec:d0ff:fef4:5fec dev br-lan.21 table local proto kernel metric 0 pref medium
local fe80::4ec:d0ff:fef4:5fec dev br-lan.14 table local proto kernel metric 0 pref medium
local fe80::4ec:d0ff:fef4:5fec dev br-lan table local proto kernel metric 0 pref medium
local fe80::4659:6bf9:219a:596f dev tun0 table local proto kernel metric 0 pref medium
local fe80::b6b2:6751:d2d8:f7ed dev tun1 table local proto kernel metric 0 pref medium
local fe80::e2cb:bcff:fe25:6e9c dev br-wan table local proto kernel metric 0 pref medium
local fe80::ed2a:e72f:a0fe:5f2e dev tun2 table local proto kernel metric 0 pref medium
multicast ff00::/8 dev eth0 table local proto kernel metric 256 pref medium
multicast ff00::/8 dev sw0 table local proto kernel metric 256 pref medium
multicast ff00::/8 dev br-lan.14 table local proto kernel metric 256 pref medium
multicast ff00::/8 dev br-lan.21 table local proto kernel metric 256 pref medium
multicast ff00::/8 dev sw1 table local proto kernel metric 256 pref medium
multicast ff00::/8 dev br-lan table local proto kernel metric 256 pref medium
multicast ff00::/8 dev br-wan table local proto kernel metric 256 pref medium
multicast ff00::/8 dev tun2 table local proto kernel metric 256 pref medium
multicast ff00::/8 dev tun1 table local proto kernel metric 256 pref medium
multicast ff00::/8 dev tun0 table local proto kernel metric 256 pref medium
0:	from all lookup local
1001:	from all iif br-wan lookup 1
2001:	from all fwmark 0x100/0x3f00 lookup 1
2061:	from all fwmark 0x3d00/0x3f00 blackhole
2062:	from all fwmark 0x3e00/0x3f00 unreachable
3001:	from all fwmark 0x100/0x3f00 unreachable
30000:	from all fwmark 0x10000/0xff0000 lookup pbr_wan
30001:	from all fwmark 0x20000/0xff0000 lookup pbr_fr
30002:	from all fwmark 0x30000/0xff0000 lookup pbr_hk
30003:	from all fwmark 0x40000/0xff0000 lookup pbr_uk
30004:	from all fwmark 0x50000/0xff0000 lookup pbr_us
32766:	from all lookup main
32767:	from all lookup default
9

I wonder if my Makefile is to blame, this is the depends section from it:

  DEPENDS+=+kmod-nft-core +kmod-nft-nat +nftables-json

Should I include the conntrack/6 kmods as well?

10

It can work with xtables bridge, it is something lke
xt_addrtype | nft-fib
and so on...

11

Could it be that it is necessary to split out pbr-iptables? Seems even fw3 still has avid following like in censorship squashing configs or da?n coovaspot.
There are known conflicts around mark and nat modules that one does not see other (-m socket not used but also here)
Then there are xt-only modules like text, u32, and ipsets which can be programmed via iptables-nft over xt bridge. Go figure.
wdyt?

12

Unless I misunderstood your question, the pbr-iptables exists in 23.05 and has its own dependencies and now separate init-script.

1 Like
13

Found the issue, it's the mwan3 package that breaks everything. I removed it and it's all working now.
Thanks for the help.

14

@stangri are mark-s overlapping (besides being set and checked inconsistently in xt and nft worlds)?

15

overlapping with what? also, please elaborate what's being set and checked inconsistently and inconsistently with what?

16

meta mark and xt mark are set and read in ruleset, thus intended logic is not applied consistently.

I wonder if same exact bits are set by mwan3

17

no, the default fwmarks are set up to not collide with any known OpenWrt package by the time vpn-policy-routing/pbr were added to packages repo. The fwmark can be overridden in pbr config as well.

Still not sure I understand what you're talking about. If you could use an example of inconsistencies from the service pbr status output it might help me understand better.

18

meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"

first post last expandable section.

it checks nft mark then sets xt mark.

19

that's not pbr

1 Like
20

Ahh ohh, ok.