VPN Policy-Based Routing + Web UI -- Discussion

Community Builds, Projects & Packages
1812

Since you've reported this issue, I've switched all nft sets the pbr creates to allow overlapping, however, presently nft only allows overlapping elements from the same "add element" command with multiple elements. When two separate nft commands are used to add overlapping elements to the same nft set, the second command will fail.

It is only a problem for adding networks, not individual IP addresses, I'm not sure if netfilter folks consider this a problem and are going to fix that. Thanks for confirming the workaround I've implemented in the user files works.

1813

I switched from 21.03.3 with vpn-policy-routing to 22.03.0 with pbr.
I just copied the /etc/config/vpn-policy-routing to /etc/config/pbr (I think that is the problem)

But anyway, my vpn tunnels are not working at the moment. Default route is WAN, fallback to WWAN and my clients from 192.168.1.100 and up are using wireguard or openvpn. But both are not working after going to pbr.

So all traffic is going through WAN at this moment.

  • content of /etc/config/dhcp 
root@Router:/etc/config# cat dhcp

config dnsmasq
	option domainneeded '1'
	option localise_queries '1'
	option rebind_protection '1'
	option rebind_localhost '1'
	option local '/lan/'
	option domain 'lan'
	option expandhosts '1'
	option authoritative '1'
	option readethers '1'
	option leasefile '/tmp/dhcp.leases'
	option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
	option localservice '1'
	option ednspacket_max '1232'
	option confdir '/tmp/dnsmasq.d'
	list server '/use-application-dns.net/'
	list server '127.0.0.1#5053'
	list server '/mask.icloud.com/'
	list server '/mask-h2.icloud.com/'
	option doh_backup_noresolv '-1'
	option noresolv '1'
	list doh_backup_server '/use-application-dns.net/'
	list doh_backup_server '127.0.0.1#5053'

config dhcp 'lan'
	option interface 'lan'
	option start '100'
	option limit '150'
	option leasetime '12h'
	option dhcpv4 'server'
	option dhcpv6 'server'
	option ra 'server'
	list ra_flags 'managed-config'
	list ra_flags 'other-config'
	option dns_service '0'

config dhcp 'wan'
	option interface 'wan'
	option ignore '1'
	option start '100'
	option limit '150'
	option leasetime '12h'
	list ra_flags 'none'

config odhcpd 'odhcpd'
	option maindhcp '0'
	option leasefile '/tmp/hosts/odhcpd'
	option leasetrigger '/usr/sbin/odhcpd-update'
	option loglevel '4'

config host
	option name 'raspberrypi'
	option ip '192.168.1.172'
	option mac 'B8:27:EB:F6:F4:DC'

config host
	option name 'HarmonyHub'
	option ip '192.168.1.236'
	option mac '00:04:20:F6:00:68'

config host
	option name 'Kodi-Pi4'
	option ip '192.168.1.108'
	option mac 'DC:A6:32:FC:A7:53'

config dhcp 'ziggo_lan'
	option interface 'ziggo_lan'
	option start '100'
	option limit '150'
	option leasetime '12h'
	option ra 'server'
	option dhcpv6 'server'
	list ra_flags 'managed-config'
	list ra_flags 'other-config'
	option force '1'
	list dns '2001:b88:1202::10'
	list dns '2001:b88:1002::10'
	list dns '2001:730:3e42:1000::53'
	list dhcp_option '6,84.116.46.23,84.116.46.22'
	list dhcp_option '6,84.116.46.20,84.116.46.21'

config host
	option name 'VMB4000'
	option mac '9C:3D:CF:49:F1:8B'
	option ip '192.168.1.20'

config domain
	option ip '192.168.1.30'
	option name 'nas.lan'

config domain
	option ip '192.168.1.1'
	option name 'router.lan'
  • content of /etc/config/firewall 
root@Router:/etc/config# cat firewall

config defaults
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	option synflood_protect '1'

config zone 'lan'
	option name 'lan'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'
	list network 'lan'
	list network 'RECOVERY'
	list network 'vpnserver'

config zone
	option name 'ziggo_lan'
	option output 'ACCEPT'
	option forward 'ACCEPT'
	option input 'ACCEPT'
	list network 'ziggo_lan'

config zone 'wan'
	option name 'wan'
	option input 'REJECT'
	option output 'ACCEPT'
	option forward 'REJECT'
	option masq '1'
	option mtu_fix '1'
	list network 'wan'
	list network 'wan6'

config zone
	option name 'vpnclient'
	option output 'ACCEPT'
	option forward 'REJECT'
	option input 'REJECT'
	option masq '1'
	list network 'tun1'
	list network 'vpnclient'

config forwarding
	option src 'lan'
	option dest 'wan'

config rule
	option dest '*'
	option src '*'
	option target 'REJECT'
	option name 'Block google DNS'
	option proto 'all'
	list dest_ip '8.8.8.8'
	list dest_ip '8.8.4.4'
	list dest_ip '2001:4860:4860::8888'
	list dest_ip '2001:4860:4860::8844'

config rule
	option name 'Allow-DHCP-Renew'
	option src 'wan'
	option proto 'udp'
	option dest_port '68'
	option target 'ACCEPT'
	option family 'ipv4'

config rule
	option name 'Allow-Ping'
	option src 'wan'
	option proto 'icmp'
	option icmp_type 'echo-request'
	option family 'ipv4'
	option target 'ACCEPT'

config rule
	option name 'Allow-IGMP'
	option src 'wan'
	option proto 'igmp'
	option family 'ipv4'
	option target 'ACCEPT'

config rule
	option name 'Allow-DHCPv6'
	option src 'wan'
	option proto 'udp'
	option src_ip 'fc00::/6'
	option dest_ip 'fc00::/6'
	option dest_port '546'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-MLD'
	option src 'wan'
	option proto 'icmp'
	option src_ip 'fe80::/10'
	list icmp_type '130/0'
	list icmp_type '131/0'
	list icmp_type '132/0'
	list icmp_type '143/0'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-ICMPv6-Input'
	option src 'wan'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	list icmp_type 'router-solicitation'
	list icmp_type 'neighbour-solicitation'
	list icmp_type 'router-advertisement'
	list icmp_type 'neighbour-advertisement'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-ICMPv6-Forward'
	option src 'wan'
	option dest '*'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-IPSec-ESP'
	option src 'wan'
	option dest 'lan'
	option proto 'esp'
	option target 'ACCEPT'

config rule
	option name 'Allow-ISAKMP'
	option src 'wan'
	option dest 'lan'
	option dest_port '500'
	option proto 'udp'
	option target 'ACCEPT'

config rule
	option name 'Support-UDP-Traceroute'
	option src 'wan'
	option dest_port '33434:33689'
	option proto 'udp'
	option family 'ipv4'
	option target 'REJECT'
	option enabled '0'

config include
	option path '/etc/firewall.user'

config redirect
	option target 'DNAT'
	option name 'http'
	option src 'wan'
	option src_dport '80'
	option dest 'lan'
	option dest_port '80'
	option dest_ip '192.168.1.30'

config redirect
	option target 'DNAT'
	option name 'NAS https'
	option src 'wan'
	option src_dport '443'
	option dest 'lan'
	option dest_port '443'
	option dest_ip '192.168.1.30'

config forwarding
	option src 'lan'
	option dest 'vpnclient'

config nat
	option name 'Prevents hardcoded DNS clients error'
	list proto 'tcp'
	list proto 'udp'
	option src 'lan'
	option dest_port '53'
	option target 'MASQUERADE'
	option enabled '0'
	option dest_ip '192.168.1.30'

config redirect
	option target 'DNAT'
	option name 'Mailserver '
	option src 'wan'
	option src_dport '25'
	option dest 'lan'
	option dest_port '25'
	option dest_ip '192.168.1.30'

config redirect
	option target 'DNAT'
	option name 'Mailserver'
	option src 'wan'
	option src_dport '465'
	option dest 'lan'
	option dest_port '465'
	option dest_ip '192.168.1.30'

config redirect
	option target 'DNAT'
	option name 'Mailserver'
	option src 'wan'
	option src_dport '587'
	option dest 'lan'
	option dest_port '587'
	option dest_ip '192.168.1.30'

config redirect
	option target 'DNAT'
	option name 'Mailserver'
	option src 'wan'
	option src_dport '995'
	option dest 'lan'
	option dest_port '995'
	option dest_ip '192.168.1.30'
	option enabled '0'

config redirect
	option target 'DNAT'
	option name 'Mailserver'
	option src 'wan'
	option src_dport '993'
	option dest 'lan'
	option dest_port '993'
	option dest_ip '192.168.1.30'

config rule 'ovpn'
	option name 'Allow-OpenVPN'
	option src 'wan'
	option dest_port '1194'
	option proto 'udp'
	option target 'ACCEPT'

config forwarding
	option src 'ziggo_lan'
	option dest 'wan'

config rule
	option name 'dhcp vlan ziggo'
	option src 'ziggo_lan'
	option dest_port '67-68'
	option target 'ACCEPT'
	list proto 'udp'

config redirect
	option dest 'lan'
	option target 'DNAT'
	option name 'NAS Admin port'
	option src 'wan'
	option src_dport '5051'
	option dest_ip '192.168.1.30'
	option dest_port '5051'

config redirect
	option dest 'lan'
	option target 'DNAT'
	option name 'Replicatie server'
	option src 'wan'
	option src_dport '5566'
	option dest_ip '192.168.1.30'
	option dest_port '5566'

config redirect 'adblock_lan53'
	option name 'Adblock DNS (lan, 53)'
	option src 'lan'
	option proto 'tcp udp'
	option src_dport '53'
	option dest_port '53'
	option target 'DNAT'

config zone
	option name 'Wwan'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'wwan'
	option input 'REJECT'
	option masq '1'

config forwarding
	option src 'lan'
	option dest 'Wwan'

config forwarding
	option src 'ziggo_lan'
	option dest 'Wwan'

config zone
	option name 'Wireguard'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'wg0'
	option masq '1'
	option input 'REJECT'

config forwarding
	option src 'lan'
	option dest 'Wireguard'

config include 'pbr'
	option fw4_compatible '1'
	option type 'script'
	option path '/usr/share/pbr/pbr.firewall.include'

root@Router:/etc/config#
  • content of /etc/config/network 
root@Router:/etc/config# cat network

config interface 'loopback'
	option device 'lo'
	option proto 'static'
	option ipaddr '127.0.0.1'
	option netmask '255.0.0.0'

config globals 'globals'
	option ula_prefix 'fdad:b459:8594::/48'

config interface 'lan'
	option proto 'static'
	option ipaddr '192.168.1.1'
	option netmask '255.255.255.0'
	option device 'br-lan'

config interface 'wan'
	option proto 'dhcp'
	option device 'eth0.10'
	option metric '10'
	option peerdns '0'
	list dns '9.9.9.9'
	list dns '149.112.112.112'

config interface 'wan6'
	option proto 'dhcpv6'
	option reqaddress 'try'
	option reqprefix 'auto'
	option peerdns '0'
	option device 'eth0.10'
	list dns '2620:fe::fe'
	list dns '2620:fe::9'

config interface 'vpnclient'
	option proto 'none'
	option device 'tun1'

config interface 'vpnserver'
	option proto 'none'
	option device 'tun0'

config device
	option name 'br-lan'
	option type 'bridge'
	list ports 'eth0.20'

config interface 'ziggo_lan'
	option proto 'static'
	option device 'eth0.30'
	list ipaddr '192.168.180.1/24'
	option ip6assign '62'

config interface 'RECOVERY'
	option proto 'static'
	option device 'eth0'
	option ipaddr '10.0.10.1'
	option netmask '255.255.255.0'

config interface 'wwan'
	option proto 'dhcp'
	option metric '20'

config interface 'wg0'
	option proto 'wireguard'
	option private_key 'KEYHERE'
	list addresses '10.2.0.2/32'

config wireguard_wg0
	option public_key 'KEYHERE'
	option route_allowed_ips '0'
	list allowed_ips '0.0.0.0/0'
	option persistent_keepalive '25'
	option description 'WG'
	option endpoint_host 'XX.XX.XX.85'
	option endpoint_port '51820'

root@Router:/etc/config#
  • content of /etc/config/pbr 
root@Router:/etc/config# cat pbr

config policy
	option name 'ignore local traffic'
	option src_addr '10.2.0.0/24'
	option interface 'ignore'

config policy
	option name 'vpn server clients'
	option src_addr '192.168.200.2/31 192.168.200.4/30 192.168.200.8/29 192.168.200.16/28 192.168.200.32/28 192.168.200.48/31 192.168.200.50/32'
	option interface 'wg0'

config policy
	option src_addr '192.168.1.100/30 192.168.1.104/29 192.168.1.112/28 192.168.1.128/25'
	option name 'Alle dhcp clients'
	option interface 'wg0'

config policy
	option interface 'wan'
	option name 'netflix'
	option dest_addr 'netflix.com'

config policy
	option interface 'wan'
	option name 'afas'
	option dest_addr 'afasinsite.nl'

config policy
	option interface 'wan'
	option name 'google'
	option dest_addr 'google.nl google.com'

config policy
	option interface 'wan'
	option name 'zalando'
	option dest_addr 'zalando.nl zalando.com'
	option enabled '0'

config pbr 'config'
	option verbosity '2'
	option src_ipset '0'
	option dest_ipset '0'
	option resolver_set 'dnsmasq.ipset'
	option boot_timeout '30'
	option procd_reload_delay '1'
	option webui_protocol_column '0'
	option webui_sorting '1'
	list webui_supported_protocol 'tcp'
	list webui_supported_protocol 'udp'
	list webui_supported_protocol 'tcp udp'
	list webui_supported_protocol 'icmp'
	list webui_supported_protocol 'all'
	option webui_enable_column '1'
	option webui_chain_column '1'
	option webui_show_ignore_target '1'
	option strict_enforcement '0'
	option ipv6_enabled '0'
	option rule_create_option 'add'
	option enabled '1'
	list ignored_interface 'vpnserver'
	list ignored_interface 'wgserver'
  • the output of /etc/init.d/pbr status 
oot@Router:/etc/config# /etc/init.d/pbr status
pbr 0.9.8-14 running on OpenWrt 22.03.0. WAN (IPv4): wan/eth0.10/XX.XX.56.1.
============================================================
Dnsmasq version 2.86  Copyright (c) 2000-2021 Simon Kelley
Compile time options: IPv6 GNU-getopt no-DBus UBus no-i18n no-IDN DHCP DHCPv6 no-Lua TFTP conntrack ipset auth cryptohash DNSSEC no-ID loop-detect inotify dumpfile
============================================================
Routes/IP Rules
default         82-75-56-1.cabl 0.0.0.0         UG    10     0        0 eth0.10
default         192.168.0.1     0.0.0.0         UG    20     0        0 wlan0

IPv4 Table 201: default via 82.75.56.1 dev eth0.10 
10.0.10.0/24 dev eth0 proto kernel scope link src 10.0.10.1 
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 
192.168.180.0/24 dev eth0.30 proto kernel scope link src 192.168.180.1 
192.168.200.0/24 dev tun0 proto kernel scope link src 192.168.200.1 
IPv4 Table 201 Rules:
30000:	from all fwmark 0x10000/0xff0000 lookup pbr_wan

IPv4 Table 202: default via 10.17.0.3 dev tun1 
10.0.10.0/24 dev eth0 proto kernel scope link src 10.0.10.1 
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 
192.168.180.0/24 dev eth0.30 proto kernel scope link src 192.168.180.1 
192.168.200.0/24 dev tun0 proto kernel scope link src 192.168.200.1 
IPv4 Table 202 Rules:
29999:	from all fwmark 0x20000/0xff0000 lookup pbr_vpnclient

IPv4 Table 203: default via 192.168.0.1 dev wlan0 
10.0.10.0/24 dev eth0 proto kernel scope link src 10.0.10.1 
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 
192.168.180.0/24 dev eth0.30 proto kernel scope link src 192.168.180.1 
192.168.200.0/24 dev tun0 proto kernel scope link src 192.168.200.1 
IPv4 Table 203 Rules:
29998:	from all fwmark 0x30000/0xff0000 lookup pbr_wwan

IPv4 Table 204: default via 10.2.0.2 dev wg0 
10.0.10.0/24 dev eth0 proto kernel scope link src 10.0.10.1 
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 
192.168.180.0/24 dev eth0.30 proto kernel scope link src 192.168.180.1 
192.168.200.0/24 dev tun0 proto kernel scope link src 192.168.200.1 
IPv4 Table 204 Rules:
29997:	from all fwmark 0x40000/0xff0000 lookup pbr_wg0

Error: ipv4: FIB table does not exist.
Dump terminated
IPv4 Table 205: 
IPv4 Table 205 Rules:

Error: ipv4: FIB table does not exist.
Dump terminated
IPv4 Table 206: 
IPv4 Table 206 Rules:

Error: ipv4: FIB table does not exist.
Dump terminated
IPv4 Table 207: 
IPv4 Table 207 Rules:

Error: ipv4: FIB table does not exist.
Dump terminated
IPv4 Table 208: 
IPv4 Table 208 Rules:

Error: ipv4: FIB table does not exist.
Dump terminated
IPv4 Table 209: 
IPv4 Table 209 Rules:

Error: ipv4: FIB table does not exist.
Dump terminated
IPv4 Table 210: 
IPv4 Table 210 Rules:
============================================================
Mangle IP Table: PREROUTING
-N PBR_PREROUTING
-A PBR_PREROUTING -s 10.2.0.0/24 -m comment --comment ignore_local_traffic -c 0 0 -j RETURN
-A PBR_PREROUTING -s 192.168.200.2/31 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.4/30 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.8/29 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.16/28 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.32/28 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.48/31 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.50/32 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.1.100/30 -m comment --comment Alle_dhcp_clients -c 11 2875 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.1.104/29 -m comment --comment Alle_dhcp_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.1.112/28 -m comment --comment Alle_dhcp_clients -c 142 22242 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.1.128/25 -m comment --comment Alle_dhcp_clients -c 893 146857 -g PBR_MARK_0x040000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_dst_ip_cfg046ff5 dst -m comment --comment netflix -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_dst_ip_cfg056ff5 dst -m comment --comment afas -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_dst_ip_cfg066ff5 dst -m comment --comment google -c 0 0 -g PBR_MARK_0x010000
============================================================
Mangle IP Table MARK Chain: PBR_MARK_0x010000
-N PBR_MARK_0x010000
-A PBR_MARK_0x010000 -c 0 0 -j MARK --set-xmark 0x10000/0xff0000
-A PBR_MARK_0x010000 -c 0 0 -j RETURN
============================================================
Mangle IP Table MARK Chain: PBR_MARK_0x020000
-N PBR_MARK_0x020000
-A PBR_MARK_0x020000 -c 0 0 -j MARK --set-xmark 0x20000/0xff0000
-A PBR_MARK_0x020000 -c 0 0 -j RETURN
============================================================
Mangle IP Table MARK Chain: PBR_MARK_0x030000
-N PBR_MARK_0x030000
-A PBR_MARK_0x030000 -c 0 0 -j MARK --set-xmark 0x30000/0xff0000
-A PBR_MARK_0x030000 -c 0 0 -j RETURN
============================================================
Mangle IP Table MARK Chain: PBR_MARK_0x040000
-N PBR_MARK_0x040000
-A PBR_MARK_0x040000 -c 1060 172702 -j MARK --set-xmark 0x40000/0xff0000
-A PBR_MARK_0x040000 -c 1060 172702 -j RETURN
============================================================
Current ipsets
create mwan3_dynamic_ipv6 hash:net family inet6 hashsize 1024 maxelem 65536
create mwan3_connected_ipv4 hash:net family inet hashsize 1024 maxelem 65536
add mwan3_connected_ipv4 10.2.0.2
add mwan3_connected_ipv4 127.0.0.0/8
add mwan3_connected_ipv4 192.168.200.0
add mwan3_connected_ipv4 224.0.0.0/3
add mwan3_connected_ipv4 10.0.10.0/24
add mwan3_connected_ipv4 127.0.0.1
add mwan3_connected_ipv4 192.168.1.0/24
add mwan3_connected_ipv4 10.0.10.1
add mwan3_connected_ipv4 82.75.57.66
add mwan3_connected_ipv4 192.168.200.255
add mwan3_connected_ipv4 192.168.200.0/24
add mwan3_connected_ipv4 192.168.180.0/24
add mwan3_connected_ipv4 127.255.255.255
add mwan3_connected_ipv4 192.168.0.0/24
add mwan3_connected_ipv4 127.0.0.0
add mwan3_connected_ipv4 192.168.180.255
add mwan3_connected_ipv4 185.107.56.85
add mwan3_connected_ipv4 192.168.180.0
add mwan3_connected_ipv4 10.17.0.0/16
add mwan3_connected_ipv4 192.168.180.1
add mwan3_connected_ipv4 192.168.200.1
add mwan3_connected_ipv4 10.0.10.255
add mwan3_connected_ipv4 10.17.0.0
add mwan3_connected_ipv4 10.17.255.255
add mwan3_connected_ipv4 192.168.1.0
add mwan3_connected_ipv4 82.75.56.0
add mwan3_connected_ipv4 82.75.56.0/23
add mwan3_connected_ipv4 192.168.1.1
add mwan3_connected_ipv4 10.17.0.3
add mwan3_connected_ipv4 192.168.0.0
add mwan3_connected_ipv4 10.0.10.0
add mwan3_connected_ipv4 82.75.57.255
add mwan3_connected_ipv4 192.168.0.255
add mwan3_connected_ipv4 192.168.1.255
add mwan3_connected_ipv4 192.168.0.109
create mwan3_connected_ipv6 hash:net family inet6 hashsize 1024 maxelem 65536
add mwan3_connected_ipv6 2001:1c04:1305:1300::/64
add mwan3_connected_ipv6 fdad:b459:8594::/64
add mwan3_connected_ipv6 fe80::/64
create mwan3_custom_ipv4 hash:net family inet hashsize 1024 maxelem 65536
create mwan3_custom_ipv6 hash:net family inet6 hashsize 1024 maxelem 65536
create mwan3_rule_ipv4_https hash:ip,mark family inet markmask 0x00003f00 hashsize 1024 maxelem 65536 timeout 600
add mwan3_rule_ipv4_https 192.168.1.122,0x00000100 timeout 10
add mwan3_rule_ipv4_https 192.168.1.172,0x00000100 timeout 595
add mwan3_rule_ipv4_https 192.168.1.211,0x00000100 timeout 328
add mwan3_rule_ipv4_https 192.168.1.230,0x00000100 timeout 491
add mwan3_rule_ipv4_https 192.168.1.119,0x00000100 timeout 598
add mwan3_rule_ipv4_https 192.168.1.30,0x00000100 timeout 485
add mwan3_rule_ipv4_https 192.168.1.173,0x00000100 timeout 347
add mwan3_rule_ipv4_https 192.168.1.123,0x00000100 timeout 176
add mwan3_rule_ipv4_https 192.168.1.142,0x00000100 timeout 560
add mwan3_rule_ipv4_https 192.168.1.226,0x00000100 timeout 326
add mwan3_rule_ipv4_https 192.168.1.121,0x00000100 timeout 503
add mwan3_rule_ipv4_https 192.168.1.233,0x00000100 timeout 530
add mwan3_rule_ipv4_https 192.168.180.221,0x00000100 timeout 538
add mwan3_rule_ipv4_https 192.168.1.20,0x00000100 timeout 498
add mwan3_rule_ipv4_https 192.168.1.217,0x00000100 timeout 45
add mwan3_rule_ipv4_https 192.168.180.165,0x00000100 timeout 468
create mwan3_rule_ipv6_https hash:ip,mark family inet6 markmask 0x00003f00 hashsize 1024 maxelem 65536 timeout 600
add mwan3_rule_ipv6_https 2001:1c04:1305:1300::68e,0x00003e00 timeout 468
add mwan3_rule_ipv6_https 2001:1c04:1305:1300::53b,0x00003e00 timeout 538
create pbr_ignore_4_src_net_cfg016ff5 hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_src_net_cfg026ff5 hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_src_net_cfg036ff5 hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_dst_ip_cfg046ff5 hash:ip family inet hashsize 1024 maxelem 65536 comment
add pbr_wan_4_dst_ip_cfg046ff5 54.195.14.93
add pbr_wan_4_dst_ip_cfg046ff5 54.73.99.78
add pbr_wan_4_dst_ip_cfg046ff5 176.34.214.23
create pbr_wan_4_dst_ip_cfg056ff5 hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_dst_ip_cfg066ff5 hash:ip family inet hashsize 1024 maxelem 65536 comment
add pbr_wan_4_dst_ip_cfg066ff5 142.251.36.4
add pbr_wan_4_dst_ip_cfg066ff5 172.217.168.206
create mwan3_dynamic_ipv4 list:set size 8
============================================================
DNSMASQ sets
ipset=/netflix.com/pbr_wan_4_dst_ip_cfg046ff5 # netflix: netflix.com
ipset=/afasinsite.nl/pbr_wan_4_dst_ip_cfg056ff5 # afas: afasinsite.nl
ipset=/google.nl/pbr_wan_4_dst_ip_cfg066ff5 # google: google.nl
ipset=/google.com/pbr_wan_4_dst_ip_cfg066ff5 # google: google.com
============================================================
Your support details have been logged to '/var/pbr-support'. [✓]
root@Router:/etc/config#
  • the output of /etc/init.d/pbr reload with verbosity setting set to 2
root@Router:/etc/config# /etc/init.d/pbr reload
Activating Traffic Killswitch [✓]
Setting up routing for 'wan/eth0.10/82.75.56.1' [✓]
Setting up routing for 'vpnclient/tun1/10.17.0.3' [✓]
Setting up routing for 'wwan/wlan0/192.168.0.1' [✓]
Setting up routing for 'wg0/10.2.0.2' [✓]
Routing 'ignore local traffic' via ignore [✓]
Routing 'vpn server clients' via wg0 [✓]
Routing 'Alle dhcp clients' via wg0 [✓]
Routing 'netflix' via wan [✓]
Routing 'afas' via wan [✓]
Routing 'google' via wan [✓]
Deactivating Traffic Killswitch [✓]
pbr 0.9.8-14 monitoring interfaces: wan vpnclient wwan wg0 
pbr 0.9.8-14 (iptables) started with gateways:
wan/eth0.10/82.75.56.1 [✓]
vpnclient/tun1/10.17.0.3
wwan/wlan0/192.168.0.1
wg0/10.2.0.2
root@Router:/etc/config#
1815

When I make openvpn or wireguard the default gateway, the route workes as expected, but setting in pbr does nothing.

Update:

Tested openvpn and wireguard vpn with pdr disabled, both working as inspected. (routing through vpn tunnel)

When I enable pbr all goes through wan, no matter what setting I alter, or rule I add.

Removed mwan3: I noticed that It takes mwan3 longer then pbr to get online, (my wwan is wireless so it takes more time to connect) before mwan3 is fully up, pbr works. As soon mwan3 is up. It al goes through wan.

force-reinstalled pbr and removed my /etc/config/pbr file. so started from scratch.

Now without mwan3 it works, but I cannot set dnsmasq.ipset (not there anymore, only the option disabled) don't know why.

Schermafbeelding 2022-10-06 om 18.34.26
Schermafbeelding 2022-10-06 om 18.34.261404×434 57.1 KB

As soon as I install mwan3 it takes over the route, and all goes over wan... ;-(

1816

Oh, sorry, the setting to ensure compatibility with mwan3 has been renamed in pbr. It's now rule_create_option and for mwan3 compatibility it needs to be set to insert.

For dnsmasq woes, please post the output of:
opkg list-installed | grep dnsmasq; opkg list-installed | grep pbr;

1817

Also, I see you're using https-dns-proxy, but your dhcp config will break dns resolution if you stop the https-dns-proxy service. You'd need to stop it, clean up the server entries and add something like public Google or Cloudflare DNS servers and then start https-dns-proxy again. It will back up your non-encrypted servers config to be reused when the https-dns-proxy stops.

1818

dnsmasq seems to be working now, don't know what was wrong but this one is fixed.

root@Router:~# opkg list-installed | grep dnsmasq; opkg list-installed | grep pbr;
dnsmasq-full - 2.86-14
luci-app-pbr - 0.9.8-14
pbr - 0.9.8-14

Schermafbeelding 2022-10-09 om 09.46.33
Schermafbeelding 2022-10-09 om 09.46.331294×382 62.5 KB

I changed the option rule in /etc/config/pbr

option rule_create_option 'add'

in

option rule_create_option 'insert'

But no succes, with mwan3 running it routes over wan, when I remove mwan3 pbr works.

FYI
I'm using mwan3 because my wan is on a managed switch (I use a Raspberry Pi 4b with only one network interface) so when my wan disconnects (for example ISP is down) it's going to the wwan because mwan3 can handle the disconnect.
I know pbr can do this, but this only works when I manually stop the wan interface. It's not working when my wan interface just disconnects but stays up.

1819

Thanks for this, fixed it.

1820

Maybe this helps, pbr status with mwan3 installed and removed;

Without mwan3

root@Router:/etc/config# /etc/init.d/pbr status
pbr 0.9.8-14 running on OpenWrt 22.03.0. WAN (IPv4): wan/eth0.10/82.75.56.1.
============================================================
Dnsmasq version 2.86  Copyright (c) 2000-2021 Simon Kelley
Compile time options: IPv6 GNU-getopt no-DBus UBus no-i18n no-IDN DHCP DHCPv6 no-Lua TFTP conntrack ipset auth cryptohash DNSSEC no-ID loop-detect inotify dumpfile
============================================================
Routes/IP Rules
default         82-75-56-1.cabl 0.0.0.0         UG    10     0        0 eth0.10
default         192.168.0.1     0.0.0.0         UG    20     0        0 wlan0

IPv4 Table 201: default via 82.75.56.1 dev eth0.10 
10.0.10.0/24 dev eth0 proto kernel scope link src 10.0.10.1 
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 
192.168.180.0/24 dev eth0.30 proto kernel scope link src 192.168.180.1 
192.168.200.0/24 dev tun0 proto kernel scope link src 192.168.200.1 
IPv4 Table 201 Rules:
30000:	from all fwmark 0x10000/0xff0000 lookup pbr_wan

IPv4 Table 202: default via 10.31.0.2 dev tun1 
10.0.10.0/24 dev eth0 proto kernel scope link src 10.0.10.1 
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 
192.168.180.0/24 dev eth0.30 proto kernel scope link src 192.168.180.1 
192.168.200.0/24 dev tun0 proto kernel scope link src 192.168.200.1 
IPv4 Table 202 Rules:
29999:	from all fwmark 0x20000/0xff0000 lookup pbr_vpnclient

IPv4 Table 203: default via 192.168.0.1 dev wlan0 
10.0.10.0/24 dev eth0 proto kernel scope link src 10.0.10.1 
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 
192.168.180.0/24 dev eth0.30 proto kernel scope link src 192.168.180.1 
192.168.200.0/24 dev tun0 proto kernel scope link src 192.168.200.1 
IPv4 Table 203 Rules:
29998:	from all fwmark 0x30000/0xff0000 lookup pbr_wwan

IPv4 Table 204: default via 10.2.0.2 dev wg0 
10.0.10.0/24 dev eth0 proto kernel scope link src 10.0.10.1 
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 
192.168.180.0/24 dev eth0.30 proto kernel scope link src 192.168.180.1 
192.168.200.0/24 dev tun0 proto kernel scope link src 192.168.200.1 
IPv4 Table 204 Rules:
29997:	from all fwmark 0x40000/0xff0000 lookup pbr_wg0
============================================================
Mangle IP Table: PREROUTING
-N PBR_PREROUTING
-A PBR_PREROUTING -s 10.2.0.0/24 -m comment --comment ignore_local_traffic -c 0 0 -j RETURN
-A PBR_PREROUTING -s 192.168.1.100/30 -m comment --comment Alle_dhcp_clients -c 173 33679 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.1.104/29 -m comment --comment Alle_dhcp_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.1.112/28 -m comment --comment Alle_dhcp_clients -c 1305 255425 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.1.128/25 -m comment --comment Alle_dhcp_clients -c 7865 4763857 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.2/31 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.4/30 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.8/29 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.16/28 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.32/28 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.48/31 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.50/32 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_dst_ip_cfg076ff5 dst -m comment --comment google -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_dst_ip_cfg086ff5 dst -m comment --comment netflix -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_dst_ip_cfg096ff5 dst -m comment --comment Afas -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_dst_ip dst -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_src_ip src -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_dst_net dst -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_src_net src -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_src_mac src -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_vpnclient_4_dst_ip dst -c 0 0 -g PBR_MARK_0x020000
-A PBR_PREROUTING -m set --match-set pbr_vpnclient_4_src_ip src -c 0 0 -g PBR_MARK_0x020000
-A PBR_PREROUTING -m set --match-set pbr_vpnclient_4_dst_net dst -c 0 0 -g PBR_MARK_0x020000
-A PBR_PREROUTING -m set --match-set pbr_vpnclient_4_src_net src -c 0 0 -g PBR_MARK_0x020000
-A PBR_PREROUTING -m set --match-set pbr_vpnclient_4_src_mac src -c 0 0 -g PBR_MARK_0x020000
-A PBR_PREROUTING -m set --match-set pbr_wwan_4_dst_ip dst -c 0 0 -g PBR_MARK_0x030000
-A PBR_PREROUTING -m set --match-set pbr_wwan_4_src_ip src -c 0 0 -g PBR_MARK_0x030000
-A PBR_PREROUTING -m set --match-set pbr_wwan_4_dst_net dst -c 0 0 -g PBR_MARK_0x030000
-A PBR_PREROUTING -m set --match-set pbr_wwan_4_src_net src -c 0 0 -g PBR_MARK_0x030000
-A PBR_PREROUTING -m set --match-set pbr_wwan_4_src_mac src -c 0 0 -g PBR_MARK_0x030000
-A PBR_PREROUTING -m set --match-set pbr_wg0_4_dst_ip dst -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -m set --match-set pbr_wg0_4_src_ip src -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -m set --match-set pbr_wg0_4_dst_net dst -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -m set --match-set pbr_wg0_4_src_net src -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -m set --match-set pbr_wg0_4_src_mac src -c 0 0 -g PBR_MARK_0x040000
============================================================
Mangle IP Table MARK Chain: PBR_MARK_0x010000
-N PBR_MARK_0x010000
-A PBR_MARK_0x010000 -c 0 0 -j MARK --set-xmark 0x10000/0xff0000
-A PBR_MARK_0x010000 -c 0 0 -j RETURN
============================================================
Mangle IP Table MARK Chain: PBR_MARK_0x020000
-N PBR_MARK_0x020000
-A PBR_MARK_0x020000 -c 0 0 -j MARK --set-xmark 0x20000/0xff0000
-A PBR_MARK_0x020000 -c 0 0 -j RETURN
============================================================
Mangle IP Table MARK Chain: PBR_MARK_0x030000
-N PBR_MARK_0x030000
-A PBR_MARK_0x030000 -c 0 0 -j MARK --set-xmark 0x30000/0xff0000
-A PBR_MARK_0x030000 -c 0 0 -j RETURN
============================================================
Mangle IP Table MARK Chain: PBR_MARK_0x040000
-N PBR_MARK_0x040000
-A PBR_MARK_0x040000 -c 9355 5054013 -j MARK --set-xmark 0x40000/0xff0000
-A PBR_MARK_0x040000 -c 9355 5054013 -j RETURN
============================================================
Current ipsets
create pbr_ignore_4_src_net_cfg046ff5 hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_src_net_cfg056ff5 hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_src_net_cfg066ff5 hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_dst_ip_cfg076ff5 hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_dst_ip_cfg086ff5 hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_dst_ip_cfg096ff5 hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_dst_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_src_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_dst_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_src_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_src_mac hash:mac hashsize 1024 maxelem 65536 comment
create pbr_vpnclient_4_dst_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_vpnclient_4_src_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_vpnclient_4_dst_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_vpnclient_4_src_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_vpnclient_4_src_mac hash:mac hashsize 1024 maxelem 65536 comment
create pbr_wwan_4_dst_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wwan_4_src_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wwan_4_dst_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wwan_4_src_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wwan_4_src_mac hash:mac hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_dst_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_src_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_dst_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_src_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_src_mac hash:mac hashsize 1024 maxelem 65536 comment
============================================================
DNSMASQ sets
ipset=/google.nl/pbr_wan_4_dst_ip_cfg076ff5 # google: google.nl
ipset=/google.com/pbr_wan_4_dst_ip_cfg076ff5 # google: google.com
ipset=/netflix.com/pbr_wan_4_dst_ip_cfg086ff5 # netflix: netflix.com
ipset=/afasinsite.nl/pbr_wan_4_dst_ip_cfg096ff5 # Afas: afasinsite.nl
============================================================
Your support details have been logged to '/var/pbr-support'. [✓]
root@Router:/etc/config#

With mwan3 installed.

root@Router:/etc/config# /etc/init.d/pbr status
pbr 0.9.8-14 running on OpenWrt 22.03.0. WAN (IPv4): wan/eth0.10/82.75.56.1.
============================================================
Dnsmasq version 2.86  Copyright (c) 2000-2021 Simon Kelley
Compile time options: IPv6 GNU-getopt no-DBus UBus no-i18n no-IDN DHCP DHCPv6 no-Lua TFTP conntrack ipset auth cryptohash DNSSEC no-ID loop-detect inotify dumpfile
============================================================
Routes/IP Rules
default         82-75-56-1.cabl 0.0.0.0         UG    10     0        0 eth0.10
default         192.168.0.1     0.0.0.0         UG    20     0        0 wlan0

IPv4 Table 201: default via 82.75.56.1 dev eth0.10 
10.0.10.0/24 dev eth0 proto kernel scope link src 10.0.10.1 
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 
192.168.180.0/24 dev eth0.30 proto kernel scope link src 192.168.180.1 
192.168.200.0/24 dev tun0 proto kernel scope link src 192.168.200.1 
IPv4 Table 201 Rules:
30000:	from all fwmark 0x10000/0xff0000 lookup pbr_wan

IPv4 Table 202: default via 10.31.0.2 dev tun1 
10.0.10.0/24 dev eth0 proto kernel scope link src 10.0.10.1 
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 
192.168.180.0/24 dev eth0.30 proto kernel scope link src 192.168.180.1 
192.168.200.0/24 dev tun0 proto kernel scope link src 192.168.200.1 
IPv4 Table 202 Rules:
29999:	from all fwmark 0x20000/0xff0000 lookup pbr_vpnclient

IPv4 Table 203: default via 192.168.0.1 dev wlan0 
10.0.10.0/24 dev eth0 proto kernel scope link src 10.0.10.1 
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 
192.168.180.0/24 dev eth0.30 proto kernel scope link src 192.168.180.1 
192.168.200.0/24 dev tun0 proto kernel scope link src 192.168.200.1 
IPv4 Table 203 Rules:
29998:	from all fwmark 0x30000/0xff0000 lookup pbr_wwan

IPv4 Table 204: default via 10.2.0.2 dev wg0 
10.0.10.0/24 dev eth0 proto kernel scope link src 10.0.10.1 
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 
192.168.180.0/24 dev eth0.30 proto kernel scope link src 192.168.180.1 
192.168.200.0/24 dev tun0 proto kernel scope link src 192.168.200.1 
IPv4 Table 204 Rules:
29997:	from all fwmark 0x40000/0xff0000 lookup pbr_wg0

Error: ipv4: FIB table does not exist.
Dump terminated
IPv4 Table 205: 
IPv4 Table 205 Rules:

Error: ipv4: FIB table does not exist.
Dump terminated
IPv4 Table 206: 
IPv4 Table 206 Rules:

Error: ipv4: FIB table does not exist.
Dump terminated
IPv4 Table 207: 
IPv4 Table 207 Rules:

Error: ipv4: FIB table does not exist.
Dump terminated
IPv4 Table 208: 
IPv4 Table 208 Rules:

Error: ipv4: FIB table does not exist.
Dump terminated
IPv4 Table 209: 
IPv4 Table 209 Rules:

Error: ipv4: FIB table does not exist.
Dump terminated
IPv4 Table 210: 
IPv4 Table 210 Rules:
============================================================
Mangle IP Table: PREROUTING
-N PBR_PREROUTING
-A PBR_PREROUTING -s 10.2.0.0/24 -m comment --comment ignore_local_traffic -c 0 0 -j RETURN
-A PBR_PREROUTING -s 192.168.1.100/30 -m comment --comment Alle_dhcp_clients -c 2177 565680 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.1.104/29 -m comment --comment Alle_dhcp_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.1.112/28 -m comment --comment Alle_dhcp_clients -c 2105 445068 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.1.128/25 -m comment --comment Alle_dhcp_clients -c 11891 5781077 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.2/31 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.4/30 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.8/29 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.16/28 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.32/28 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.48/31 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -s 192.168.200.50/32 -m comment --comment vpn_server_clients -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_dst_ip_cfg076ff5 dst -m comment --comment google -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_dst_ip_cfg086ff5 dst -m comment --comment netflix -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_dst_ip_cfg096ff5 dst -m comment --comment Afas -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_dst_ip dst -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_src_ip src -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_dst_net dst -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_src_net src -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_wan_4_src_mac src -c 0 0 -g PBR_MARK_0x010000
-A PBR_PREROUTING -m set --match-set pbr_vpnclient_4_dst_ip dst -c 0 0 -g PBR_MARK_0x020000
-A PBR_PREROUTING -m set --match-set pbr_vpnclient_4_src_ip src -c 0 0 -g PBR_MARK_0x020000
-A PBR_PREROUTING -m set --match-set pbr_vpnclient_4_dst_net dst -c 0 0 -g PBR_MARK_0x020000
-A PBR_PREROUTING -m set --match-set pbr_vpnclient_4_src_net src -c 0 0 -g PBR_MARK_0x020000
-A PBR_PREROUTING -m set --match-set pbr_vpnclient_4_src_mac src -c 0 0 -g PBR_MARK_0x020000
-A PBR_PREROUTING -m set --match-set pbr_wwan_4_dst_ip dst -c 0 0 -g PBR_MARK_0x030000
-A PBR_PREROUTING -m set --match-set pbr_wwan_4_src_ip src -c 0 0 -g PBR_MARK_0x030000
-A PBR_PREROUTING -m set --match-set pbr_wwan_4_dst_net dst -c 0 0 -g PBR_MARK_0x030000
-A PBR_PREROUTING -m set --match-set pbr_wwan_4_src_net src -c 0 0 -g PBR_MARK_0x030000
-A PBR_PREROUTING -m set --match-set pbr_wwan_4_src_mac src -c 0 0 -g PBR_MARK_0x030000
-A PBR_PREROUTING -m set --match-set pbr_wg0_4_dst_ip dst -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -m set --match-set pbr_wg0_4_src_ip src -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -m set --match-set pbr_wg0_4_dst_net dst -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -m set --match-set pbr_wg0_4_src_net src -c 0 0 -g PBR_MARK_0x040000
-A PBR_PREROUTING -m set --match-set pbr_wg0_4_src_mac src -c 0 0 -g PBR_MARK_0x040000
============================================================
Mangle IP Table MARK Chain: PBR_MARK_0x010000
-N PBR_MARK_0x010000
-A PBR_MARK_0x010000 -c 0 0 -j MARK --set-xmark 0x10000/0xff0000
-A PBR_MARK_0x010000 -c 0 0 -j RETURN
============================================================
Mangle IP Table MARK Chain: PBR_MARK_0x020000
-N PBR_MARK_0x020000
-A PBR_MARK_0x020000 -c 0 0 -j MARK --set-xmark 0x20000/0xff0000
-A PBR_MARK_0x020000 -c 0 0 -j RETURN
============================================================
Mangle IP Table MARK Chain: PBR_MARK_0x030000
-N PBR_MARK_0x030000
-A PBR_MARK_0x030000 -c 0 0 -j MARK --set-xmark 0x30000/0xff0000
-A PBR_MARK_0x030000 -c 0 0 -j RETURN
============================================================
Mangle IP Table MARK Chain: PBR_MARK_0x040000
-N PBR_MARK_0x040000
-A PBR_MARK_0x040000 -c 16184 6792634 -j MARK --set-xmark 0x40000/0xff0000
-A PBR_MARK_0x040000 -c 16184 6792634 -j RETURN
============================================================
Current ipsets
create pbr_ignore_4_src_net_cfg046ff5 hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_src_net_cfg056ff5 hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_src_net_cfg066ff5 hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_dst_ip_cfg076ff5 hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_dst_ip_cfg086ff5 hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_dst_ip_cfg096ff5 hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_dst_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_src_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_dst_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_src_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wan_4_src_mac hash:mac hashsize 1024 maxelem 65536 comment
create pbr_vpnclient_4_dst_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_vpnclient_4_src_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_vpnclient_4_dst_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_vpnclient_4_src_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_vpnclient_4_src_mac hash:mac hashsize 1024 maxelem 65536 comment
create pbr_wwan_4_dst_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wwan_4_src_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wwan_4_dst_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wwan_4_src_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wwan_4_src_mac hash:mac hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_dst_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_src_ip hash:ip family inet hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_dst_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_src_net hash:net family inet hashsize 1024 maxelem 65536 comment
create pbr_wg0_4_src_mac hash:mac hashsize 1024 maxelem 65536 comment
create mwan3_dynamic_ipv6 hash:net family inet6 hashsize 1024 maxelem 65536
create mwan3_connected_ipv4 hash:net family inet hashsize 1024 maxelem 65536
add mwan3_connected_ipv4 192.168.200.255
add mwan3_connected_ipv4 10.0.10.0
add mwan3_connected_ipv4 127.0.0.1
add mwan3_connected_ipv4 192.168.200.0
add mwan3_connected_ipv4 192.168.0.0/24
add mwan3_connected_ipv4 10.31.0.2
add mwan3_connected_ipv4 10.31.0.0
add mwan3_connected_ipv4 127.0.0.0/8
add mwan3_connected_ipv4 192.168.1.255
add mwan3_connected_ipv4 10.31.255.255
add mwan3_connected_ipv4 192.168.200.1
add mwan3_connected_ipv4 82.75.57.255
add mwan3_connected_ipv4 82.75.56.0/23
add mwan3_connected_ipv4 10.31.0.0/16
add mwan3_connected_ipv4 10.0.10.1
add mwan3_connected_ipv4 192.168.1.1
add mwan3_connected_ipv4 127.255.255.255
add mwan3_connected_ipv4 192.168.180.0
add mwan3_connected_ipv4 192.168.0.109
add mwan3_connected_ipv4 192.168.180.1
add mwan3_connected_ipv4 192.168.180.255
add mwan3_connected_ipv4 192.168.180.0/24
add mwan3_connected_ipv4 192.168.200.0/24
add mwan3_connected_ipv4 127.0.0.0
add mwan3_connected_ipv4 82.75.56.0
add mwan3_connected_ipv4 10.0.10.0/24
add mwan3_connected_ipv4 185.107.56.85
add mwan3_connected_ipv4 10.2.0.2
add mwan3_connected_ipv4 10.0.10.255
add mwan3_connected_ipv4 192.168.1.0
add mwan3_connected_ipv4 192.168.0.255
add mwan3_connected_ipv4 [my wan address]
add mwan3_connected_ipv4 192.168.0.0
add mwan3_connected_ipv4 192.168.1.0/24
add mwan3_connected_ipv4 224.0.0.0/3
create mwan3_connected_ipv6 hash:net family inet6 hashsize 1024 maxelem 65536
add mwan3_connected_ipv6 fe80::/64
add mwan3_connected_ipv6 fdad:b459:8594::/64
add mwan3_connected_ipv6 2001:1c04:1305:1300::/64
create mwan3_custom_ipv4 hash:net family inet hashsize 1024 maxelem 65536
create mwan3_custom_ipv6 hash:net family inet6 hashsize 1024 maxelem 65536
create mwan3_rule_ipv4_https hash:ip,mark family inet markmask 0x00003f00 hashsize 1024 maxelem 65536 timeout 600
add mwan3_rule_ipv4_https 192.168.1.102,0x00000100 timeout 580
add mwan3_rule_ipv4_https 192.168.1.142,0x00000100 timeout 586
add mwan3_rule_ipv4_https 192.168.1.103,0x00000100 timeout 586
add mwan3_rule_ipv4_https 192.168.1.119,0x00000100 timeout 588
add mwan3_rule_ipv4_https 192.168.1.211,0x00000100 timeout 595
add mwan3_rule_ipv4_https 10.2.0.2,0x00000100 timeout 596
add mwan3_rule_ipv4_https 192.168.1.121,0x00000100 timeout 579
add mwan3_rule_ipv4_https [my wan address],0x00000100 timeout 556
add mwan3_rule_ipv4_https 192.168.180.221,0x00000100 timeout 596
add mwan3_rule_ipv4_https 192.168.1.173,0x00000100 timeout 586
add mwan3_rule_ipv4_https 192.168.1.129,0x00000100 timeout 593
create mwan3_rule_ipv6_https hash:ip,mark family inet6 markmask 0x00003f00 hashsize 1024 maxelem 65536 timeout 600
add mwan3_rule_ipv6_https 2001:1c04:1305:1300::53b,0x00003e00 timeout 596
create mwan3_dynamic_ipv4 list:set size 8
============================================================
DNSMASQ sets
ipset=/google.nl/pbr_wan_4_dst_ip_cfg076ff5 # google: google.nl
ipset=/google.com/pbr_wan_4_dst_ip_cfg076ff5 # google: google.com
ipset=/netflix.com/pbr_wan_4_dst_ip_cfg086ff5 # netflix: netflix.com
ipset=/afasinsite.nl/pbr_wan_4_dst_ip_cfg096ff5 # Afas: afasinsite.nl
============================================================
Your support details have been logged to '/var/pbr-support'. [✓]
1821

Sorry, I can't be of any more help with regards to mwan3. If/when it's converted to nft, it might be easier to ensure compatibility, because (I'd hope) it would use the same fw4 table as everything else in OpenWrt.

1822

Ok thanks @stangri, I'll put it (mwan3 issue) on hold for now, ti's only my fallback wwan, that I have but never needed so far.

1823

Another question, why is pbr telling me this?

"Please note that dnsmasq.nftset is not supported on this system."

I can now set "disable' and "dnsmasq.ipset" what I use because of the domain resolving, but just curieus what dnsmasq.nfset does.

1824

Uses dnsmasq nft sets support for domains encountered in policies.

1 Like
1825

I installed a fresh 22.03.1, (because maybe my old settings are the reason that some stuff is not working correctly) then installed pbr edit the config file with my old settings and al works.

Installed https-dns-proxy, this breaks the domain resolving with the default settings. The option for dnsmasq.ipset is gone in pbr and errors appear that the domain resolving is gone.

I'm back to 22.03.0 now, (where this config works) I will try to troubleshoot it end of the week when I'm back home.

1826

Hi,
I'm using OpenWrt 22.03.2 and [pbr 0.9.8-18]
When a Custom User File is enabled i see that the ip list from it is going through the wan and not tun0.
How can i change it to go through tun0 ?

i tried and it didnt work:
TARGET_SET='pbr_tun0_4_dst_ip'
TARGET_IPSET='pbr_tun0_4_dst_net'

also tried adding to /etc/config/pbr :
config include
option path '/usr/share/pbr/pbr.user.nbc'
option interface 'tun0'
option enabled '1'

1827

Post your /etc/config/network file.

1828 
config interface 'loopback'
	option device 'lo'
	option proto 'static'
	option ipaddr '127.0.0.1'
	option netmask '255.0.0.0'

config globals 'globals'
	option ula_prefix 'fdf7:xxxx:xxxx::/48'
	option packet_steering '1'

config device
	option name 'br-lan'
	option type 'bridge'
	list ports 'lan1'
	list ports 'lan2'
	list ports 'lan3'

config interface 'lan'
	option device 'br-lan'
	option proto 'static'
	option netmask '255.255.255.0'
	option ip6assign '60'
	option ipaddr '192.168.0.1'

config device
	option name 'wan'
	option macaddr 'xx:xx:xx:xx:xx:xx'

config interface 'wan'
	option device 'wan'
	option proto 'dhcp'
	option metric '1024'
	option hostname '*'
	option peerdns '0'
	list dns 'x.x.x.x'

config interface 'wan6'
	option device 'wan'
	option proto 'dhcpv6'
	option reqaddress 'try'
	option reqprefix 'auto'
	option auto '0'

config interface 'tun0'
	option proto 'none'
	option device 'tun0'
	option peerdns '0'
	list dns 'x.x.x.x'
1829

Hi back at home base.

Installed 22.03.2 missing the option to resolv hostnames with dnsmasq ipset. I don't have the option available as I mentioned earlier.

root@Router:~# opkg list-installed | grep dnsmasq; opkg list-installed | grep pb
r;
dnsmasq-full - 2.86-14
luci-app-pbr - 0.9.8-18
pbr - 0.9.8-18
root@Router:~#
1830

Could you please post the output of dnsmasq -v?

1831

And your user file and output of service pbr restart; sleep 3; service pbr status; please.

1832

The user file is exactly the same like your pbr.user.netflix.
I've change in the user file to:

TARGET_SET='pbr_tun0_4_dst_ip'
TARGET_IPSET='pbr_tun0_4_dst_net'
TARGET_ASN='17035'

And now it's working okay.

Thank you stangri.