OK, here goes. Wireguard is the primary route. I've got some simple IPv4 rules set up that work great for directing traffic on my lan vlan subnet to wan (bypassing wireguard) but I can't get IPv6 to work.
uci export network:
package network
config globals 'globals'
option ula_prefix 'fdc2:9aea:13b1::/48'
config interface 'loopback'
option proto 'static'
option ipaddr '127.0.0.1'
option netmask '255.0.0.0'
option device 'lo'
config interface 'lan'
option proto 'static'
option netmask '255.255.255.0'
option ipaddr '192.168.1.1'
option ip6assign '64'
option ip6ifaceid '::1'
option ip6hint '1'
option device 'br-lan'
config interface 'wan'
option proto 'dhcp'
option device 'eth0'
config interface 'wan6'
option proto 'dhcpv6'
option reqaddress 'try'
option reqprefix '56'
option device 'eth0'
option ip6table '2'
config interface 'wireguard'
option proto 'wireguard'
option private_key 'redacted'
list addresses 'redacted/32'
list addresses 'fc00:bbbb:redacted/128'
config wireguard_wireguard
option persistent_keepalive '25'
option public_key 'redacted'
option endpoint_host 'redacted'
option description 'redacted'
list allowed_ips '0.0.0.0/0'
list allowed_ips '::0/0'
option endpoint_port '51820'
option route_allowed_ips '1'
config interface 'vpn'
option proto 'static'
option ipaddr '192.168.2.1'
option netmask '255.255.255.0'
list dns 'redacted'
option device 'br-vpn'
option ip6ifaceid '::1'
option ip6assign '64'
option ip6hint '2'
list ip6class 'local'
config interface 'dmz'
option proto 'static'
option ipaddr '192.168.3.1'
option netmask '255.255.255.0'
option ip6assign '64'
option ip6ifaceid '::1'
option ip6hint '3'
option device 'br-dmz'
config interface 'iot'
option proto 'static'
option ipaddr '192.168.4.1'
option ip6assign '64'
option ip6hint '4'
option ip6ifaceid '::1'
option netmask '255.255.255.0'
option device 'br-iot'
config device
option name 'br-lan'
option type 'bridge'
option stp '1'
list ports 'eth1'
config device
option name 'br-vpn'
option type 'bridge'
list ports 'eth1.2'
config device
option name 'br-dmz'
option type 'bridge'
list ports 'eth1.3'
config device
option name 'br-iot'
option type 'bridge'
list ports 'eth1.4'
uci export firewall:
package firewall
config defaults
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'REJECT'
option synflood_protect '1'
config zone
option name 'lan'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
list network 'dmz'
list network 'lan'
config zone
option name 'wan'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
option masq '1'
option mtu_fix '1'
list network 'wan'
list network 'wan6'
config zone
option name 'iot'
option output 'ACCEPT'
list network 'iot'
option forward 'REJECT'
option input 'ACCEPT'
config zone
option name 'vpn'
option output 'ACCEPT'
list network 'vpn'
option forward 'REJECT'
option input 'ACCEPT'
config zone
option name 'wireguard'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
option masq '1'
option mtu_fix '1'
list network 'wireguard'
config forwarding
option src 'lan'
option dest 'wan'
config forwarding
option src 'lan'
option dest 'vpn'
config forwarding
option src 'lan'
option dest 'iot'
config forwarding
option src 'vpn'
option dest 'lan'
config forwarding
option src 'vpn'
option dest 'wireguard'
config forwarding
option src 'vpn'
option dest 'iot'
config forwarding
option src 'iot'
option dest 'wireguard'
config forwarding
option src 'iot'
option dest 'wan'
config rule
option name 'Allow-DHCP-Renew'
option src 'wan'
option proto 'udp'
option dest_port '68'
option target 'ACCEPT'
option family 'ipv4'
config rule
option name 'Allow-Ping'
option src 'wan'
option proto 'icmp'
option icmp_type 'echo-request'
option family 'ipv4'
option target 'ACCEPT'
config rule
option name 'Allow-IGMP'
option src 'wan'
option proto 'igmp'
option family 'ipv4'
option target 'ACCEPT'
config rule
option name 'Allow-DHCPv6'
option src 'wan'
option proto 'udp'
option src_ip 'fc00::/6'
option dest_ip 'fc00::/6'
option dest_port '546'
option family 'ipv6'
option target 'ACCEPT'
config rule
option name 'Allow-MLD'
option src 'wan'
option proto 'icmp'
option src_ip 'fe80::/10'
list icmp_type '130/0'
list icmp_type '131/0'
list icmp_type '132/0'
list icmp_type '143/0'
option family 'ipv6'
option target 'ACCEPT'
config rule
option name 'Allow-ICMPv6-Input'
option src 'wan'
option proto 'icmp'
list icmp_type 'echo-request'
list icmp_type 'echo-reply'
list icmp_type 'destination-unreachable'
list icmp_type 'packet-too-big'
list icmp_type 'time-exceeded'
list icmp_type 'bad-header'
list icmp_type 'unknown-header-type'
list icmp_type 'router-solicitation'
list icmp_type 'neighbour-solicitation'
list icmp_type 'router-advertisement'
list icmp_type 'neighbour-advertisement'
option limit '1000/sec'
option family 'ipv6'
option target 'ACCEPT'
config rule
option name 'Allow-ICMPv6-Forward'
option src 'wan'
option dest '*'
option proto 'icmp'
list icmp_type 'echo-request'
list icmp_type 'echo-reply'
list icmp_type 'destination-unreachable'
list icmp_type 'packet-too-big'
list icmp_type 'time-exceeded'
list icmp_type 'bad-header'
list icmp_type 'unknown-header-type'
option limit '1000/sec'
option family 'ipv6'
option target 'ACCEPT'
option enabled ''\''0'\'''
config rule
option name 'Allow-IPSec-ESP'
option src 'wan'
option dest 'lan'
option proto 'esp'
option target 'ACCEPT'
config rule
option name 'Allow-ISAKMP'
option src 'wan'
option dest 'lan'
option dest_port '500'
option proto 'udp'
option target 'ACCEPT'
config include
option path '/etc/firewall.user'
config include 'miniupnpd'
option type 'script'
option path '/usr/share/miniupnpd/firewall.include'
option family 'any'
option reload '1'
config include 'pbr'
option type 'script'
option path '/usr/share/pbr/pbr.firewall.include'
option family 'any'
option reload '1'
/etc/init.d/pbr status -d:
pbr 0.9.4-10 running on OpenWrt SNAPSHOT.
============================================================
Dnsmasq version 2.86 Copyright (c) 2000-2021 Simon Kelley
Compile time options: IPv6 GNU-getopt no-DBus UBus no-i18n no-IDN DHCP DHCPv6 no-Lua TFTP conntrack ipset auth cryptohash DNSSEC no-ID loop-detect inotify dumpfile
============================================================
Routes/IP Rules
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
default * 0.0.0.0 U 0 0 0 wireguard
redacted * 255.255.192.0 U 0 0 0 eth0
redacted 035-143-192-001 255.255.255.255 UGH 0 0 0 eth0
192.168.1.0 * 255.255.255.0 U 0 0 0 br-lan
192.168.2.0 * 255.255.255.0 U 0 0 0 br-vpn
192.168.3.0 * 255.255.255.0 U 0 0 0 br-dmz
192.168.4.0 * 255.255.255.0 U 0 0 0 br-iot
0: from all lookup local
29999: from all fwmark 0x20000/0xff0000 lookup wireguard
30000: from all fwmark 0x10000/0xff0000 lookup wan
32766: from all lookup main
32767: from all lookup default
IPv4 Table 201: default via redacted dev eth0
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1
192.168.2.0/24 dev br-vpn proto kernel scope link src 192.168.2.1
192.168.3.0/24 dev br-dmz proto kernel scope link src 192.168.3.1
192.168.4.0/24 dev br-iot proto kernel scope link src 192.168.4.1
IPv4 Table 201 Rules:
30000: from all fwmark 0x10000/0xff0000 lookup wan
IPv4 Table 202: default via 10.65.5.239 dev wireguard
192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1
192.168.2.0/24 dev br-vpn proto kernel scope link src 192.168.2.1
192.168.3.0/24 dev br-dmz proto kernel scope link src 192.168.3.1
192.168.4.0/24 dev br-iot proto kernel scope link src 192.168.4.1
IPv4 Table 202 Rules:
29999: from all fwmark 0x20000/0xff0000 lookup wireguard
Error: ipv6: FIB table does not exist.
Dump terminated
IPv6 Table 202: default dev wireguard proto static metric 1024 pref medium
============================================================
Mangle IP Table
# Warning: iptables-legacy tables present, use iptables-legacy to see them
Chain PREROUTING (policy ACCEPT)
target prot opt source destination
TTL all -- anywhere base-address.mcast.net/4 TTL increment by 1
TTL all -- anywhere base-address.mcast.net/4 TTL increment by 1
PBR_PREROUTING all -- anywhere anywhere mark match 0x0/0xff0000
Chain INPUT (policy ACCEPT)
target prot opt source destination
Chain FORWARD (policy ACCEPT)
target prot opt source destination
Chain OUTPUT (policy ACCEPT)
target prot opt source destination
Chain POSTROUTING (policy ACCEPT)
target prot opt source destination
Chain PBR_MARK0x010000 (5 references)
target prot opt source destination
MARK all -- anywhere anywhere MARK xset 0x10000/0xff0000
RETURN all -- anywhere anywhere
Chain PBR_MARK0x020000 (0 references)
target prot opt source destination
MARK all -- anywhere anywhere MARK xset 0x20000/0xff0000
RETURN all -- anywhere anywhere
Chain PBR_PREROUTING (1 references)
target prot opt source destination
PBR_MARK0x010000 all -- 192.168.1.0/24 anywhere [goto] /* dmz4 */
============================================================
Mangle IPv6 Table
Chain PREROUTING (policy ACCEPT)
target prot opt source destination
PBR_PREROUTING all anywhere anywhere mark match 0x0/0xff0000
Chain INPUT (policy ACCEPT)
target prot opt source destination
Chain FORWARD (policy ACCEPT)
target prot opt source destination
TCPMSS tcp anywhere anywhere tcp flags:SYN,RST/SYN /* !fw3: Zone wan MTU fixing */ TCPMSS clamp to PMTU
TCPMSS tcp anywhere anywhere tcp flags:SYN,RST/SYN /* !fw3: Zone wan MTU fixing */ TCPMSS clamp to PMTU
TCPMSS tcp anywhere anywhere tcp flags:SYN,RST/SYN /* !fw3: Zone wireguard MTU fixing */ TCPMSS clamp to PMTU
TCPMSS tcp anywhere anywhere tcp flags:SYN,RST/SYN /* !fw3: Zone wireguard MTU fixing */ TCPMSS clamp to PMTU
Chain OUTPUT (policy ACCEPT)
target prot opt source destination
Chain POSTROUTING (policy ACCEPT)
target prot opt source destination
Chain PBR_MARK0x010000 (0 references)
target prot opt source destination
MARK all anywhere anywhere MARK xset 0x10000/0xff0000
RETURN all anywhere anywhere
Chain PBR_MARK0x020000 (0 references)
target prot opt source destination
MARK all anywhere anywhere MARK xset 0x20000/0xff0000
RETURN all anywhere anywhere
Chain PBR_PREROUTING (1 references)
target prot opt source destination
============================================================
Mangle IP Table MARK Chain: PBR_MARK0x010000
# Warning: iptables-legacy tables present, use iptables-legacy to see them
-N PBR_MARK0x010000
-A PBR_MARK0x010000 -c 77 4587 -j MARK --set-xmark 0x10000/0xff0000
-A PBR_MARK0x010000 -c 77 4587 -j RETURN
============================================================
Mangle IP Table MARK Chain: PBR_MARK0x020000
# Warning: iptables-legacy tables present, use iptables-legacy to see them
-N PBR_MARK0x020000
-A PBR_MARK0x020000 -c 0 0 -j MARK --set-xmark 0x20000/0xff0000
-A PBR_MARK0x020000 -c 0 0 -j RETURN
============================================================
NAT IP Table
# Warning: iptables-legacy tables present, use iptables-legacy to see them
Chain PREROUTING (policy ACCEPT)
target prot opt source destination
Chain INPUT (policy ACCEPT)
target prot opt source destination
Chain OUTPUT (policy ACCEPT)
target prot opt source destination
Chain POSTROUTING (policy ACCEPT)
target prot opt source destination
============================================================
NAT IPv6 Table
ip6tables v1.8.7 (legacy): can't initialize ip6tables table `nat': Table does not exist (do you need to insmod?)
Perhaps ip6tables or your kernel needs to be upgraded.
============================================================
Current ipsets
============================================================
Your support details have been logged to '/var/pbr-support'. [✓]
OK, first question. I am a little confused about directing IPv6 using PBR. At some point in the past I think I had this working correctly w/ VPR but it seems to have stopped working on my new device (that I need to use a snapshot on). I'm also unsure if IPv6 is even supported in this manner any longer and whether NAT6 is required or forbidden (I have completely removed NAT6 for the time being and am just using IPv4 on the vpn vlan, although I would like to eventually enable it for my vpn vlan in the future so I can NAT a single /128). The documentation is a little contradictory (clearly IPv6 is "supported" as it's an option although the README suggests setting up a route6 instead--which I can do am not sure if strictly required nor how to perform this per subnet).
The naive me figured I could just enter either the ULA/GUA IPv6 subnet in CIDR notation and route to wan6 interface but PBR gives me a Unknown fw_mark for wan6 error. I've also tried using a single devices /128 and that didn't seem to work either. So I assume that it is not possible to use PBR in this manner to route IPv6 traffic. Am I forced to set wan as default route and then route ipv4 to wireguard? What if I eventually want to route IPv6 through wireguard with or without NAT6?
I'm also confused about the use of the @device notation. Could I, for instance, just use my @br-lan device to direct all traffic from the lan network to wan/wan6 instead of the associated subnet (192.168.2.0/24). Would this also handle IPv6 (it doesn't seem to make a difference in my tests)? That would allow me to change IP subnets without having to fix in PBR. I've tried this but still no luck.
One common theme is that I don't think clients aren't receiving an IPv6 gateway address. I'm not sure if this is PBR's fault or how this works when making wireguard the default route (with 0.0.0.0/0 and ::0/0).
Thank you so much for this tool and any insight you can provide!
PBR is working well for me, and seems to have solved an issue I had with it not being applied at startup.