PBR: am I making this more complicated than it needs to be?

Installing and Using OpenWrt
1

Looking for some advice on Policy Based Routing (PBR), I don't know if I am making things more complicated than it needs to be or if I am on the right track or maybe there are better approaches that don't rely on explicit ip rule based on IP addressing?

I have 3 outgoing network interfaces to the internet:

  1. WAN to ISP
  2. Wireguard
  3. OpenVPN

Then I have my local networks using VLANs.

  1. LAN (br-lan that comes from openwrt). I leave this interface as default and going thru WAN. I rarely put any devices here.
  2. vlan100 (using wireguard for outbound)
  3. vlan666 (using openvpn for outbound)

How I have set things up for #2 and #3 above is as follows:

  • Wireguard tunnel interface uses routable table ID 20
  • OpenVPN I am doing some post-up routing setup manually in my ovpn file, uses route table ID 30

To try to be lazy, I configured my vlans with 'lookup' rules.

  1. vlan100 rule6 and rule lookup table 20 (out via wireguard)
  2. vlan666 rule lookup table 30 (out via openvpn)

This is all working perfectly, now that I have put some devices into each vlan comes some issues related to inter-vlan communication or routing.

My WIFI network uses vlan100 and some VMs live in vlan666. Trying to connect locally without traversing the internet is not possible, even with forwarding zones. Unless I manually setup a higher priority ip rule for the IPv4 netblock of that interface

ip rule add to 172.100.0.0/24 lookup main
ip rule add to 192.168.100.0/24 lookup main

These two rules make it so packets no longer go via outbound tunnels for the local traffic and makes it possible for vlan100 host 192.168.100.20 to ping 172.100.0.0/24 and receive responses.

In summary, my current setup to make a simple vlan that is routed via a specific gateway depends on:

  1. Static IPv4 address ranges (for ip rules)
  2. Route tables with unique IDs and binded to interfaces to make default route lookups skip 'main' table. (explicit 'rule6' and 'rule' lookup rules)
  3. Firewall zones forwarding between each vlan interface towards the gateway/wan port.
  4. (if I want to allow local communication btwn 2 lans) need to allow forwarding between the 2 vlans
  5. To fix routing, I have to manually do the ip rule trick which has higher priority. I feel this is kinda silly to have to add 2 rules for each VLAN/DHCP scope.

If possible I wanted to set this up in a way that is not too complicated to manage; or if I forget later and change my DHCP range on an interface things will probably break and I may scratch my head for a bit.

I guess my question is to get your opinion on this and see if there are other ways; or perhaps there is a setting that basically tells openwrt to route all packets that are local scopes before it executes the default table lookup?

root@meow:~# ip rule list
0:      from all lookup local
9998:   from all to 192.168.100.0/24 lookup main
9999:   from all to 172.66.6.0/24 lookup main
10000:  from 10.100.100.10 lookup 20
20000:  from all to 10.100.100.10/24 lookup 20
30000:  from all iif eth1.666 lookup 30
30000:  from all iif eth1.100 lookup 20
32766:  from all lookup main
32767:  from all lookup default
90017:  from all iif lo lookup 20
90018:  from all iif lo lookup 30
root@meow:~# ip rule add to 172.100.0.0/24 lookup main
root@meow:~# uci export firewall
package firewall

config defaults
        option input 'ACCEPT'
        option output 'ACCEPT'
        option forward 'REJECT'
        option synflood_protect '1'

config zone
        option name 'lan'
        option input 'ACCEPT'
        option output 'ACCEPT'
        option forward 'ACCEPT'
        option network 'lan'

config zone
        option name 'wan'
        list network 'wan'
        list network 'wan6'
        option output 'ACCEPT'
        option forward 'REJECT'
        option masq '1'
        option mtu_fix '1'
        option masq6 '1'
        option input 'ACCEPT'

config forwarding
        option src 'lan'
        option dest 'wan'

config forwarding
        option src 'lan'
        option dest 'untrusted'

config rule
        option name 'Allow-DHCP-Renew'
        option src 'wan'
        option proto 'udp'
        option dest_port '68'
        option target 'ACCEPT'
        option family 'ipv4'

config rule
        option name 'Allow-Ping'
        option src 'wan'
        option proto 'icmp'
        option icmp_type 'echo-request'
        option family 'ipv4'
        option target 'ACCEPT'

config rule
        option name 'Allow-IGMP'
        option src 'wan'
        option proto 'igmp'
        option family 'ipv4'
        option target 'ACCEPT'

config rule
        option name 'Allow-DHCPv6'
        option src 'wan'
        option proto 'udp'
        option src_ip 'fc00::/6'
        option dest_ip 'fc00::/6'
        option dest_port '546'
        option family 'ipv6'
        option target 'ACCEPT'

config rule
        option name 'Allow-MLD'
        option src 'wan'
        option proto 'icmp'
        option src_ip 'fe80::/10'
        list icmp_type '130/0'
        list icmp_type '131/0'
        list icmp_type '132/0'
        list icmp_type '143/0'
        option family 'ipv6'
        option target 'ACCEPT'

config rule
        option name 'Allow-ICMPv6-Input'
        option src 'wan'
        option proto 'icmp'
        list icmp_type 'echo-request'
        list icmp_type 'echo-reply'
        list icmp_type 'destination-unreachable'
        list icmp_type 'packet-too-big'
        list icmp_type 'time-exceeded'
        list icmp_type 'bad-header'
        list icmp_type 'unknown-header-type'
        list icmp_type 'router-solicitation'
        list icmp_type 'neighbour-solicitation'
        list icmp_type 'router-advertisement'
        list icmp_type 'neighbour-advertisement'
        option limit '1000/sec'
        option family 'ipv6'
        option target 'ACCEPT'

config rule
        option name 'Allow-ICMPv6-Forward'
        option src 'wan'
        option dest '*'
        option proto 'icmp'
        list icmp_type 'echo-request'
        list icmp_type 'echo-reply'
        list icmp_type 'destination-unreachable'
        list icmp_type 'packet-too-big'
        list icmp_type 'time-exceeded'
        list icmp_type 'bad-header'
        list icmp_type 'unknown-header-type'
        option enabled '0'

config rule
        option name 'Allow-IPSec-ESP'
        option src 'wan'
        option dest 'lan'
        option proto 'esp'
        option target 'ACCEPT'

config rule
        option name 'Allow-ISAKMP'
        option src 'wan'
        option dest 'lan'
        option dest_port '500'
        option proto 'udp'
        option target 'ACCEPT'

config rule
        option name 'Support-UDP-Traceroute'
        option src 'wan'
        option dest_port '33434:33689'
        option proto 'udp'
        option family 'ipv4'
        option target 'REJECT'
        option enabled '0'

config include
        option path '/etc/firewall.user'

config zone 'docker'
        option input 'ACCEPT'
        option output 'ACCEPT'
        option forward 'ACCEPT'
        option name 'docker'
        list network 'docker'

config include
        option path '/etc/firewall.nat6'
        option name 'nat6'
        option reload '1'

config zone
        option name 'vpn_privacy'
        option input 'ACCEPT'
        option forward 'ACCEPT'
        option network 'vpn_privacy'
        option output 'ACCEPT'
        list device 'eth1.666'

config rule
        option name 'Allow-vlan666-Ping'
        option src 'vpn_privacy'
        option target 'ACCEPT'
        option proto 'icmp'

config rule
        option name 'vlan666-to-router'
        option src 'vpn_privacy'
        option target 'ACCEPT'

config forwarding
        option dest 'sharktun'
        option src 'vpn_privacy'

config zone
        option name 'untrusted'
        option input 'ACCEPT'
        option forward 'ACCEPT'
        option network 'untrusted'
        option output 'ACCEPT'
        list device 'eth1.100'

config rule
        option name 'Allow-vlan100-Ping'
        option src 'untrusted'
        option target 'ACCEPT'
        option proto 'icmp'

config rule
        option name 'vlan100-to-router'
        option src 'untrusted'
        option target 'ACCEPT'

config forwarding
        option dest 'vpsgw'
        option src 'untrusted'

config forwarding
        option dest 'sharktun'
        option src 'untrusted'

config zone
        option name 'vms'
        option device 'eth1.200'
        option input 'ACCEPT'
        option forward 'ACCEPT'
        option network 'vms'
        option output 'ACCEPT'

config rule
        option name 'Allow-vlan200-Ping'
        option src 'vms'
        option target 'ACCEPT'
        option proto 'icmp'

config rule
        option name 'vlan200-to-router'
        option src 'vms'
        option target 'ACCEPT'

config forwarding
        option dest 'lan'
        option src 'vms'

config forwarding
        option dest 'wan'
        option src 'vms'

config zone
        option name 'sharktun'
        option network 'sharktun'
        option forward 'REJECT'
        option masq '1'
        option output 'ACCEPT'
        option device 'tun0'
        option input 'REJECT'
        option mtu_fix '1'

config redirect
        option src 'vpsgw'
        option src_dport '32400'
        option dest_port '32400'
        option target 'DNAT'
        option proto 'tcp'
        option dest 'untrusted'
        option dest_ip '172.100.0.20'
        option name 'vpsgw-plex'

config rule
        option src_ip '10.100.100.0/24'
        option src 'vpsgw'
        option proto 'all'
        option target 'ACCEPT'
        option name 'vpsgw-internal-tunnel-allow'

config zone
        option device 'vpsgw'
        option masq6 '1'
        option forward 'REJECT'
        option masq '1'
        option output 'ACCEPT'
        option input 'REJECT'
        option network 'vpsgw'
        option name 'vpsgw'

config rule
        list proto 'all'
        option src 'sharktun'
        list src_mac '3x'
        option dest 'untrusted'
        option target 'ACCEPT'

config zone
        option name 'policy_route'
        option input 'ACCEPT'
        option output 'ACCEPT'
        option forward 'REJECT'
        list network 'pbr'

config forwarding
        option src 'vpn_privacy'
        option dest 'untrusted'

config forwarding
        option src 'untrusted'
        option dest 'vpn_privacy'

network and dhcp

root@meow:~# uci export network
package network

config interface 'loopback'
        option device 'lo'
        option proto 'static'
        option ipaddr '127.0.0.1'
        option netmask '255.0.0.0'

config globals 'globals'
        option ula_prefix 'fd87:bfc9:eb96::/48'

config device
        option name 'br-lan'
        option type 'bridge'
        list ports 'eth1'

config device
        option name 'eth1'
        option macaddr '6axa'

config interface 'lan'
        option device 'br-lan'
        option proto 'static'
        option netmask '255.255.255.0'
        option ip6assign '60'
        option ipaddr '192.168.100.1'

config device
        option name 'eth0'
        option macaddr '6x'

config interface 'wan'
        option device 'eth0'
        option proto 'dhcp'

config interface 'wan6'
        option device 'eth0'
        option proto 'dhcpv6'
        option reqaddress 'force'
        option reqprefix 'no'
        option defaultroute '1'

config interface 'docker'
        option device 'docker0'
        option proto 'none'
        option auto '0'

config device
        option type 'bridge'
        option name 'docker0'

config interface 'vpn_privacy'
        option device 'eth1.666'
        option netmask '255.255.255.0'
        option ipaddr '172.66.6.1'
        option proto 'static'

config interface 'untrusted'
        option device 'eth1.100'
        option netmask '255.255.255.0'
        option ipaddr '172.100.0.1'
        option ip6assign '64'
        option proto 'static'

config interface 'vms'
        option device 'eth1.200'
        option netmask '255.255.255.0'
        option ipaddr '172.200.0.1'
        option ip6assign '64'
        option proto 'static'

config rule 'vpn_privacy_routing4'
        option priority '30000'
        option lookup '30'
        option in 'vpn_privacy'

config rule 'untrusted_routing4'
        option priority '30000'
        option lookup '20'
        option in 'untrusted'

config rule6 'untrusted_routing6'
        option priority '30000'
        option lookup '20'
        option in 'untrusted'

config interface 'sharktun'
        option device 'tun0'
        option ip4table '30'
        option ip6table '30'
        option proto 'none'

config interface 'vpsgw'
x
        option proto 'wireguard'
        option peerdns '0'
        option mtu '1350'
        option ip4table '20'
        option ip6table '20'

config wireguard_vpsgw 'wgserver'
x
        list allowed_ips '0.0.0.0/0'
        list allowed_ips '::/0'
        option endpoint_host 'x'
        option persistent_keepalive '19'
        option endpoint_port '88'

config interface 'pbr'
        option proto 'static'
        option device 'eth1.500'
        option ipaddr '172.50.0.1'
        option netmask '255.255.255.0'

root@meow:~# uci export dhcp
package dhcp

config dnsmasq
        option domainneeded '1'
        option boguspriv '1'
        option filterwin2k '0'
        option localise_queries '1'
        option rebind_protection '1'
        option rebind_localhost '1'
        option expandhosts '1'
        option nonegcache '0'
        option authoritative '1'
        option readethers '1'
        option leasefile '/tmp/dhcp.leases'
        option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
        option nonwildcard '1'
        option localservice '1'
        option ednspacket_max '1232'
        option confdir '/tmp/dnsmasq.d'
        option domain 'gfm'
        option local '/gfm/'
        option noresolv '1'
        list server '127.0.0.1#5453'
        list server '0::1#5453'

config dhcp 'lan'
        option interface 'lan'
        option leasetime '12h'
        option dhcpv4 'server'
        option dhcpv6 'server'
        option ra 'server'
        option ra_slaac '1'
        list ra_flags 'managed-config'
        list ra_flags 'other-config'
        option ra_default '1'
        option start '20'
        option limit '50'
        option ra_management '1'

config dhcp 'wan'
        option interface 'wan'
        option ignore '1'

config odhcpd 'odhcpd'
        option maindhcp '0'
        option leasefile '/tmp/hosts/odhcpd'
        option leasetrigger '/usr/sbin/odhcpd-update'
        option loglevel '4'

config dhcp 'vpn_privacy'
        option interface 'vpn_privacy'
        option start '200'
        option limit '220'
        option leasetime '12h'

config dhcp 'untrusted'
        option leasetime '12h'
        option ra_default '1'
        option start '20'
        option dhcpv6 'server'
        option limit '50'
        option ra 'server'
        option interface 'untrusted'
        option ra_management '1'

config dhcp 'vms'
        option leasetime '12h'
        option ra_default '1'
        option start '20'
        option dhcpv6 'server'
        option limit '50'
        option ra 'server'
        option interface 'vms'
        option ra_management '1'

config host
        option mac '36x'
        option ip '172.100.0.20'
        option name 'plex'
        option dns '1'

config host
        option mac 'C6x6'
        option ip '172.66.0.99'
        option name 'test2'
        option dns '1'

config dhcp 'pbr'
        option interface 'pbr'
        option start '100'
        option leasetime '12h'
        option limit '120'
        option ra 'server'
        option ra_default '2'
        option dhcpv6 'server'
4

Assign each interface to a separate routing table as explained here:
https://openwrt.org/docs/guide-user/network/routing/pbr_netifd