Custom route fails to work without setting gateway

This is my network config:

config interface 'loopback'
	option device 'lo'
	option proto 'static'
	option ipaddr '127.0.0.1'
	option netmask '255.0.0.0'

config globals 'globals'
	option ula_prefix 'fdac:2bfe:d5cf::/48'

config device
	option name 'br-lan'
	option type 'bridge'
	list ports 'lan1'
	list ports 'lan2'
	list ports 'lan3'
	list ports 'sfp2'

config interface 'lan'
	option device 'br-lan'
	option proto 'static'
	option ipaddr '10.254.100.1/24'
	option netmask '255.255.255.0'
	option ip6assign '60'
	option dns '10.254.100.1'

config device
	option name 'br-wan'
	option type 'bridge'
	option macaddr 'CE:3B:48:7D:DB:62'
	list ports 'eth1'
	list ports 'wan'

config device
	option name 'eth1'
	option macaddr 'ce:3b:48:7d:db:62'

config device
	option name 'wan'
	option macaddr 'ce:3b:48:7d:db:62'

config interface 'wan'
	option device 'br-wan'
	option start '100'
	option limit '150'
	option leasetime '12h'
	option metric '10'
	option proto 'pppoe'
	option username '<username>'
	option password '<password>'
	option ipv6 '0'

config interface 'wan6'
	option device '@wan'
	option metric '10'
	option reqaddress 'try'
	option reqprefix 'auto'
	option sourcefilter '0'
	option proto 'dhcpv6'

config interface '4G'
	option proto 'qmi'
	option auth 'none'
	option pdptype 'ipv4v6'
	option device '/dev/cdc-wdm0'
	option metric '50'
	option delegate '0'

config interface 'BSNLTrunk'
	option device 'lan4'
	option proto 'dhcp'
	option metric '1000'
	option delegate '0'

config interface 'tailscale'
	option proto 'none'
	option device 'tailscale0'

This is my firewall config:

config defaults
	option input 'REJECT'
	option output 'ACCEPT'
	option forward 'REJECT'
	option synflood_protect '1'

config zone
	option name 'lan'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'
	list network 'lan'

config zone
	option name 'wan'
	option input 'REJECT'
	option output 'ACCEPT'
	option forward 'REJECT'
	option masq '1'
	option mtu_fix '1'
	list network 'wan'
	list network 'wan6'
	list network '4G'
	list network 'BSNLTrunk'

config forwarding
	option src 'lan'
	option dest 'wan'
	
config rule
	option name 'Allow-DHCP-Renew'
	option src 'wan'
	option proto 'udp'
	option dest_port '68'
	option target 'ACCEPT'
	option family 'ipv4'

config rule
	option name 'Allow-IGMP'
	option src 'wan'
	option proto 'igmp'
	option family 'ipv4'
	option target 'ACCEPT'

config rule
	option name 'Allow-DHCPv6'
	option src 'wan'
	option proto 'udp'
	option dest_port '546'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-MLD'
	option src 'wan'
	option proto 'icmp'
	option src_ip 'fe80::/10'
	list icmp_type '130/0'
	list icmp_type '131/0'
	list icmp_type '132/0'
	list icmp_type '143/0'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-ICMPv6-Input'
	option src 'wan'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	list icmp_type 'router-solicitation'
	list icmp_type 'neighbour-solicitation'
	list icmp_type 'router-advertisement'
	list icmp_type 'neighbour-advertisement'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-ICMPv6-Forward'
	option src 'wan'
	option dest '*'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'

config include 'miniupnpd'
	option type 'script'
	option path '/usr/share/miniupnpd/firewall.include'

config zone
	option name 'tailscale'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'
	option masq '1'
	list network 'tailscale'

config forwarding
	option src 'tailscale'
	option dest 'lan'

config forwarding
	option src 'tailscale'
	option dest 'wan'

config forwarding
	option src 'lan'
	option dest 'tailscale'

Before I set anything up, I can ping 172.30.249.11 and 172.30.249.33 by using the command ping <ip> -I lan4, what I want to achieve is all packets directed to these IPs (and some other IPs that I will add later) to automatically use lan4 (BSNLTrunk). I can see the following data in the overview page as well:

Protocol: DHCP client
Address: 10.148.113.22/19
Gateway: 10.148.96.1
DNS 1: 172.30.249.11
DNS 2: 172.30.249.33
Expires: 1h 15m 56s
Connected: 0h 14m 4s

Now this is my setup :

  1. I mark the packets from anywhere 0.0.0.0/0 to 172.30.249.0/24 using these nftables rules:

/usr/share/nftables.d/chain-pre/mangle_output/0-pbr.nft :

ip saddr { 0.0.0.0/0 } ip daddr { 172.30.249.0/24 } counter meta mark set 720801 comment "!fw4: 720801"

/usr/share/nftables.d/chain-pre/mangle_prerouting/0-pbr.nft :

iifname "br-lan" ip saddr { 0.0.0.0/0 } ip daddr { 172.30.249.0/24 } counter meta mark set 720801 comment "!fw4: 720801"
  1. I setup rules and routes to use the marked packets using these rules in the /etc/config/network file:
config route
	option table '47167'
	option interface 'BSNLTrunk'
	option target '0.0.0.0/0'

config rule
	option mark '720801'
	option lookup '47167'

Now I should ideally be able to ping 172.30.249.0/24 from anywhere in the network right? But that's not the case the pings seemingly go nowhere and the traceroute (from the router to the range) looks like this:

root@openwrt-box:/usr/share/nftables.d# traceroute 172.30.249.11
traceroute to 172.30.249.11 (172.30.249.11), 30 hops max, 46 byte packets
 1  10.148.113.22 (10.148.113.22)  423.972 ms !H  3118.365 ms !H  3120.005 ms !H

Also I should note that in the overview page the gateway has turned into 0.0.0.0?

Protocol: DHCP client
Address: 10.148.113.22/19
Gateway: 0.0.0.0
DNS 1: 172.30.249.11
DNS 2: 172.30.249.33
Expires: 1h 9m 38s
Connected: 0h 20m 22s

Now I should add that I use the same method to PBR using other interfaces (My 4G interface, some wireguard interfaces) and they all work perfectly fine except this one interface. Also when I add the gateway '10.148.96.1' to this route:

config route
	option table '47167'
	option interface 'BSNLTrunk'
	option target '0.0.0.0/0'
	option gateway '10.148.96.1'

It works as expected again, but I don't want to use the gateway as it's not reliable and prone to change? I want to use the interface directly. I have tried everything I could think of but I can't seem to get it to work. Any help would be appreciated. Thanks in advance.

Assign the interface to a separate routing table, then you can use it with custom routing rules.

Yes that works fine too, but I'm curious is there a reason why my current setup fails to work?

I'm on the latest stable branch as far as openwrt goes and since I'm not using the PBR package I'm not sure what you meant.

Create a routing rule for 172.30.249.0/24 and there's no need for traffic marking.
In addition, Tailscale also needs a workaround to bypass its built-in PBR:
https://openwrt.org/docs/guide-user/network/routing/pbr_netifd#route_lan_to_tailscale

I have a script that updates the marking files dynamically, there sometimes are multiple IPs (ranges from 1-100s) in this marking file and I would rather update that than constantly make changes to the network config directly (as these updates happen rather frequently and I have had issues where service network reload doesn't work quite as well as a restart) and the script handles this pbr method for all interfaces (except tailscale and WAN).

Regardless I've made a submission to this repo regarding the issue I'm having and is waiting to hear back (if it is indeed a bug)

Let's check your runtime configs:

ip -4 route show table all; ip -4 rule show
1 Like

I'm not 100% sure but....
If the network is not directly reachable/assigned on a link, a route has to have a gateway. Otherwise how should the kernel know how to reach that network? Because like you can have more then one router on a link and each one could be able to reach different networks...

2 Likes

Before making any of my changes:

root@openwrt-box:~# ip -4 route show table all; ip -4 rule show

10.254.248.0/24 via <my-pppoe-gateway> dev pppoe-wan table 220 proto static src 10.254.100.1 
10.254.254.0/26 via <my-pppoe-gateway> dev pppoe-wan table 220 proto static src 10.254.100.1 
default via <my-pppoe-gateway> dev pppoe-wan proto static metric 10 
default via 100.78.134.152 dev wwan0 proto static src 100.78.134.151 metric 50 
default via 10.148.96.1 dev lan4 proto static src 10.148.113.22 metric 1000 
10.148.96.0/19 dev lan4 proto static scope link metric 1000 
10.250.250.0/24 dev wg_server proto static scope link metric 2000 
10.254.100.0/24 dev br-lan proto kernel scope link src 10.254.100.1 
<my-pppoe-gateway> dev pppoe-wan proto kernel scope link src <my-public-ip> 
100.78.134.144/28 dev wwan0 proto static scope link metric 50 
172.16.1.0/24 dev wg_server proto static scope link metric 2000 
172.16.1.2 dev wg_server proto static scope link metric 2000 
172.16.1.3 dev wg_server proto static scope link metric 2000 
172.16.1.4 dev wg_server proto static scope link metric 2000 
172.16.1.7 dev wg_server proto static scope link metric 2000 
local 10.148.113.22 dev lan4 table local proto kernel scope host src 10.148.113.22 
broadcast 10.148.127.255 dev lan4 table local proto kernel scope link src 10.148.113.22 
local 10.254.100.1 dev br-lan table local proto kernel scope host src 10.254.100.1 
broadcast 10.254.100.255 dev br-lan table local proto kernel scope link src 10.254.100.1 
local <my-public-ip> dev pppoe-wan table local proto kernel scope host src <my-public-ip> 
local 100.78.134.151 dev wwan0 table local proto kernel scope host src 100.78.134.151 
broadcast 100.78.134.159 dev wwan0 table local proto kernel scope link src 100.78.134.151 
local 127.0.0.0/8 dev lo table local proto kernel scope host src 127.0.0.1 
local 127.0.0.1 dev lo table local proto kernel scope host src 127.0.0.1 
broadcast 127.255.255.255 dev lo table local proto kernel scope link src 127.0.0.1 
local 172.16.1.1 dev wg_server table local proto kernel scope host src 172.16.1.1 
broadcast 172.16.1.255 dev wg_server table local proto kernel scope link src 172.16.1.1 

0:	from all lookup local
220:	from all lookup 220
5210:	from all fwmark 0x80000/0xff0000 lookup main
5230:	from all fwmark 0x80000/0xff0000 lookup default
5250:	from all fwmark 0x80000/0xff0000 unreachable
5270:	from all lookup 52
32766:	from all lookup main
32767:	from all lookup default

After making my changes :

root@openwrt-box:~# ip -4 route show table all; ip -4 rule show
default dev lan4 table 6856 proto static scope link metric 1000 
10.254.248.0/24 via <my-pppoe-gateway> dev pppoe-wan table 220 proto static src 10.254.100.1 
10.254.254.0/26 via <my-pppoe-gateway> dev pppoe-wan table 220 proto static src 10.254.100.1 
default via <my-pppoe-gateway> dev pppoe-wan proto static metric 10 
default via 100.78.134.152 dev wwan0 proto static src 100.78.134.151 metric 50 
default via 10.148.96.1 dev lan4 proto static src 10.148.113.22 metric 1000 
10.148.96.0/19 dev lan4 proto static scope link metric 1000 
10.250.250.0/24 dev wg_server proto static scope link metric 2000 
10.254.100.0/24 dev br-lan proto kernel scope link src 10.254.100.1 
<my-pppoe-gateway> dev pppoe-wan proto kernel scope link src <my-public-ip> 
100.78.134.144/28 dev wwan0 proto static scope link metric 50 
172.16.1.0/24 dev wg_server proto static scope link metric 2000 
172.16.1.2 dev wg_server proto static scope link metric 2000 
172.16.1.3 dev wg_server proto static scope link metric 2000 
172.16.1.4 dev wg_server proto static scope link metric 2000 
172.16.1.7 dev wg_server proto static scope link metric 2000 
local 10.148.113.22 dev lan4 table local proto kernel scope host src 10.148.113.22 
broadcast 10.148.127.255 dev lan4 table local proto kernel scope link src 10.148.113.22 
local 10.254.100.1 dev br-lan table local proto kernel scope host src 10.254.100.1 
broadcast 10.254.100.255 dev br-lan table local proto kernel scope link src 10.254.100.1 
local <my-public-ip> dev pppoe-wan table local proto kernel scope host src <my-public-ip> 
local 100.78.134.151 dev wwan0 table local proto kernel scope host src 100.78.134.151 
broadcast 100.78.134.159 dev wwan0 table local proto kernel scope link src 100.78.134.151 
local 127.0.0.0/8 dev lo table local proto kernel scope host src 127.0.0.1 
local 127.0.0.1 dev lo table local proto kernel scope host src 127.0.0.1 
broadcast 127.255.255.255 dev lo table local proto kernel scope link src 127.0.0.1 
local 172.16.1.1 dev wg_server table local proto kernel scope host src 172.16.1.1 
broadcast 172.16.1.255 dev wg_server table local proto kernel scope link src 172.16.1.1 

0:	from all lookup local
1:	from all fwmark 0xaffa1 lookup 6856
220:	from all lookup 220
5210:	from all fwmark 0x80000/0xff0000 lookup main
5230:	from all fwmark 0x80000/0xff0000 lookup default
5250:	from all fwmark 0x80000/0xff0000 unreachable
5270:	from all lookup 52
32766:	from all lookup main
32767:	from all lookup default

My changes are identical to the ones I mentioned in the first post, just that the table number has been changed. Also another thing to note I mentioned the gateway turns into 0.0.0.0 here:

image

Going through the network interface dump UBUS call I can see the following routes (10.148.96.1 is the gateway):

This kind of route can only work for point-to-point interfaces like VPN, and if lan4 is not the one, you need a gateway in order to make it routable.
The most simple way is moving this interface to a separate table as mentioned earlier by using the options ip4table and ip6table and creating custom rules to reach that table.
Here are some examples implementing PBR like that:
https://openwrt.org/docs/guide-user/network/routing/pbr_netifd

2 Likes