Strange issue with packet lengths and pbr

Hello!,

I'm creating a advanced network but I noticed something very strange, my suspicioun might be that it is MTU issue/clamping related.

so I got the following:

I use one wireguard server (wifivpn), which cascades to a other wireguard instance (wgclient), now I want to make wifivpn talk to interface ps5 for remote play, so I added a pbr ignore rule on the highest priority.

now when I take a look into my tcpdump I do see the packets comming in and out but the packet length is 0.

see:

root@MT6000:~# tcpdump -i wifivpn dst 10.56.2.2
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on wifivpn, link-type RAW (Raw IP), snapshot length 262144 bytes
21:39:03.870837 IP 10.39.95.2.48950 > 10.56.2.2.9295: Flags [S], seq 308802394, win 65535, options [mss 1344,sackOK,TS val 1303878561 ecr 0,nop,wscale 9], length 0
21:39:04.793178 IP 10.39.95.2.48950 > 10.56.2.2.9295: Flags [S], seq 308802394, win 65535, options [mss 1344,sackOK,TS val 1303879573 ecr 0,nop,wscale 9], length 0
21:39:06.811557 IP 10.39.95.2.48950 > 10.56.2.2.9295: Flags [S], seq 308802394, win 65535, options [mss 1344,sackOK,TS val 1303881589 ecr 0,nop,wscale 9], length 0
21:39:11.040234 IP 10.39.95.2.48950 > 10.56.2.2.9295: Flags [S], seq 308802394, win 65535, options [mss 1344,sackOK,TS val 1303885817 ecr 0,nop,wscale 9], length 0
21:39:19.234946 IP 10.39.95.2.48950 > 10.56.2.2.9295: Flags [S], seq 308802394, win 65535, options [mss 1344,sackOK,TS val 1303894009 ecr 0,nop,wscale 9], length 0
21:42:13.495554 IP 10.39.95.2.50482 > 10.56.2.2.9295: Flags [S], seq 83787783, win 65535, options [mss 1344,sackOK,TS val 1303919452 ecr 0,nop,wscale 9], length 0
21:42:14.511909 IP 10.39.95.2.50482 > 10.56.2.2.9295: Flags [S], seq 83787783, win 65535, options [mss 1344,sackOK,TS val 1303920469 ecr 0,nop,wscale 9], length 0
21:42:16.528937 IP 10.39.95.2.50482 > 10.56.2.2.9295: Flags [S], seq 83787783, win 65535, options [mss 1344,sackOK,TS val 1303922485 ecr 0,nop,wscale 9], length 0

when i disable PBR the packets go through, although screen remains black but sound goes through, i'd still think its MTU but it is possible it could be PBR.

here is my configuration:

network

config interface 'loopback'
	option device 'lo'
	option proto 'static'
	option ipaddr '127.0.0.1'
	option netmask '255.0.0.0'

config globals 'globals'
	option packet_steering '2'
	option steering_flows '128'

config device
	option name 'br-lan'
	option type 'bridge'
	list ports 'lan1'
	list ports 'lan2'
	list ports 'lan3'
	list ports 'lan4'
	list ports 'lan5'
	list ports 'vx0'

config interface 'lan'
	option device 'br-lan.169'
	option proto 'static'
	option ipaddr '10.234.53.1'
	option netmask '255.255.255.0'
	option delegate '0'

config interface 'wan'
	option proto 'pppoe'
	option device 'eth1.6'
	option username '<snip>'
	option password 'ppp'
	option ipv6 '0'
	option sourcefilter '0'
	option delegate '0'
	option classlessroute '0'
	option mtu '1500'

config bridge-vlan
	option device 'br-lan'
	option vlan '169'
	list ports 'lan1:u*'
	list ports 'lan2:u*'
	list ports 'lan3:u*'
	list ports 'lan4:u*'
	list ports 'lan5:u*'

config bridge-vlan
	option device 'br-lan'
	option vlan '49'
	list ports 'lan1:t'
	list ports 'lan2:t'
	list ports 'lan4:t'
	list ports 'lan5:t'

config bridge-vlan
	option device 'br-lan'
	option vlan '6'
	list ports 'lan3:t'

config bridge-vlan
	option device 'br-lan'
	option vlan '50'
	list ports 'lan1:t'
	list ports 'lan2:t'
	list ports 'lan4:t'
	list ports 'lan5:t'
	list ports 'vx0:t'

config bridge-vlan
	option device 'br-lan'
	option vlan '51'
	list ports 'lan1:t'
	list ports 'lan2:t'
	list ports 'lan4:t'
	list ports 'lan5:t'
	list ports 'vx0:t'

config bridge-vlan
	option device 'br-lan'
	option vlan '52'
	list ports 'lan1:t'
	list ports 'lan2:t'
	list ports 'lan4:t'
	list ports 'lan5:t'

config bridge-vlan
	option device 'br-lan'
	option vlan '53'
	list ports 'lan1:t'
	list ports 'lan2:t'
	list ports 'lan4:t'
	list ports 'lan5:t'

config bridge-vlan
	option device 'br-lan'
	option vlan '89'
	list ports 'lan1:t'
	list ports 'lan2:t'
	list ports 'lan4:t'
	list ports 'lan5:t'
	list ports 'vx0:t'

config bridge-vlan
	option device 'br-lan'
	option vlan '90'
	list ports 'lan1:t'
	list ports 'lan2:t'
	list ports 'lan4:t'
	list ports 'lan5:t'
	list ports 'vx0:t'

config bridge-vlan
	option device 'br-lan'
	option vlan '178'
	list ports 'lan1:t'
	list ports 'lan2:t'
	list ports 'lan4:t'
	list ports 'lan5:t'

config bridge-vlan
	option device 'br-lan'
	option vlan '179'
	list ports 'lan1:t'
	list ports 'lan2:t'
	list ports 'lan4:t'
	list ports 'lan5:t'

config interface 'aria'
	option proto 'static'
	option device 'br-lan.6'
	option ipaddr '192.168.99.1'
	option netmask '255.255.255.0'
	option defaultroute '0'
	option delegate '0'

config interface 'pcnet'
	option proto 'static'
	option device 'br-lan.49'
	option ipaddr '10.34.79.1'
	option netmask '255.255.255.0'
	option defaultroute '0'
	option delegate '0'

config interface 'wlan0'
	option proto 'static'
	option device 'br-lan.50'
	option ipaddr '10.234.80.1'
	option netmask '255.255.255.0'
	option defaultroute '0'
	option delegate '0'

config interface 'wlan1'
	option proto 'static'
	option device 'br-lan.51'
	option ipaddr '10.234.81.1'
	option netmask '255.255.255.0'
	option defaultroute '0'
	option delegate '0'

config interface 'iot'
	option proto 'static'
	option device 'br-lan.52'
	option ipaddr '10.33.77.1'
	option netmask '255.255.255.0'
	option defaultroute '0'
	option delegate '0'

config interface 'tvnet'
	option proto 'static'
	option device 'br-lan.53'
	option ipaddr '172.22.33.1'
	option netmask '255.255.255.0'
	option defaultroute '0'
	option delegate '0'

config interface 'ps5'
	option proto 'static'
	option device 'br-lan.89'
	option ipaddr '10.56.2.1'
	option netmask '255.255.255.0'
	option defaultroute '0'
	option delegate '0'

config interface 'ayaneo'
	option proto 'static'
	option device 'br-lan.90'
	option ipaddr '10.87.32.1'
	option netmask '255.255.255.0'
	option defaultroute '0'
	option delegate '0'

config interface 'aqaranet'
	option proto 'static'
	option device 'br-lan.178'
	option ipaddr '10.233.10.1'
	option netmask '255.255.255.0'
	option defaultroute '0'
	option delegate '0'

config interface 'hwnet'
	option proto 'static'
	option device 'br-lan.179'
	option ipaddr '10.182.32.1'
	option netmask '255.255.255.0'
	option defaultroute '0'
	option delegate '0'

config interface 'wifivpn'
	option proto 'wireguard'
	option private_key '<snip>'
	option listen_port '51820'
	list addresses '10.39.95.1/24'
	option defaultroute '0'
	option delegate '0'
	option multicast '1'
	option mtu '1420'

config wireguard_wifivpn
	option description 'poco-x6-pro'
	option public_key '<snip>'
	option private_key '<snip>'
	option preshared_key '<snip>'
	option endpoint_host '10.234.80.1'
	option endpoint_port '51820'
	option persistent_keepalive '25'
	list allowed_ips '10.39.95.2/32'

config wireguard_wifivpn
	option description 'ayaneo'
	option public_key '<snip>'
	option private_key '<snip>'
	option preshared_key '<snip>'
	list allowed_ips '10.39.95.3/32'
	option endpoint_host '10.234.80.1'
	option endpoint_port '51820'

config interface 'wgclient'
	option proto 'wireguard'
	option private_key '<snip>'
	list addresses '10.64.132.53/32'
	option defaultroute '0'
	option delegate '0'

config wireguard_wgclient
	option description 'Netherlands_nl-ams-wg-001'
	list allowed_ips '0.0.0.0/0'
	option endpoint_host '<snip>'
	option endpoint_port '3004'
	option persistent_keepalive '0'
	option public_key '<snip>'
	option disabled '1'

config bridge-vlan
	option device 'br-lan'
	option vlan '23'
	list ports 'lan1:t'
	list ports 'lan2:t'
	list ports 'lan4:t'
	list ports 'lan5:t'

config interface 'tvboxnet'
	option proto 'static'
	option device 'br-lan.23'
	option ipaddr '192.168.59.1'
	option netmask '255.255.255.0'
	option defaultroute '0'
	option delegate '0'

config interface 'vx0'
	option proto 'vxlan'
	option peeraddr '10.6.7.2'
	option ipaddr '10.6.7.1'
	option vid '4921'
	option tunlink 'wgserver'
	option rxcsum '0'
	option txcsum '0'
	option defaultroute '0'
	option delegate '0'
	option force_link '1'

config device
	option name 'vx0'

config interface 'wgserver'
	option proto 'wireguard'
	option private_key '<snip>'
	option listen_port '4443'
	list addresses '10.6.7.1/24'
	option delegate '0'
	option force_link '1'
	option mtu '1492'

config wireguard_wgserver
	option description 'MT3000'
	option public_key '<snip>'
	option endpoint_port '4443'
	list allowed_ips '10.6.7.2/32'
	option private_key '<snip>'
	option preshared_key '<snip>'
	option persistent_keepalive '25'
	option endpoint_host '<snip>'
pbr

config pbr 'config'
	option enabled '1'
	option nft_file_mode '1'
	option verbosity '2'
	option strict_enforcement '1'
	option resolver_set 'dnsmasq.nftset'
	list resolver_instance '*'
	option ipv6_enabled '0'
	option nft_file_support '0'
	option boot_timeout '30'
	option rule_create_option 'add'
	option procd_boot_delay '0'
	option procd_reload_delay '30'
	option webui_show_ignore_target '1'
	option nft_set_auto_merge '1'
	option nft_set_counter '1'
	option nft_set_flags_interval '1'
	option nft_set_flags_timeout '0'
	option nft_set_policy 'performance'
	list webui_supported_protocol 'all'
	list webui_supported_protocol 'tcp'
	list webui_supported_protocol 'udp'
	list webui_supported_protocol 'tcp udp'
	list webui_supported_protocol 'icmp'
	list supported_interface 'wgclient'
	list supported_interface 'wifivpn'
	list ignored_interface 'vx0'
	list ignored_interface 'br-lan.169'
	list ignored_interface 'br-lan.23'
	list ignored_interface 'br-lan.53'
	list ignored_interface 'br-lan.52'
	list ignored_interface 'br-lan.178'
	list ignored_interface 'br-lan.179'
	list ignored_interface 'wgserver'
	list ignored_interface 'br-lan.89'

config policy
	option name 'test'
	option interface 'ignore'
	option src_addr '10.39.95.0/24'
	option dest_addr '10.56.2.2/32'

config policy
	option name 'allow-maintenance'
	option src_addr '10.34.79.0/24 10.39.95.0/24'
	option dest_addr '10.234.53.3/32 10.234.53.10/32 10.234.53.15/32 10.234.53.20/32 10.234.53.25/32'
	option interface 'wan'

config policy
	option name 'bypass domain'
	option src_addr '10.34.79.0/24 10.39.95.0/24'
	option interface 'wan'
	option dest_addr 'whatismyip.com aqara.com reddit.com grc.com'

config policy
	option name 'cascade-vpn'
	option src_addr '10.6.7.0/24'
	option interface 'wgclient'

config policy
	option name 'route-vpn'
	option src_addr '10.34.79.0/24 10.39.95.0/24 192.168.99.0/24 10.87.32.0/24 10.56.2.0/24'
	option interface 'wgclient'
firewall

config defaults
	option input 'DROP'
	option output 'ACCEPT'
	option forward 'REJECT'
	option flow_offloading '1'
	option flow_offloading_hw '1'

config zone
	option name 'lan'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'
	list network 'lan'

config zone
	option name 'wan'
	option input 'DROP'
	option output 'ACCEPT'
	option forward 'REJECT'
	option masq '1'
	option mtu_fix '1'
	list network 'wan'

config forwarding
	option src 'lan'
	option dest 'wan'

config rule
	option name 'Allow-DHCP-Renew'
	option src 'wan'
	option proto 'udp'
	option dest_port '68'
	option target 'ACCEPT'
	option family 'ipv4'

config rule
	option name 'Allow-Ping'
	option src 'wan'
	option proto 'icmp'
	option icmp_type 'echo-request'
	option family 'ipv4'
	option target 'ACCEPT'
	option enabled '0'

config rule
	option name 'Allow-IGMP'
	option src 'wan'
	option proto 'igmp'
	option family 'ipv4'
	option target 'ACCEPT'
	option enabled '0'

config rule
	option name 'Allow-DHCPv6'
	option src 'wan'
	option proto 'udp'
	option dest_port '546'
	option family 'ipv6'
	option target 'ACCEPT'
	option enabled '0'

config rule
	option name 'Allow-MLD'
	option src 'wan'
	option proto 'icmp'
	option src_ip 'fe80::/10'
	list icmp_type '130/0'
	list icmp_type '131/0'
	list icmp_type '132/0'
	list icmp_type '143/0'
	option family 'ipv6'
	option target 'ACCEPT'
	option enabled '0'

config rule
	option name 'Allow-ICMPv6-Input'
	option src 'wan'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	list icmp_type 'router-solicitation'
	list icmp_type 'neighbour-solicitation'
	list icmp_type 'router-advertisement'
	list icmp_type 'neighbour-advertisement'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'
	option enabled '0'

config rule
	option name 'Allow-ICMPv6-Forward'
	option src 'wan'
	option dest '*'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'
	option enabled '0'

config rule
	option name 'Allow-IPSec-ESP'
	option src 'wan'
	option dest 'lan'
	option proto 'esp'
	option target 'ACCEPT'
	option enabled '0'

config rule
	option name 'Allow-ISAKMP'
	option src 'wan'
	option dest 'lan'
	option dest_port '500'
	option proto 'udp'
	option target 'ACCEPT'
	option enabled '0'

config zone
	option name 'aria'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'aria'

config zone
	option name 'pcnet'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'pcnet'

config zone
	option name 'wlan0'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'wlan0'

config zone
	option name 'wlan1'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'wlan1'

config zone
	option name 'iot'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'iot'

config zone
	option name 'tvnet'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'tvnet'

config zone
	option name 'ps5'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'
	list network 'ps5'

config zone
	option name 'aya'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'ayaneo'

config zone
	option name 'aqara'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'aqaranet'

config zone
	option name 'hwnet'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'hwnet'

config zone
	option name 'wgclient'
	option input 'DROP'
	option output 'ACCEPT'
	option forward 'REJECT'
	option masq '1'
	option mtu_fix '1'
	list network 'wgclient'

config forwarding
	option src 'aria'
	option dest 'wgclient'

config forwarding
	option src 'pcnet'
	option dest 'wgclient'

config forwarding
	option src 'iot'
	option dest 'wan'

config forwarding
	option src 'tvnet'
	option dest 'wan'

config forwarding
	option src 'ps5'
	option dest 'wgclient'

config forwarding
	option src 'aya'
	option dest 'wgclient'

config forwarding
	option src 'aqara'
	option dest 'wan'

config forwarding
	option src 'hwnet'
	option dest 'wan'

config zone
	option name 'tvboxnet'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'tvboxnet'

config forwarding
	option src 'tvboxnet'
	option dest 'wan'

config rule
	option name 'allow-vpn-bypass'
	option target 'ACCEPT'
	option mark '0x10000/0xff0000'
	option src '*'
	option dest 'wan'

config zone
	option name 'wgserver'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'vx0'
	list network 'wgserver'
	option masq '1'
	option mtu_fix '1'

config forwarding
	option src 'wgserver'
	option dest 'wgclient'

config rule
	option name 'allow-maintenance'
	option src 'pcnet'
	list dest_ip '10.234.53.10'
	list dest_ip '10.234.53.3'
	list dest_ip '10.234.53.15'
	list dest_ip '10.234.53.20'
	list dest_ip '10.234.53.25'
	option target 'ACCEPT'
	list proto 'all'

config rule
	option name 'wgserver-allow-vxlan'
	option src 'wgserver'
	option dest_port '4789'
	option target 'ACCEPT'

config redirect
	option target 'DNAT'
	list proto 'udp'
	option src 'wan'
	option src_dport '4443'
	option dest 'wgserver'
	option dest_ip '10.6.7.1'
	option name 'forward-wgserver'
	option reflection '0'
	option dest_port '4443'

config forwarding
	option src 'wgserver'
	option dest 'wan'

config redirect
	option dest 'wgserver'
	option target 'DNAT'
	option src 'pcnet'
	option src_dport '4443'
	option dest_port '4443'
	option dest_ip '10.6.7.1'
	list proto 'udp'

config include 'pbr'
	option fw4_compatible '1'
	option type 'script'
	option path '/usr/share/pbr/firewall.include'

config rule
	option src 'aya'
	option dest 'ps5'
	option dest_port '987 9295-9305'
	option target 'ACCEPT'
	option name 'allow-psn-remote-play'
	list dest_ip '10.56.2.2'

config rule
	option name 'allow-psn-remote-play'
	option src 'pcnet'
	option dest 'ps5'
	list dest_ip '10.56.2.2'
	option target 'ACCEPT'
	option dest_port '987 9295-9305'

config zone
	option name 'wifivpn'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	list network 'wifivpn'

config forwarding
	option src 'wifivpn'
	option dest 'wgclient'

config rule
	option src 'wifivpn'
	option dest 'ps5'
	option target 'ACCEPT'
	list proto 'tcp'
	list proto 'udp'
	list dest_ip '10.56.2.2'
	option dest_port '987 9295-9305'
	option name 'allow-psn-remote-play'
ubus board info
root@MT6000:~# ubus call system board
{
        "kernel": "6.6.35",
        "hostname": "MT6000",
        "system": "ARMv8 Processor rev 4",
        "model": "GL.iNet GL-MT6000",
        "board_name": "glinet,gl-mt6000",
        "rootfs_type": "squashfs",
        "release": {
                "distribution": "OpenWrt",
                "version": "SNAPSHOT",
                "revision": "r26615+289-4cfb14f11f",
                "target": "mediatek/filogic",
                "description": "OpenWrt SNAPSHOT r26615+289-4cfb14f11f"
        }
}

^ self compiled, only some cherry picked commit from dangowrt's branch , and 256 qam patches.

I still don't understand how PBR is not ignoring it, perhaps it is some type of bug or it changes how it behaves with mtu?

thank you :smiley:

I figured out the issue :smiley:

for some reason only with interface wifivpn I had to also add a reverse ignore rule from PS5 to wifivpn, still a little strange since I expected the ps5 to react on the same line like i've seen on other of my networks.

heres my new pbr config where test2 is added:

click to expand
config pbr 'config'
	option enabled '1'
	option nft_file_mode '1'
	option verbosity '2'
	option strict_enforcement '1'
	option resolver_set 'dnsmasq.nftset'
	list resolver_instance '*'
	option ipv6_enabled '0'
	option nft_file_support '0'
	option boot_timeout '30'
	option rule_create_option 'add'
	option procd_boot_delay '0'
	option procd_reload_delay '30'
	option webui_show_ignore_target '1'
	option nft_set_auto_merge '1'
	option nft_set_counter '1'
	option nft_set_flags_interval '1'
	option nft_set_flags_timeout '0'
	option nft_set_policy 'performance'
	list webui_supported_protocol 'all'
	list webui_supported_protocol 'tcp'
	list webui_supported_protocol 'udp'
	list webui_supported_protocol 'tcp udp'
	list webui_supported_protocol 'icmp'
	list supported_interface 'wgclient'
	list supported_interface 'wifivpn'
	list ignored_interface 'vx0'
	list ignored_interface 'br-lan.169'
	list ignored_interface 'br-lan.23'
	list ignored_interface 'br-lan.53'
	list ignored_interface 'br-lan.52'
	list ignored_interface 'br-lan.178'
	list ignored_interface 'br-lan.179'
	list ignored_interface 'wgserver'
	list ignored_interface 'br-lan.89'

config policy
	option name 'test'
	option interface 'ignore'
	option src_addr '10.39.95.0/24'
	option dest_addr '10.56.2.2/32'

config policy
	option name 'test2'
	option src_addr '10.56.2.2/32'
	option dest_addr '10.39.95.2/32'
	option interface 'ignore'

config policy
	option name 'allow-maintenance'
	option src_addr '10.34.79.0/24 10.39.95.0/24'
	option dest_addr '10.234.53.3/32 10.234.53.10/32 10.234.53.15/32 10.234.53.20/32 10.234.53.25/32'
	option interface 'wan'

config policy
	option name 'bypass domain'
	option src_addr '10.34.79.0/24 10.39.95.0/24'
	option interface 'wan'
	option dest_addr 'whatismyip.com aqara.com reddit.com grc.com'

config policy
	option name 'cascade-vpn'
	option src_addr '10.6.7.0/24'
	option interface 'wgclient'

config policy
	option name 'route-vpn'
	option src_addr '10.34.79.0/24 10.39.95.0/24 192.168.99.0/24 10.87.32.0/24 10.56.2.0/24'
	option interface 'wgclient'

still thanks for reading and time :+1::smiley:

This topic was automatically closed 10 days after the last reply. New replies are no longer allowed.