OK, got it, check offload device list. Offload output is fifo, bypassing configured qdisc.
Huh? 
I'm a noob in this topic 
Expected:
table inet fw4 {
flowtable ft {
hook ingress priority filter
devices = { br-lan, br-wan }
counter
}
In command line list firewall ruleset and show 10 first lines:
nft list ruleset | head -10
root@OpenWrt:~# nft list ruleset | head -10
table inet fw4 {
flowtable ft {
hook ingress priority filter
devices = { lan1, lan2, lan3, lan4 }
counter
}
chain input {
type filter hook input priority filter; policy accept;
iifname "lo" accept comment "!fw4: Accept traffic from loopback"
This is with nft list ruleset | less
table inet fw4 {
flowtable ft {
hook ingress priority filter
devices = { lan1, lan2, lan3, lan4 }
counter
}
Update one component of firewall:
wget -O /usr/share/ucode/fw4.uc https://github.com/openwrt/firewall4/raw/master/root/usr/share/ucode/fw4.uc
Then run:
fw4 check
fw4 print | head -10 # here confirm that offload interface list is reduces
service firewall restart
# or if tests fail
# cat /rom/usr/share/ucode/fw4.uc > /usr/share/ucode/fw4.uc
root@OpenWrt:~# wget -O /usr/share/ucode/fw4.uc https://github.com/openwrt/firew
all4/raw/master/root/usr/share/ucode/fw4.uc
Downloading 'https://github.com/openwrt/firewall4/raw/master/root/usr/share/ucode/fw4.uc'
Connecting to 140.82.121.4:443
Redirected to /openwrt/firewall4/master/root/usr/share/ucode/fw4.uc on raw.githubusercontent.c om
Writing to '/usr/share/ucode/fw4.uc'
/usr/share/ucode/fw4 100% |*******************************| 79820 0:00:00 ETA
Download completed (79820 bytes)
root@OpenWrt:~# fw4 check
Automatically including '/usr/share/nftables.d/table-post/20-miniupnpd.nft'
Automatically including '/usr/share/nftables.d/chain-post/dstnat/20-miniupnpd.nft'
Automatically including '/usr/share/nftables.d/chain-post/forward/20-miniupnpd.nft'
Automatically including '/usr/share/nftables.d/chain-post/srcnat/20-miniupnpd.nft'
Ruleset passes nftables check.
root@OpenWrt:~# fw4 check
Automatically including '/usr/share/nftables.d/table-post/20-miniupnpd.nft'
Automatically including '/usr/share/nftables.d/chain-post/dstnat/20-miniupnpd.nft'
Automatically including '/usr/share/nftables.d/chain-post/forward/20-miniupnpd.nft'
Automatically including '/usr/share/nftables.d/chain-post/srcnat/20-miniupnpd.nft'
Ruleset passes nftables check.
root@OpenWrt:~# fw4 print | head -10 # here confirm that offload interface list is reduces
Automatically including '/usr/share/nftables.d/table-post/20-miniupnpd.nft'
Automatically including '/usr/share/nftables.d/chain-post/dstnat/20-miniupnpd.nft'
Automatically including '/usr/share/nftables.d/chain-post/forward/20-miniupnpd.nft'
Automatically including '/usr/share/nftables.d/chain-post/srcnat/20-miniupnpd.nft'
table inet fw4
flush table inet fw4
delete flowtable inet fw4 ft
table inet fw4 {
#
# Flowtable
#
flowtable ft {
root@OpenWrt:~# service firewall restart
Automatically including '/usr/share/nftables.d/table-post/20-miniupnpd.nft'
Automatically including '/usr/share/nftables.d/chain-post/dstnat/20-miniupnpd.nft'
Automatically including '/usr/share/nftables.d/chain-post/forward/20-miniupnpd.nft'
Automatically including '/usr/share/nftables.d/chain-post/srcnat/20-miniupnpd.nft'
I think i was really dumb. I set the Download speed (ingress) in SQM to 1Gigabit cause of the connection between client-router-modem(Outdoor Unit).
I've limited now to ~200Mbit/s and now Ping doesn't increase under load.
Is there a way to get more bandwith without increasing ping under load und reduce/improve unloaded ping?
Disable QoS, standard fq_codel will magically work if not sabotaged by offload.
I have "cake" enabled. not fq_codel
Disable SQM totally and check against default config if it is needed at all.
And now ->sqm disabled, and frewall offload restricted to 2 bridges - what latency you get?
I'll reboot the router quick to apply all changes correctly
1 Like
Now it's like before.
https://www.waveform.com/tools/bufferbloat?test-id=014a08f3-5e26-4455-841c-2654295c43f6
With SQM limited to 200Mbit/s i get best results in bufferbloat. Unloaded ping won't change (i think cause it's 5G GSM non-standalone).
Quick verification:
# should contain br-lan br-wan
nft list ruleset | less
# should be fq_codel or noqueue on adapters
tc -s qdisc
Should not make difference it is SA/NSA 5G
root@OpenWrt:~# tc -s qdisc
qdisc noqueue 0: dev lo root refcnt 2
Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
qdisc mq 0: dev eth0 root
Sent 3058254475 bytes 3101617 pkt (dropped 0, overlimits 0 requeues 36)
backlog 0b 0p requeues 36
qdisc fq_codel 0: dev eth0 parent :4 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 4Mb ecn drop_batch 64
Sent 740771057 bytes 739920 pkt (dropped 0, overlimits 0 requeues 12)
backlog 0b 0p requeues 12
maxpacket 20356 drop_overlimit 0 new_flow_count 5731 ecn_mark 0
new_flows_len 0 old_flows_len 0
qdisc fq_codel 0: dev eth0 parent :3 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 4Mb ecn drop_batch 64
Sent 973246782 bytes 967131 pkt (dropped 0, overlimits 0 requeues 7)
backlog 0b 0p requeues 7
maxpacket 17448 drop_overlimit 0 new_flow_count 7568 ecn_mark 0
new_flows_len 0 old_flows_len 0
qdisc fq_codel 0: dev eth0 parent :2 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 4Mb ecn drop_batch 64
Sent 661928414 bytes 691823 pkt (dropped 0, overlimits 0 requeues 10)
backlog 0b 0p requeues 10
maxpacket 21810 drop_overlimit 0 new_flow_count 3914 ecn_mark 0
new_flows_len 0 old_flows_len 0
qdisc fq_codel 0: dev eth0 parent :1 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 4Mb ecn drop_batch 64
Sent 682308222 bytes 702743 pkt (dropped 0, overlimits 0 requeues 7)
backlog 0b 0p requeues 7
maxpacket 26172 drop_overlimit 0 new_flow_count 4628 ecn_mark 0
new_flows_len 0 old_flows_len 0
qdisc noqueue 0: dev lan1 root refcnt 2
Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
qdisc noqueue 0: dev lan2 root refcnt 2
Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
qdisc noqueue 0: dev lan3 root refcnt 2
Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
qdisc noqueue 0: dev lan4 root refcnt 2
Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
qdisc noqueue 0: dev br-lan root refcnt 2
Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
qdisc tbf 1: dev br-wan root refcnt 2 rate 1Gbit burst 125000b lat 300ms
Sent 14973 bytes 105 pkt (dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
qdisc fq_codel 110: dev br-wan parent 1: limit 1001p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 4Mb ecn drop_batch 64
Sent 14973 bytes 105 pkt (dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
maxpacket 1438 drop_overlimit 0 new_flow_count 105 ecn_mark 0
new_flows_len 0 old_flows_len 0
qdisc ingress ffff: dev br-wan parent ffff:fff1 ----------------
Sent 39393 bytes 107 pkt (dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
qdisc tbf 1: dev ifb4br-wan root refcnt 2 rate 200Mbit burst 25000b lat 300ms
Sent 41091 bytes 107 pkt (dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
qdisc fq_codel 110: dev ifb4br-wan parent 1: limit 1001p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 4Mb ecn drop_batch 64
Sent 41091 bytes 107 pkt (dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
maxpacket 3708 drop_overlimit 0 new_flow_count 102 ecn_mark 0
new_flows_len 0 old_flows_len 0
You sqm is enabled.
Set ingress to zero to get rid of ingress shaper. Repeat test.
Same with egress.
table inet fw4 {
chain input {
type filter hook input priority filter; policy accept;
iifname "lo" accept comment "!fw4: Accept traffic from loopback"
ct state established,related accept comment "!fw4: Allow inbound established and related flows"
tcp flags syn / fin,syn,rst,ack jump syn_flood comment "!fw4: Rate limit TCP syn packets"
iifname "br-lan" jump input_lan comment "!fw4: Handle lan IPv4/IPv6 input traffic"
iifname "br-wan" jump input_wan comment "!fw4: Handle wan IPv4/IPv6 input traffic"
}
chain forward {
type filter hook forward priority filter; policy accept;
ct state established,related accept comment "!fw4: Allow forwarded established and related flows"
iifname "br-lan" jump forward_lan comment "!fw4: Handle lan IPv4/IPv6 forward traffic"
iifname "br-wan" jump forward_wan comment "!fw4: Handle wan IPv4/IPv6 forward traffic"
jump upnp_forward comment "Hook into miniupnpd forwarding chain"
}
chain output {
type filter hook output priority filter; policy accept;
oifname "lo" accept comment "!fw4: Accept traffic towards loopback"
ct state established,related accept comment "!fw4: Allow outbound established and related flows"
oifname "br-lan" jump output_lan comment "!fw4: Handle lan IPv4/IPv6 output traffic"
oifname "br-wan" jump output_wan comment "!fw4: Handle wan IPv4/IPv6 output traffic"
}
chain prerouting {
type filter hook prerouting priority filter; policy accept;
iifname "br-lan" jump helper_lan comment "!fw4: Handle lan IPv4/IPv6 helper assignment"
}
chain handle_reject {
meta l4proto tcp reject with tcp reset comment "!fw4: Reject TCP traffic"
reject comment "!fw4: Reject any other traffic"
}
chain syn_flood {
limit rate 25/second burst 50 packets return comment "!fw4: Accept SYN packets below rate-limit"
drop comment "!fw4: Drop excess packets"
}
chain input_lan {
jump accept_from_lan
}
chain output_lan {
jump accept_to_lan
}
chain forward_lan {
jump accept_to_wan comment "!fw4: Accept lan to wan forwarding"
jump accept_to_lan
}
chain helper_lan {
}
chain accept_from_lan {
iifname "br-lan" counter packets 393 bytes 110251 accept comment "!fw4: accept lan IPv4/IPv6 traffic"
}
chain accept_from_lan {
iifname "br-lan" counter packets 393 bytes 110251 accept comment "!fw4: accept lan IPv4/IPv6 traffic"
}
chain accept_to_lan {
oifname "br-lan" counter packets 884 bytes 410845 accept comment "!fw4: accept lan IPv4/IPv6 traffic"
}
chain input_wan {
meta nfproto ipv4 udp dport 68 counter packets 0 bytes 0 accept comment "!fw4: Allow-DHCP-Renew"
icmp type echo-request counter packets 2 bytes 122 accept comment "!fw4: Allow-Ping"
meta nfproto ipv4 meta l4proto igmp counter packets 7 bytes 224 accept comment "!fw4: Allow-IGMP"
meta nfproto ipv6 udp dport 546 counter packets 0 bytes 0 accept comment "!fw4: Allow-DHCPv6"
ip6 saddr fe80::/10 icmpv6 type . icmpv6 code { mld-listener-query . no-route, mld-listener-report . no-route, mld-listener-done . no-route, mld2-listener-report . no-route } counter packets 0 bytes 0 accept comment "!fw4
: Allow-MLD"
icmpv6 type { destination-unreachable, time-exceeded, echo-request, echo-reply, nd-router-solicit, nd-router-advert } limit rate 1000/second counter packets 0 bytes 0 accept comment "!fw4: Allow-ICMPv6-Input"
icmpv6 type . icmpv6 code { packet-too-big . no-route, parameter-problem . no-route, nd-neighbor-solicit . no-route, nd-neighbor-advert . no-route, parameter-problem . admin-prohibited } limit rate 1000/second counter pac
kets 0 bytes 0 accept comment "!fw4: Allow-ICMPv6-Input"
jump accept_from_wan
}
chain output_wan {
jump accept_to_wan
}
chain forward_wan {
icmpv6 type { destination-unreachable, time-exceeded, echo-request, echo-reply } limit rate 1000/second counter packets 0 bytes 0 accept comment "!fw4: Allow-ICMPv6-Forward"
icmpv6 type . icmpv6 code { packet-too-big . no-route, parameter-problem . no-route, parameter-problem . admin-prohibited } limit rate 1000/second counter packets 0 bytes 0 accept comment "!fw4: Allow-ICMPv6-Forward"
meta l4proto esp counter packets 0 bytes 0 jump accept_to_lan comment "!fw4: Allow-IPSec-ESP"
udp dport 500 counter packets 0 bytes 0 jump accept_to_lan comment "!fw4: Allow-ISAKMP"
jump accept_to_wan
}
chain accept_from_wan {
iifname "br-wan" counter packets 216 bytes 11488 accept comment "!fw4: accept wan IPv4/IPv6 traffic"
}
chain accept_to_wan {
meta nfproto ipv4 oifname "br-wan" ct state invalid counter packets 48 bytes 2412 drop comment "!fw4: Prevent NAT leakage"
oifname "br-wan" counter packets 2038 bytes 581599 accept comment "!fw4: accept wan IPv4/IPv6 traffic"
}
chain dstnat {
type nat hook prerouting priority dstnat; policy accept;
jump upnp_prerouting comment "Hook into miniupnpd prerouting chain"
}
chain srcnat {
type nat hook postrouting priority srcnat; policy accept;
oifname "br-wan" jump srcnat_wan comment "!fw4: Handle wan IPv4/IPv6 srcnat traffic"
jump upnp_postrouting comment "Hook into miniupnpd postrouting chain"
}
chain srcnat_wan {
meta nfproto ipv4 masquerade comment "!fw4: Masquerade IPv4 wan traffic"
}
chain raw_prerouting {
type filter hook prerouting priority raw; policy accept;
}
chain raw_output {
type filter hook output priority raw; policy accept;
}
chain mangle_prerouting {
type filter hook prerouting priority mangle; policy accept;
}
chain mangle_postrouting {
type filter hook postrouting priority mangle; policy accept;
}
chain mangle_input {
type filter hook input priority mangle; policy accept;
}
chain mangle_output {
type route hook output priority mangle; policy accept;
}
chain mangle_forward {
type filter hook forward priority mangle; policy accept;
iifname "br-wan" tcp flags syn tcp option maxseg size set rt mtu comment "!fw4: Zone wan IPv4/IPv6 ingress MTU fixing"
oifname "br-wan" tcp flags syn tcp option maxseg size set rt mtu comment "!fw4: Zone wan IPv4/IPv6 egress MTU fixing"
}
chain upnp_forward {
}
chain upnp_prerouting {
}
chain upnp_postrouting {
}
}