Improved layer_cake.qos and SQM settings for under 4ms latency increase under load

Hi everyone! i just wanted to share this.

I'm doing 4g over tethering on a RPI4.

i'm using mwan3 for load balancing between an iphone SE 2022(5g ) and s9 starlte connected through the usb cable to the rpi4 to increase total bandwith(no gb limit on both sims, the s9 is dedicated with lineageOS installed on it and 0 gapps, the iphone is my personal phone and i attach it when i'm home to increase total bandwith)

I can do A+ on waveform bufferbloat even while bittorrent is saturating the connection.

First file is custom_cake.qos that needs to be put in /usr/lib/sqm/

. ${SQM_LIB_DIR}/defaults.sh
QDISC=cake

EGRESS_CAKE_OPTS="${EGRESS_CAKE_OPTS} diffserv4 rtt 5ms "

INGRESS_CAKE_OPTS="$INGRESS_CAKE_OPTS diffserv4 rtt 5ms "

egress() {
    SILENT=1 $TC qdisc del dev $IFACE root
    $TC qdisc add dev $IFACE root $( get_stab_string ) cake \
        bandwidth ${UPLINK}kbit $( get_cake_lla_string ) ${EGRESS_CAKE_OPTS} ${EQDISC_OPTS}
}

ingress() {
    SILENT=1 $TC qdisc del dev $IFACE handle ffff: ingress
    $TC qdisc add dev $IFACE handle ffff: ingress

    SILENT=1 $TC qdisc del dev $DEV root

    [ "$IGNORE_DSCP_INGRESS" -eq "1" ] && INGRESS_CAKE_OPTS="$INGRESS_CAKE_OPTS besteffort"
    [ "$ZERO_DSCP_INGRESS" -eq "1" ] && INGRESS_CAKE_OPTS="$INGRESS_CAKE_OPTS wash"

    # Ensure that DSCP markings are honored
    #INGRESS_CAKE_OPTS="$INGRESS_CAKE_OPTS diffserv4"

    # Prioritize different classes based on DSCP markings
    case "$DSCP" in
        "CS7") INGRESS_CAKE_OPTS="$INGRESS_CAKE_OPTS priority 1" ;;
        "CS6") INGRESS_CAKE_OPTS="$INGRESS_CAKE_OPTS priority 2" ;;
        "CS4") INGRESS_CAKE_OPTS="$INGRESS_CAKE_OPTS priority 3" ;;
        "CS5") INGRESS_CAKE_OPTS="$INGRESS_CAKE_OPTS priority 4" ;;
        "CS1") INGRESS_CAKE_OPTS="$INGRESS_CAKE_OPTS priority 5" ;;
    esac


    $TC qdisc add dev $DEV root $( get_stab_string ) cake \
        bandwidth ${DOWNLINK}kbit $( get_cake_lla_string ) ${INGRESS_CAKE_OPTS} ${IQDISC_OPTS}
    $IP link set dev $DEV up

    # Redirect all IP packets arriving in $IFACE to ifb0
    $TC filter add dev $IFACE parent ffff: protocol all prio 10 u32 \
        match u32 0 0 flowid 1:1 action mirred egress redirect dev $DEV
}

sqm_prepare_script() {
    do_modules
    verify_qdisc $QDISC "cake" || return 1
}

Corresponding example firewall rules:
Add these lines into /etc/config/firewall

config rule
	option name 'DNS and ICMP'
	list proto 'tcp'
	list proto 'udp'
	option set_dscp 'CS7'
	option counter '0'
	option dest_port '53 853 5353'
	option target 'DSCP'
	option src '*'
	option dest '*'

config rule
	option name 'League of Legends'
	list proto 'udp'
	option set_dscp 'CS6'
	option counter '0'
	option dest_port '5000-5500 8393-8400'
	option target 'DSCP'
	option src '*'

config rule
    option name 'HTTPS'
    list proto 'tcp'
    list proto 'udp'
    option set_dscp 'CS5'
    option counter '0'
    option dest_port '443'
    option target 'DSCP'
    option src '*'

config rule
    option name 'BitTorrent'
    list proto 'tcp'
    list proto 'udp'
    option set_dscp 'CS1'
    option counter '0'
    option dest_port '6881-6889 6969 1337 6960-6969 6881'
    option target 'DSCP'
    option src '*'
config rule
    option name 'Other-Traffic'
    list proto 'tcp'
    list proto 'udp'
    option set_dscp 'CS3'  # Adjust DSCP as needed
    option counter '0'
    option dest_port '0-65535'  # Adjust port range as needed
    option target 'DSCP'
    option src '*'
    option dest '*'

SQM config file:

This one is directly into /etc/config/sqm. adjust the interface names and bandwith limits as you see fit.

In my case i have 2 WANs with 50% load balancing throgh mwan3.


config queue
	option interface 'eth1'
	option enabled '1'
	option download '70000'
	option upload '10000'
	option script 'custom_cake.qos'
	option qdisc_advanced '1'
	option linklayer 'ethernet'
	option overhead '44'
	option linklayer_advanced '1'
	option wash '0'
	option qdisc 'cake'
	option ehard 'default'
	option quantum '3000'
	option qdisc_really_really_advanced '1'
	option flows '1024'
	option flowhash '1'
	option mpu '64'
	option target 'default'
	option interval '50'
	option burst '1600'
	option linklayer_adaptation_mechanism 'cake'
	option flent '0'
	option ingress_ecn 'ECN'
	option egress_ecn 'ECN'
	option ecn 'ECN'
	option dequeue_ecn 'ECN'
	option debug_logging '0'
	option verbosity '5'
	option squash_dscp '0'
	option squash_ingress '0'
	option tcMTU '1484'
	option tcTSIZE '128'
	option tcMPU '64'
	option itarget '10'
	option etarget '10'

config queue
	option interface 'usb0'
	option enabled '1'
	option download '70000'
	option upload '10000'
	option script 'custom_cake.qos'
	option qdisc_advanced '1'
	option linklayer 'ethernet'
	option overhead '44'
	option linklayer_advanced '1'
	option wash '0'
	option qdisc 'cake'
	option ehard 'default'
	option quantum '3000'
	option qdisc_really_really_advanced '1'
	option flows '1024'
	option flowhash '1'
	option mpu '64'
	option target 'default'
	option interval '50'
	option burst '1600'
	option linklayer_adaptation_mechanism 'cake'
	option flent '0'
	option ingress_ecn 'ECN'
	option egress_ecn 'ECN'
	option ecn 'ECN'
	option dequeue_ecn 'ECN'
	option debug_logging '0'
	option verbosity '5'
	option squash_dscp '0'
	option squash_ingress '0'
	option tcMTU '1484'
	option tcTSIZE '128'
	option tcMPU '64'
	option itarget '10'
	option etarget '10'

4g tether is a refurbished s9. Best 4g modem ever by performance to price.
i honestly gonna try increase the donwload bandwith a bit and see if i can still keep latency under control.

Feel free to add this to the project if anyone wants to

x750 honestly is so bad. switched over to this solution.

2 Likes

Very nice.
I use T-Mobile 5G Home Internet.
I use the default SQM QOS layer cake settings via luci.
I have gotten A ratings via waveform bufferbloat site, but usually mine stays in the C rating or worse lately.

Being a noob, how would I use your specific settings?
I did change some of mine to yours, the one's I understand, as I use the luci GUI interface.

these are ready to be put inside the configuration files.

i'll edit the post to make this clearer

1 Like

Directories of where the files are would be good to add.
I found my sqm configuration in /etc/config/sqm

If I edit these outside of luci, using vi, how would this affect the luci menus?

these are designed so that all of these settings will show up in the LUCI gui

oha, that is going to hurt for longer RTT traffic... (the default is 100, ad this should be in the same order of magnitude as your normally encountered RTTs, if this is small longer RTT flows will get less throughput than expected), it will however make short RTT traffic snappier. If you know the trade-off and are happy with it, go for it.

These are as far as I can tell not part of variables SQM scripts actually evaluates from the config file...
Now you might have changed/modified all of sqm-scripts machinery, but in that case maybe post links to these modified versions as well?

Cake will ignore tcMTU and tcTSIZE, only tcMPU will be honored, that said, not sure what you want to achieve with tcMTU '1484', for tc-stab tcMTU is not a precise MTU but the size over which to create the size table... (tc-stab is quite complicated to allow accounting for ATM cell effects via a table and hence cake does not use it, but simply calculates the size for every packet, on most CPUs that is fast enough).

tcMTU was calculated as 1440 mtu real+ overhead that is about 44.

the options you mentioned were simply things present on the config file

yes, the bandwith was cut by quite a bit i admit, but the upside is that i get a stupidly stable line where even when i have someone else in the house using multiple streaming service and bittorrent at the same time i have 0 latency increase while playing league of legends. my line has a baseline median ping of about 34ms

Yeah, but that is not the correct way to do that for tc's stab (size table) option, lucky for you this will only be effective if you:
a) use HTB as traffic shaper
b) you use ATM encapsulation

However this is still incorrect and I would recommend to revert to the default values, UNLESS you understand what this actually does and think this is the right thing to do.

If you are fine with that throughput sacrifice and rarely use long RTT connections, go for it. Just keep in mind which value to change if that essential download from the other side of the world only ever crawls along at glacial speed :wink:

That means you even over-throttle near by downloads... theoretically you could set rtt to ~30 then to at least make local downloads fly, but your network, your rules, if you are happy with rtt 5 no need to change.

so i went ahead and built this.

RTT is still 5ms

increasing donwload bandwith limits to 120mbit had the magic effect of not cutting bandwith in half but it's eating about 30mbit right now.

so what i think it should be is to use a combination of over-committing on the bandwith side and then be more restrictive on the rtt setting

Sadly rtt value i discovered that it's something that it's fundamental to keep latency increase under downloads.

Either cake it's not doing it's job properly or something else is going on, because any rtt value near the real ones introduces latency in the order of 25-30ms.

i'll keep experimenting

this script is structured as a service to be placed into init.d but it still takes the /etc/config/sqm file as input.

#!/bin/sh /etc/rc.common

START=99
STOP=01
USE_PROCD=1
CONFIG="/etc/config/sqm"

log() {
    logger -t sqm-scripts "$1"
}

# Function to parse SQM configuration and add debug logging
parse_sqm() {
    local iface_config="$1"
    local var="$2"
    if [ -f "$CONFIG" ]; then
        local value=$(awk -v iface="$iface_config" -v var="$var" '$0 ~ iface {p=1} p && $0 ~ var {print $3; exit}' "$CONFIG")
        # Removing the leading and trailing quotes
        local parsed_value=$(echo $value | sed "s/'//g")
        log "Parsed $var for $iface_config: $parsed_value"
        echo "$parsed_value"
    else
        log "Configuration file $CONFIG not found."
        return 1
    fi
}

# Function to apply QoS for a given interface
apply_qos_for_interface() {
    local iface="$1"
    local enabled
    enabled=$(parse_sqm "$iface" "option enabled") || return 1

    if [ "$enabled" = "1" ]; then
        local download_bw upload_bw overhead mpu itarget ifb_device
        download_bw=$(parse_sqm "$iface" "option download")kbit || return 1
        upload_bw=$(parse_sqm "$iface" "option upload")kbit || return 1
        overhead=$(parse_sqm "$iface" "option overhead") || return 1
        mpu=$(parse_sqm "$iface" "option mpu") || return 1
        itarget=$(parse_sqm "$iface" "option itarget")ms || return 1

        ifb_device="ifb_${iface}"

        tc qdisc del dev "$iface" root 2>/dev/null
        tc qdisc del dev "$iface" ingress 2>/dev/null
        tc qdisc del dev "$ifb_device" root 2>/dev/null
        ip link show "$ifb_device" > /dev/null 2>&1 || ip link add name "$ifb_device" type ifb
        ip link set dev "$ifb_device" up

        tc qdisc add dev "$iface" handle ffff: ingress
        tc filter add dev "$iface" parent ffff: protocol ip u32 match u32 0 0 action mirred egress redirect dev "$ifb_device"
        tc qdisc add dev "$ifb_device" root cake bandwidth "$download_bw" rtt "$itarget" diffserv8 flows nat ack-filter-aggressive mpu "$mpu" overhead "$overhead" 
        tc qdisc add dev "$iface" root cake bandwidth "$upload_bw" rtt "$itarget" diffserv8 flows nat ack-filter-aggressive mpu "$mpu" overhead "$overhead"



        log "Traffic shaping configured for $iface."
    else
        log "SQM is not enabled for $iface. Skipping configuration."
    fi
}

# Function to start the service
start_service() {
    if [ -f "$CONFIG" ]; then
        local interfaces
        interfaces=$(awk '/config queue/{getline; print $3}' "$CONFIG" | tr -d "'")

        for iface in $interfaces; do
            apply_qos_for_interface "$iface" || log "Failed to apply QoS for interface $iface"
        done

        log "All interfaces configured."
    else
        log "Configuration file $CONFIG not found. Service not started."
    fi
}

# Function to stop the service
stop_service() {
    if [ -f "$CONFIG" ]; then
        local interfaces
        interfaces=$(awk '/config queue/{getline; print $3}' "$CONFIG" | tr -d "'")

        for iface in $interfaces; do
            local ifb_device
            ifb_device="ifb_${iface}"

            tc qdisc del dev "$iface" root 2>/dev/null
            tc qdisc del dev "$iface" ingress 2>/dev/null
            tc qdisc del dev "$ifb_device" root 2>/dev/null
            ip link set dev "$ifb_device" down
            ip link delete "$ifb_device"

            log "Traffic shaping stopped for $iface."
        done

        log "All interfaces have been cleared."
    else
        log "Configuration file $CONFIG not found. Service not stopped."
    fi
}

no sqm:

with sqm:

SQM config file

config queue
	option interface 'usb0'
	option enabled '1'
	option download '165000'
	option upload '10000'
	option script 'custom_cake.qos'
	option qdisc_advanced '1'
	option linklayer 'ethernet'
	option overhead '44'
	option linklayer_advanced '1'
	option wash '0'
	option qdisc 'cake'
	option ehard 'default'
	option quantum '3000'
	option qdisc_really_really_advanced '1'
	option flows '1024'
	option flowhash '1'
	option mpu '64'
	option target 'default'
	option interval '50'
	option burst '1600'
	option linklayer_adaptation_mechanism 'cake'
	option flent '0'
	option ingress_ecn 'ECN'
	option egress_ecn 'ECN'
	option ecn 'ECN'
	option dequeue_ecn 'ECN'
	option debug_logging '0'
	option verbosity '5'
	option squash_dscp '0'
	option squash_ingress '0'
	option tcMTU '1484'
	option tcTSIZE '128'
	option tcMPU '64'
	option itarget '5'
	option etarget '5'

oversizing the donwload bandwith did the trick.

DSCP Marking rules have changed.

config rule
	option name 'DNS and ICMP'
	list proto 'tcp'
	list proto 'udp'
	option set_dscp 'AF43'
	option target 'DSCP'
	option src_port '53 853 5353'
	option src '*'

config rule
	option name 'Egress-DNS-ICMP'
	list proto 'tcp'
	list proto 'udp'
	option set_dscp 'AF43'
	option target 'DSCP'
	option dest_port '53 853 5353'
	option src 'wan'
	option dest 'lan'

config rule
	option name 'Egress-League-of-Legends'
	list proto 'udp'
	option set_dscp 'AF41'
	option target 'DSCP'
	option dest_port '5000-5500 8393-8400'
	option src 'wan'
	option dest 'lan'

config rule
	option name 'Egress-CSGO'
	list proto 'udp'
	option set_dscp 'AF41'
	option target 'DSCP'
	option dest_port '27015-27030'
	option src 'wan'
	option dest 'lan'

config rule
	option name 'League of Legends'
	list proto 'udp'
	option set_dscp 'AF41'
	option target 'DSCP'
	option src_port '5000-5500 8393-8400'
	option src 'wan'
	option dest 'lan'

config rule
	option name 'CS:GO'
	list proto 'udp'
	option set_dscp 'AF41'
	option target 'DSCP'
	option src_port '27015-27030'
	option src 'wan'
	option dest 'lan'

config rule
	option name 'Egress-HTTPS'
	list proto 'tcp'
	list proto 'udp'
	option set_dscp 'AF21'
	option target 'DSCP'
	option dest_port '443'
	option src 'wan'
	option dest 'lan'

config rule
	option name 'HTTPS'
	list proto 'tcp'
	list proto 'udp'
	option set_dscp 'AF21'
	option target 'DSCP'
	option src_port '443'
	option src 'wan'
	option dest 'lan'

config rule
	option name 'Egress-BitTorrent'
	list proto 'tcp'
	list proto 'udp'
	option set_dscp 'AF11'
	option target 'DSCP'
	option dest_port '6881-6889 6969 1337 6960-6969 6881'
	option src 'wan'
	option dest 'lan'

config rule
	option name 'BitTorrent'
	list proto 'tcp'
	list proto 'udp'
	option set_dscp 'AF11'
	option target 'DSCP'
	option src_port '6881-6889 6969 1337 6960-6969 6881'
	option src 'wan'
	option dest 'lan'

config rule
	option name 'Default-Low-Priority'
	list proto 'all'
	option set_dscp 'BE'
	option target 'DSCP'
	option src '*'
	option dest '*'