root@OpenWrt:/etc/init.d# tc -s class show dev ifb
class cake 8014:ed parent 8014:
(dropped 6, overlimits 0 requeues 0)
backlog 9940b 7p requeues 0
deficit -1416 count 1 dropping drop_next -23.8ms blue_prob 0
class cake 8014:fa parent 8014:
(dropped 4, overlimits 0 requeues 0)
backlog 15620b 11p requeues 0
deficit -367 count 1 dropping drop_next -23.2ms blue_prob 0
class cake 8014:133 parent 8014:
(dropped 3, overlimits 0 requeues 0)
backlog 5680b 4p requeues 0
deficit -1377 count 2 dropping drop_next -62.5ms blue_prob 0
class cake 8014:244 parent 8014:
(dropped 5, overlimits 0 requeues 0)
backlog 9940b 7p requeues 0
deficit -193 count 1 dropping drop_next -92.2ms blue_prob 0
class cake 8014:371 parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 115 count 0 blue_prob 0
root@OpenWrt:/etc/init.d# tc -s class show dev ifb
class cake 8014:ed parent 8014:
(dropped 6, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 270 count 1 blue_prob 0
class cake 8014:fa parent 8014:
(dropped 6, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 28 count 1 blue_prob 0
class cake 8014:133 parent 8014:
(dropped 4, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit -736 count 0 blue_prob 0
class cake 8014:244 parent 8014:
(dropped 5, overlimits 0 requeues 0)
backlog 7100b 5p requeues 0
deficit -1297 count 1 dropping drop_next -94.5ms blue_prob 0
root@OpenWrt:/etc/init.d# tc -s class show dev ifb
class cake 8014:b parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 808 count 0 blue_prob 0
class cake 8014:ed parent 8014:
(dropped 6, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit -944 count 1 dropping drop_next -96.5ms blue_prob 0
class cake 8014:fa parent 8014:
(dropped 6, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 162 count 1 blue_prob 0
class cake 8014:133 parent 8014:
(dropped 4, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 350 count 1 blue_prob 0
class cake 8014:244 parent 8014:
(dropped 5, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 208 count 1 blue_prob 0
root@OpenWrt:/etc/init.d# tc -s class show dev ifb
class cake 8014:133 parent 8014:
(dropped 4, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit -1367 count 0 blue_prob 0
class cake 8014:16b parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 814 count 0 blue_prob 0
root@OpenWrt:/etc/init.d# tc -s class show dev ifb
class cake 8014:133 parent 8014:
(dropped 4, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit -882 count 0 blue_prob 0
class cake 8014:16b parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 814 count 0 blue_prob 0
class cake 8014:2a6 parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 782 count 0 blue_prob 0
root@OpenWrt:/etc/init.d# tc -s class show dev ifb
class cake 8014:5b parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 814 count 0 blue_prob 0
class cake 8014:e8 parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 427 count 0 blue_prob 0
class cake 8014:151 parent 8014:
(dropped 13, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 814 count 0 blue_prob 0
class cake 8014:1d3 parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 802 count 0 blue_prob 0
class cake 8014:30c parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 750 count 0 blue_prob 0
root@OpenWrt:/etc/init.d# tc -s class show dev ifb
class cake 8014:5b parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 814 count 0 blue_prob 0
class cake 8014:e8 parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 28 count 0 blue_prob 0
class cake 8014:151 parent 8014:
(dropped 13, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 814 count 0 blue_prob 0
class cake 8014:1d3 parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 814 count 0 blue_prob 0
class cake 8014:24f parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 802 count 0 blue_prob 0
class cake 8014:30c parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 750 count 0 blue_prob 0
root@OpenWrt:/etc/init.d# tc -s class show dev ifb
class cake 8014:5b parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 814 count 0 blue_prob 0
class cake 8014:e8 parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit -52 count 0 blue_prob 0
class cake 8014:151 parent 8014:
(dropped 13, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 814 count 0 blue_prob 0
class cake 8014:1d3 parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 814 count 0 blue_prob 0
class cake 8014:30c parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 750 count 0 blue_prob 0
root@OpenWrt:/etc/init.d# tc -s class show dev ifb
class cake 8014:5b parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 814 count 0 blue_prob 0
class cake 8014:e8 parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 424 count 0 blue_prob 0
class cake 8014:151 parent 8014:
(dropped 13, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 714 count 0 blue_prob 0
class cake 8014:1d3 parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 814 count 0 blue_prob 0
class cake 8014:30c parent 8014:
(dropped 0, overlimits 0 requeues 0)
backlog 0b 0p requeues 0
deficit 750 count 0 blue_prob 0
If you use your Asus router as an AP with USB & Samba, and disable wireless on this OpenWrt router, you could put cake on br-lan (or better the real physical interface) and let the Asus AP worry about the wired and wireless bridging.
Then all your download traffic will be unencrypted and de-NATted on br-lan when CAKE applies download fairness.
Well, you need to still test it 
Say run 3 parallel speedtests, one bypassing the VPN (NoVPN) and two (VPN1, VP@)via the VPN. If flow fairness is operational (and all tests use the same number of flows) you would expect each individual speetest to give more or less the same number (NoVPN = VPN1 = VPN2), if VPN is treated as one opaque flow, you can expect (NoVPN = VPN1+VPN2). No running speedtests in parallel is a bit tricky, but if the tests run for long enough that might be close enough.
As far as I can tell this looks okay. But again, being simple minded, I probably would have switched to a wired-only primary router with SQM on the LAN interface, but then at the same I am impressed with your approach of making the more complicated approach work.
1 Like
Thanks @moeller0 will test.
@dlakelan or @moeller0 concerning @dlakelan's strategy:
For download, it seems you could hack things to have wireguard (outer) packets not tc mirred over to the IFB, but then set up an ingress filter on the wireguard interface itself and mirred the inner packets from there?
Thinking about it more, and mindful of @dave14305's observation above, on a theoretical level, can CAKE still properly work with the individual flows given the encryption? As in, is source X and destination Y always consistently encrypted to source A and destination B? Or does that vary? I presume for CAKE to work it doesn't really need to know the true source or destination, but they need to be consistent across packets for it to work correctly? Is there some other consideration here I am missing that is relevant and that might break this approach?
If not, does this approach need some modification still? Should I be using the 'nat' keyword?
tcpdump -i ifb -n |grep -i " > "
13:23:27.663813 40:00:3c:11:f2:27 > 45:00:00:36:00:00, ethertype Unknown (0x8efa), length 54:
13:23:27.665825 40:00:25:11:27:a4 > 45:00:02:e2:39:7b, ethertype Unknown (0x86e0), length 738:
13:23:27.666034 40:00:25:11:27:88 > 45:00:02:fd:39:7c, ethertype Unknown (0x86e0), length 765:
13:23:27.667826 40:00:26:11:28:5c > 45:00:01:2b:39:7a, ethertype Unknown (0x86e0), length 299:
13:23:27.667918 40:00:26:11:28:54 > 45:00:01:30:39:7d, ethertype Unknown (0x86e0), length 304:
13:23:27.673799 40:00:26:11:29:24 > 45:00:00:5c:39:81, ethertype Unknown (0x86e0), length 92:
13:23:27.676850 40:00:25:11:27:9c > 45:00:02:e2:39:83, ethertype Unknown (0x86e0), length 738:
13:23:27.677059 40:00:25:11:28:68 > 45:00:02:15:39:84, ethertype Unknown (0x86e0), length 533:
13:23:27.677212 40:00:25:11:28:67 > 45:00:02:15:39:85, ethertype Unknown (0x86e0), length 533:
13:23:27.677369 40:00:25:11:26:a8 > 45:00:03:d3:39:86, ethertype Unknown (0x86e0), length 979:
13:23:27.685890 40:00:25:11:27:76 > 45:00:02:fd:39:8e, ethertype Unknown (0x86e0), length 765:
13:23:27.686850 40:00:26:11:27:e4 > 45:00:01:90:39:8d, ethertype Unknown (0x86e0), length 400:
13:23:27.686968 40:00:26:11:28:c7 > 45:00:00:ab:39:8f, ethertype Unknown (0x86e0), length 171:
13:23:27.696906 40:00:25:11:26:9a > 45:00:03:d2:39:95, ethertype Unknown (0x86e0), length 978:
13:23:27.697176 40:00:25:11:28:57 > 45:00:02:14:39:96, ethertype Unknown (0x86e0), length 532:
13:23:27.697330 40:00:25:11:28:56 > 45:00:02:14:39:97, ethertype Unknown (0x86e0), length 532:
13:23:27.697488 40:00:25:11:28:35 > 45:00:02:34:39:98, ethertype Unknown (0x86e0), length 564:
13:23:27.698875 40:00:3c:11:f1:f5 > 45:00:00:68:00:00, ethertype Unknown (0x8efa), length 104:
13:23:27.698950 40:00:3c:11:f2:28 > 45:00:00:35:00:00, ethertype Unknown (0x8efa), length 53:
13:23:27.698955 40:00:3c:11:f2:28 > 45:00:00:35:00:00, ethertype Unknown (0x8efa), length 53:
13:23:27.701855 40:00:25:11:29:cf > 45:00:00:97:39:9b, ethertype Unknown (0x86e0), length 151:
13:23:27.706831 40:00:26:11:28:d0 > 45:00:00:91:39:a0, ethertype Unknown (0x86e0), length 145:
13:23:27.706920 40:00:3c:11:ef:1d > 45:00:03:40:00:00, ethertype Unknown (0x8efa), length 832:
13:23:27.707786 40:00:3c:11:f2:28 > 45:00:00:35:00:00, ethertype Unknown (0x8efa), length 53:
13:23:27.716902 40:00:25:11:28:1d > 45:00:02:3d:39:a7, ethertype Unknown (0x86e0), length 573:
13:23:27.717064 40:00:25:11:28:1c > 45:00:02:3d:39:a8, ethertype Unknown (0x86e0), length 573:
13:23:27.717229 40:00:25:11:28:1c > 45:00:02:3c:39:a9, ethertype Unknown (0x86e0), length 572:
13:23:27.717395 40:00:25:11:27:fb > 45:00:02:5c:39:aa, ethertype Unknown (0x86e0), length 604:
13:23:27.727835 40:00:25:11:28:52 > 45:00:01:fe:39:b1, ethertype Unknown (0x86e0), length 510:
13:23:27.727986 40:00:25:11:28:3d > 45:00:02:12:39:b2, ethertype Unknown (0x86e0), length 530:
13:23:27.728827 40:00:26:11:27:b8 > 45:00:01:99:39:b0, ethertype Unknown (0x86e0), length 409:
13:23:27.736927 40:00:25:11:27:51 > 45:00:02:fc:39:b4, ethertype Unknown (0x86e0), length 764:
13:23:27.737137 40:00:25:11:28:3a > 45:00:02:12:39:b5, ethertype Unknown (0x86e0), length 530:
13:23:27.737288 40:00:25:11:26:95 > 45:00:03:b6:39:b6, ethertype Unknown (0x86e0), length 950:
13:23:27.737557 40:00:25:11:25:ab > 45:00:04:9f:39:b7, ethertype Unknown (0x86e0), length 1183:
13:23:27.746945 40:00:25:11:28:32 > 45:00:02:11:39:be, ethertype Unknown (0x86e0), length 529:
13:23:27.747092 40:00:25:11:28:44 > 45:00:01:fe:39:bf, ethertype Unknown (0x86e0), length 510:
13:23:27.747239 40:00:25:11:28:01 > 45:00:02:40:39:c0, ethertype Unknown (0x86e0), length 576:
13:23:27.747408 40:00:25:11:28:00 > 45:00:02:40:39:c1, ethertype Unknown (0x86e0), length 576:
13:23:27.747612 40:00:25:11:28:41 > 45:00:01:fe:39:c2, ethertype Unknown (0x86e0), length 510:
13:23:27.747730 40:00:25:11:28:17 > 45:00:02:27:39:c3, ethertype Unknown (0x86e0), length 551:
13:23:27.747894 40:00:26:11:28:0a > 45:00:01:3a:39:bd, ethertype Unknown (0x86e0), length 314:
13:23:27.747993 40:00:26:11:27:ff > 45:00:01:3c:39:c6, ethertype Unknown (0x86e0), length 316:
13:23:27.748092 40:00:25:11:28:16 > 45:00:02:27:39:c4, ethertype Unknown (0x86e0), length 551:
13:23:27.748268 40:00:25:11:28:15 > 45:00:02:27:39:c5, ethertype Unknown (0x86e0), length 551:
13:23:27.757004 40:00:25:11:28:0f > 45:00:02:27:39:cb, ethertype Unknown (0x86e0), length 551:
13:23:27.757158 40:00:25:11:28:24 > 45:00:02:11:39:cc, ethertype Unknown (0x86e0), length 529:
13:23:27.757309 40:00:25:11:28:0d > 45:00:02:27:39:cd, ethertype Unknown (0x86e0), length 551:
13:23:27.757469 40:00:25:11:27:ed > 45:00:02:46:39:ce, ethertype Unknown (0x86e0), length 582:
13:23:27.757640 40:00:25:11:28:34 > 45:00:01:fe:39:cf, ethertype Unknown (0x86e0), length 510:
13:23:27.757794 40:00:25:11:28:33 > 45:00:01:fe:39:d0, ethertype Unknown (0x86e0), length 510:
13:23:27.757946 40:00:25:11:28:13 > 45:00:02:1d:39:d1, ethertype Unknown (0x86e0), length 541:
13:23:27.758135 40:00:25:11:27:f0 > 45:00:02:3f:39:d2, ethertype Unknown (0x86e0), length 575:
13:23:27.766972 40:00:25:11:27:e8 > 45:00:02:3f:39:da, ethertype Unknown (0x86e0), length 575:
13:23:27.767152 40:00:25:11:27:c7 > 45:00:02:5f:39:db, ethertype Unknown (0x86e0), length 607:
Might a better approach be, as follows.
So it is possible to create an IFB based on ingress using:
tc qdisc add dev wan handle ffff: ingress
tc filter add dev wan parent ffff: protocol ip prio 1 u32 match ip sport 51820 0xffff action pass
tc filter add dev wan parent ffff: prio 2 matchall action mirred egress redirect dev ifb
But can an IFB be created for egress traffic to facilitate tc filtering before shaping? If so, what is the recipe for this? @dlakelan?
Because if I knew that, couldn't I create ifb-ul and ifb-dl for br-lan and then just 'tc filter pass' on 192.168.1.0/24? And then shape on ifb-ul and ifb-dl? That way everything would be unencrypted:
CAKE supports use of tc filter to prioritise, but it is a pity it doesn't support tc filter to 'pass' on traffic. Because that would mean being able to selectively apply CAKE on an interface, without creating an IFB, right?
CAKE might not be the best qdisc choice here. You might find more flexibility with a classful qdisc like htb, paired with fq_codel. Then have a htb inner class unthrottled for LAN traffic and another htb inner class for WAN download traffic, shaped to your 30Mbit.
1 Like
I think he wants a global speed limit for all traffic, but can not use the VPN for some streaming media providers, probably because they detect the VPN and stop servicing those connections (due to IMHO interesting licensing practices in the entertainment field)....
I am mostly with you, BTW, and personally I would run cake on the LAN side of a wire only-router, so all flows are seen before en- or after de-cryption, but then I am eying a raspberry pi4B frankenrouter for some time anyway and that would just push me over to do this 
2 Likes
Thanks all for the very interesting and enjoyable discussion on this, and I am very appreciative of the insight offered, not least of all the suggestion from @dlakelan as to a possible solution.
Indeed - I am looking for global speed limit 30Mbit, and for that to be fairly issued to flows primarily made up of VPN but also bypassed VPN (to deal with Netflix/Prime VPN detection).
I don't like the idea of a separate device because it introduces another failure point. If the RPi4 is down my whole connection is down. I already introduce an extra failure point but making my 4G router a modem and adding the OpenWrt router, but that seems justifiable given that VPN overcomes ISP throttling and QoS overcomes bufferbloat. Also the RT3200 is a beautiful product. I have three times put the RPi4 components into my Amazon basket and then thought the better of it, and each time because I have to purchase a whole bunch of components and it seems highly intellectually unsatisfying when my shiny RT3200 can surely be tweaked to make it work.
@dlakelan I am keenly awaiting your thoughts regarding whether the encryption aspect breaks your proposed implementation, mindful that I have actually been able to implement that - and other users following this thread can just use this script to provide CAKE on VPN with PBR bypass. I am keen to end up with a solution that can be used by others.
well the decrypted packet is not necessarily the exact same size as the encrypted packet, and it's the encrypted packet that goes through the bottleneck... so that breaks things a little. And then, if you want flow fairness, I really don't know how well that all works at all. The IFB should see a mix of internally addressed packets (those from WG) and externally addressed packets (ie pre-NAT) from the WAN directly. That may be a bit wonky. Not sure.
I'd suggest to put a fairly high per packet overhead (maybe 100 bytes?), and do some testing, and hope it works
Hmm, OK. Do you know if it is possible to create IFB based on egress (not just ingress), and if so how to implement that? Then I can easily create two IFB's for br-lan....
Along the lines:
tc qdisc add dev br-lan handle ???: egress
tc filter add dev br-lan parent ffff: protocol ip prio 1 u32 match ip dst 192.168.1.0/24 action pass
tc filter add dev br-lan parent ffff: prio 2 matchall action mirred egress redirect dev ifb
I just lack the syntax / knowledge as to how to create such an IFB for flow in the opposite direction to the normal ingress case.
I don't understand the goal here, which packets are going to which IFB? You want all packets to go through ONE device if you want to limit TOTAL bandwidth.
@dlakelan - it's 30Mbit for upload and 30Mbit for download, i.e. one ifb for upload traffic @ 30Mbit/s and a separate ifb for download traffic @ 30Mbit/s. Is that possible and if so how, see below.
I can just shape on br-lan - that works except it includes LAN<>ROUTER and ROUTER<>LAN traffic. So I would like to take out 192.168.1.0/24 traffic from interface to apply CAKE on. Then I have solved the problem. But how do I do that? It seems that aside from rerouting traffic on router the best way is using 'tc filter' like:
tc qdisc add dev br-lan handle ffff: ingress
tc filter add dev br-lan parent ffff: protocol ip prio 1 u32 match ip dst 192.168.1.0/24 action pass
tc filter add dev br-lan parent ffff: prio 2 matchall action mirred egress redirect dev ifb
That beautifully takes out local traffic.
For br-lan that would only take care of the outgoing upload, but how would I then do the same for the download? I am thinking I could create another IFB to 'tc filter' on that, but I don't know how to do that...
Something like the below (I just can't figure out the syntax - can you help?):
tc qdisc add dev br-lan handle ???: egress
tc filter add dev br-lan parent ???: protocol ip prio 1 u32 match ip src 192.168.1.0/24 action pass
tc filter add dev br-lan parent ???: prio 2 matchall action mirred egress redirect dev ifb2
A br-lan based solution would have the benefit of working in all cases even without use of a VPN at all. And VPN with or without PBR would not break it.
And all the flows are nice and unencrypted.
Ok I think I see the issue.
I think for "download" direction you probably want a routing based solution. Put a Veth with one side connected to br-lan and route all incoming traffic from the WAN or WG interfaces to that. put cake on that.
2 Likes
So is the dual IFB not possible? I think I can see how the veth would work. I know how to do that for download. So then I just do the IFB for upload only. But before I do would be nice to know I can't just use two IFB's. I would prefer another IFB because I don't have to tinker with routing then.
Yeah, the problem is you want the IFB to be handling unencrypted an un-natted packets, which means that you have to handle them well after the ingress from the WAN/WG so the only way to handle them at that stage of the game is to get them routed to you...
1 Like
no for the download direction you don't need IFB1 at all and you just put Cake directly on the veth0.
1 Like
isn't it ingress on veth0, and so IFB is needed?
No, you put it on the egress of veth0, when it's routed, it will go through the egress, be sent to the br-lan and then distributed along the bridge.
1 Like
OK many thanks! working on this now..