Ath10k_pci not using all CPUs

meshing · January 17, 2025, 5:00am

I have a mesh(80211s + batman adv) with google onhub devices running 24.10-rc5.
They work like magic! Thanks for all the work.

Now I am itching to optimize use of CPU to increase throughput.
It has two cores but it seems that no matter what I do I cannot seem to utilize them(one core will go to 100% and the other might be just sitting when I run iperf3 tests) and that I think limits my speeds.
I am seeing ~250 Mbps(5 Ghz 80 Mhz) from mesh node to mesh node. But I think I should get more.
So looking into it a little bit I found that if I use smp_affinity to assign cores to irq's, that might make a difference. It did. I assigned core1 to eth0 and that made a boost to wifi speeds upto 350 mbps as I think all ethernet interrupts are now serviced by core 1 so core 0 has more room.

For some reason I cannot change the core assigned to wifi as I get invalid param for it's irq when I do:

echo 2 > /proc/irq/39/smp_affinity # same command works for ethernet irq's

How do I distribute ath10k_pci interrupts to both cores so that it distributes load and makes it go faster ? ( I have like -45 dbm mesh signal and I am using a frequency that has zero overlap with any of my neighbors.. that's what got the throughput between mesh<=>mesh upto 350 in addition to the irq optimization)

Or am I reading the data wrong and I have already reached the practical limit ?

To be clear, I see the same problem with 23.05 also.
Also, irqbalance does not help in distributing ath10k_pci interrupts.

brada4 · January 17, 2025, 8:00am

Please add output of:

ubus call system board
opkg list-installed | grep th10k
cat /etc/config/network
irqbalance --debug --once

Under normal conditions 1/2 of theoretical bw over mesh is expected.

Borromini · January 17, 2025, 9:37am

I don't think ath10k does multicore, but I might be mistaken. Either way, you're pretty much at the mercy of Qualcomm.

brada4 · January 17, 2025, 11:18am

It is one cpu for certain. Question is whether irqbalance functions at all

meshing · January 17, 2025, 9:14pm

ubus call system board
{
        "kernel": "6.6.69",
        "hostname": "ap4",
        "system": "ARMv7 Processor rev 0 (v7l)",
        "model": "TP-Link OnHub",
        "board_name": "tplink,onhub",
        "rootfs_type": "squashfs",
        "release": {
                "distribution": "OpenWrt",
                "version": "24.10.0-rc5",
                "revision": "r28304-6dacba30a7",
                "target": "ipq806x/chromium",
                "description": "OpenWrt 24.10.0-rc5 r28304-6dacba30a7",
                "builddate": "1736026537"
        }
}

 opkg list-installed | grep th10k
ath10k-board-qca988x - 20241110-r1
ath10k-firmware-qca988x - 20241110-r1
kmod-ath10k - 6.6.69.6.12.6-r1

cat /etc/config/network

config interface 'loopback'
        option device 'lo'
        option proto 'static'
        option ipaddr '127.0.0.1'
        option netmask '255.0.0.0'

config globals 'globals'
        option ula_prefix 'fd98:c839:3d5a::/48'

config interface 'lan'
        option device 'br-omni.99'
        option proto 'dhcp'

config interface 'cam'
        option proto 'dhcp'
        option device 'br-omni.3'
        option defaultroute '0'

config interface 'guest'
        option proto 'dhcp'
        option device 'br-omni.4'
        option defaultroute '0'

config device
        option type 'bridge'
        option name 'br-mgmt'
        option macaddr '9E:DE:D0:81:B1:41'
        option bridge_empty '1'

config interface 'mgmt'
        option proto 'static'
        option device 'br-mgmt'
        option ipaddr '192.168.14.1'
        option netmask '255.255.255.0'
        option defaultroute '0'

config interface 'bat0'
        option proto 'batadv'
        option routing_algo 'BATMAN_IV'
        option gw_mode 'off'
        option hop_penalty '30'
        option bridge_loop_avoidance '1'

config interface 'batwire'
        option proto 'batadv_hardif'
        option device 'wan'
        option master 'bat0'
        option mtu '1542'

config device
        option name 'br-omni'
        option type 'bridge'
        option stp '1'
        option igmp_snooping '1'
        option ipv6 '0'
        list ports 'bat0.4'
        list ports 'bat0.3'
        list ports 'bat0.99'
        list ports 'lan1'

config bridge-vlan
        option device 'br-omni'
        option vlan '3'
        list ports 'bat0.3'
        option macaddr '9E:DE:D0:81:B1:46'

config bridge-vlan
        option device 'br-omni'
        option vlan '4'
        list ports 'bat0.4'
        option macaddr '9E:DE:D0:81:B1:42'

config bridge-vlan
        option device 'br-omni'
        option vlan '99'
        option macaddr '9E:DE:D0:81:B1:43'
        list ports 'bat0.99'
        list ports 'lan1'

config device
        option name 'br-omni.3'
        option type '8021q'
        option ifname 'br-omni'
        option vid '3'
        option macaddr '9E:DE:D0:81:B1:47'

config device
        option name 'br-omni.4'
        option type '8021q'
        option ifname 'br-omni'
        option vid '4'
        option macaddr '9E:DE:D0:81:B1:44'

config device
        option name 'br-omni.99'
        option type '8021q'
        option ifname 'br-omni'
        option vid '99'
        option macaddr '9E:DE:D0:81:B1:45'

This machine seems not NUMA capable.
Prevent irq assignment to these isolated CPUs: 00000000
Prevent irq assignment to these adaptive-ticks CPUs: 00000000
Banned CPUs: 00000000
Package -1:  numa_node -1 cpu mask is 00000002 (load 0)
        Cache domain 0:  numa_node is -1 cpu mask is 00000002  (load 0)
                CPU number 1  numa_node is -1 (load 0)
Package -1:  numa_node -1 cpu mask is 00000001 (load 0)
        Cache domain 1:  numa_node is -1 cpu mask is 00000001  (load 0)
                CPU number 0  numa_node is -1 (load 0)
Adding IRQ 38 to database
Adding IRQ 57 to database
Adding IRQ 40 to database
Adding IRQ 58 to database
Adding IRQ 44 to database
Adding IRQ 56 to database
Adding IRQ 24 to database
Adding IRQ 26 to database
Adding IRQ 27 to database
Adding IRQ 28 to database
Adding IRQ 29 to database
Adding IRQ 32 to database
Adding IRQ 34 to database
Adding IRQ 35 to database
Adding IRQ 36 to database
Adding IRQ 41 to database
Adding IRQ 42 to database
Adding IRQ 45 to database
Adding IRQ 46 to database
Adding IRQ 48 to database
Adding IRQ 49 to database
Adding IRQ 50 to database
Adding IRQ 51 to database
Adding IRQ 52 to database
Adding IRQ 53 to database
Adding IRQ 54 to database
Adding IRQ 55 to database
NUMA NODE NUMBER: -1
LOCAL CPU MASK: ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff,ffffffff




-----------------------------------------------------------------------------
Package -1:  numa_node -1 cpu mask is 00000002 (load 0)
        Cache domain 0:  numa_node is -1 cpu mask is 00000002  (load 0)
                CPU number 1  numa_node is -1 (load 0)
                  Interrupt 56 node_num is -1 (ethernet/0:2369)
                  Interrupt 57 node_num is -1 (ethernet/0:1555)
          Interrupt 40 node_num is -1 (legacy/0:0)
  Interrupt 26 node_num is -1 (other/0:0)
  Interrupt 35 node_num is -1 (other/0:345)
  Interrupt 32 node_num is -1 (other/0:0)
  Interrupt 28 node_num is -1 (other/0:0)
  Interrupt 54 node_num is -1 (other/0:0)
  Interrupt 52 node_num is -1 (other/0:0)
  Interrupt 50 node_num is -1 (other/0:0)
  Interrupt 48 node_num is -1 (other/0:0)
  Interrupt 45 node_num is -1 (other/0:0)
  Interrupt 41 node_num is -1 (other/0:0)
  Interrupt 55 node_num is -1 (other/0:0)
Package -1:  numa_node -1 cpu mask is 00000001 (load 0)
        Cache domain 1:  numa_node is -1 cpu mask is 00000001  (load 0)
                CPU number 0  numa_node is -1 (load 0)
                  Interrupt 58 node_num is -1 (ethernet/0:0)
          Interrupt 44 node_num is -1 (legacy/0:0)
          Interrupt 38 node_num is -1 (legacy/0:0)
  Interrupt 24 node_num is -1 (other/0:723)
  Interrupt 34 node_num is -1 (other/0:0)
  Interrupt 29 node_num is -1 (other/0:0)
  Interrupt 27 node_num is -1 (other/0:0)
  Interrupt 53 node_num is -1 (other/0:0)
  Interrupt 51 node_num is -1 (other/0:0)
  Interrupt 49 node_num is -1 (other/0:0)
  Interrupt 46 node_num is -1 (other/0:0)
  Interrupt 42 node_num is -1 (other/0:0)
  Interrupt 36 node_num is -1 (other/0:0)

meshing · January 17, 2025, 9:16pm

I can see more than one core used on TP Link Deco M9 which is also using ath10k. It still does not use all cores though.(It has 4)

brada4 · January 17, 2025, 9:45pm

It is 2 cpus with SMT aka hyperthreads, so you have to balance between 1-2 (0x1 or 0x3) 2-3 (0x4 or 0xc), and irqbalance does not work out, you can completely safely uninstall it at no ill or well effects.

Lets check further:

ls -l /sys/class/net/*/queues/

ath10k is one queue one cpu, so it can run on just one of 2 cores at once.

meshing · January 17, 2025, 10:07pm

It is 2 cpus with SMT aka hyperthreads, so you have to balance between 1-2 (0x1 or 0x3) 2-3 (0x4 or 0xc), and irqbalance does not work out, you can completely safely uninstall it at no ill or well effects.

Are you talking about onhub or Deco M9 plus here ?

ls -l /sys/class/net/*/queues/
/sys/class/net/bat0.3/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 14:04 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/bat0.4/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 14:04 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/bat0.99/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 14:04 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/bat0/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 14:04 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/br-mgmt/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 14:04 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/br-omni.3/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 14:04 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/br-omni.4/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 14:04 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/br-omni.99/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 14:04 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/br-omni/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 14:04 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/eth0/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 04:36 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/eth1/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 04:36 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/lan1/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 14:04 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/lo/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 14:04 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/phy0-ap0/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 04:36 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/phy0-ap1/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 04:36 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/phy0-ap2/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 04:36 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/phy1-ap0/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 04:36 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/phy1-ap1/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 04:38 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

/sys/class/net/wan/queues/:
drwxr-xr-x    2 root     root             0 Jan 17 14:04 rx-0
drwxr-xr-x    3 root     root             0 Jan 17 14:04 tx-0

brada4 · January 17, 2025, 10:18pm

All queues are bound to one cpu.
You can try examining

cat /proc/interrupts

Try to identify biggest network interrupt source

Then
echo "2-3" /proc/irq/NN/smp_affinity_list

it is more or less what irqbalance is supposed to do.

meshing · January 17, 2025, 10:49pm

How do I fix this error: Result not representable

root@ap4:~# cat /proc/interrupts | grep ath10
 56:    9569837          0   PCI-MSI 524288 Edge      ath10k_pci
 57:    6640913          0   PCI-MSI 134742016 Edge      ath10k_pci
 58:         27          0   PCI-MSI 268959744 Edge      ath10k_pci
root@ap4:~# echo -n "2-3" > /proc/irq/56/smp_affinity_list
ash: write error: Result not representable
root@ap4:~# echo  "2-3" > /proc/irq/56/smp_affinity_list
ash: write error: Result not representable
root@ap4:~# echo  "2-3" > /proc/irq/57/smp_affinity_list
ash: write error: Result not representable
root@ap4:~# echo  "2-3" > /proc/irq/58/smp_affinity_list
ash: write error: Result not representable
root@ap4:~# cat /proc/irq/58/smp_affinity_list
0-1

brada4 · January 17, 2025, 11:15pm

It is fairly obvious you have 2 cores, no threading at all. echo 1 ok?
Also wired irq-s count.

meshing · January 17, 2025, 11:28pm

root@ap4:~# echo 1 > /proc/irq/58/smp_affinity_list
ash: write error: Invalid argument
root@ap4:~# echo 1 > /proc/irq/58/smp_affinity
ash: write error: Invalid argument

brada4 · January 17, 2025, 11:45pm

And echo 0 ? Strange, usually things like IPI and other percpu stuff acts like that.
can you try to move ethernet (or usb if applicable) irq-s away?

meshing · January 18, 2025, 12:46am

root@ap4:~# echo 0 > /proc/irq/56/smp_affinity
ash: write error: Invalid argument

can you try to move ethernet (or usb if applicable) irq-s away?

That is what I have done to get to 350 Mbps but maybe that got reverted.

brada4 · January 18, 2025, 12:47am

It should alwsys show up in numbers

meshing · January 18, 2025, 12:50am

I think another good question to ask is.. should I be happy with 350 and not expect this to go higher any which way ?
Or I can patch openwrt code somehow and make it go higher with something custom built?

brada4 · January 18, 2025, 12:52am

Wifi transfers at half of link speed.

meshing · January 18, 2025, 12:58am

link speed as per luci is 950/1100/1300 depends on when I check.
350mbps was for when luci showed 1100. I guess I am hitting the limit.

brada4 · January 18, 2025, 1:00am

Yes, mesh halves it again unless you connecto to router at internet

lmc · February 12, 2025, 4:56pm

Any progress on this? same issue here

# ubus call system board
{
        "kernel": "5.10.176",
        "hostname": "OpenWrt",
        "system": "ARMv7 Processor rev 0 (v7l)",
        "model": "Buffalo WXR-2533DHP",
        "board_name": "buffalo,wxr-2533dhp",
        "rootfs_type": "squashfs",
        "release": {
                "distribution": "OpenWrt",
                "version": "22.03.5",
                "revision": "r20134-5f15225c1e",
                "target": "ipq806x/generic",
                "description": "OpenWrt 22.03.5 r20134-5f15225c1e"
        }
}

root@OpenWrt:~# cat /proc/interrupts
           CPU0       CPU1
 24:  837019228  413413955     GIC-0  18 Edge      gp_timer
 26:       1831          0     GIC-0  51 Edge      qcom_rpm_ack
 27:          0          0     GIC-0  53 Edge      qcom_rpm_err
 28:          0          0     GIC-0  54 Edge      qcom_rpm_wakeup
 37:      60255     401118     GIC-0 258 Level     eth1
 38:    2097458   11565198     GIC-0 202 Level     adm_dma
 39:          0          0     GIC-0 130 Level     bam_dma
 40:          0          0     GIC-0 128 Level     bam_dma
 41:          0          0     GIC-0 210 Edge      900000.clock-controller:thermal-sensor@900000
 43:          0          0   PCI-MSI   0 Edge      aerdrv
 45:          0          0   PCI-MSI 134217728 Edge      aerdrv
 46:        729       5044     GIC-0 184 Level     msm_serial0
 47:       6824       2268     GIC-0 187 Level     1a280000.spi
 48:          1          0   msmgpio  58 Edge      keys
 49:          0          0   msmgpio  54 Edge      keys
 50:          0          0   msmgpio  65 Edge      keys
 51:          2          0   msmgpio   6 Edge      keys
 52:          0          0   msmgpio  64 Edge      keys
 53:          0          0   msmgpio  55 Edge      keys
 54:          1          0   msmgpio  56 Edge      keys
 55:          1          0   msmgpio  57 Edge      keys
 56:  153660521  424461431     GIC-0 237 Level     xhci-hcd:usb1
 57: 2511292291 1100534286     GIC-0 142 Level     xhci-hcd:usb3
 58: 1667410086          0   PCI-MSI 524288 Edge      ath10k_pci
 59: 1204528211          0   PCI-MSI 134742016 Edge      ath10k_pci
IPI0:          0          0  CPU wakeup interrupts
IPI1:          0          0  Timer broadcast interrupts
IPI2:   10607093   11504848  Rescheduling interrupts
IPI3:  256438275  423455531  Function call interrupts
IPI4:          0          0  CPU stop interrupts
IPI5:       1879        764  IRQ work interrupts
IPI6:          0          0  completion interrupts
Err:          0

root@OpenWrt:~# echo 2 > /proc/irq/58/smp_affinity
ash: write error: Invalid argument
root@OpenWrt:~# echo 2 > /proc/irq/59/smp_affinity
ash: write error: Invalid argument

irqbalance is configured as...

root@OpenWrt:~# cat /etc/config/irqbalance
config irqbalance 'irqbalance'
        option enabled '1'

        # Level at which irqbalance partitions cache domains.
        # Default is 2 (L2$).
        option deepestcache '2'

        # The default value is 10 seconds
        option interval '10'

        # List of IRQ's to ignore
        #list banirq '36'
        #list banirq '69'

But it is not making any difference as expected since it is impossible to change the affinity manually for some reason.

Setting irq_mode=1 on ath10k_pci didn't make a difference either

# opkg list-installed | grep th10k
ath10k-board-qca99x0 - 20230804-1
ath10k-firmware-qca99x0-ct - 2020-11-08-1
kmod-ath10k-ct - 5.10.176+2022-05-13-f808496f-1