Netgear R6220 crash on 21.02.0

[126775.919000] ------------[ cut here ]------------
[126775.928481] WARNING: CPU: 0 PID: 862 at backports-5.10.42-1/net/mac80211/airtime.c:457 0x86e57080 [mac80211@458b198c+0x7d9d0]
[126775.951275] Modules linked in: pppoe ppp_async iptable_nat xt_state xt_nat xt_conntrack xt_REDIRECT xt_MASQUERADE xt_FLOWOFFLOAD xt_CT pppox ppp_generic nf_nat nf_flow_table_hw nf_flow_table nf_conntrack mt76x2e mt76x2_common mt76x02_lib mt7603e mt76 mac80211 ipt_REJECT cfg80211 xt_time xt_tcpudp xt_multiport xt_mark xt_mac xt_limit xt_comment xt_TCPMSS xt_LOG slhc nf_reject_ipv4 nf_log_ipv4 nf_defrag_ipv6 nf_defrag_ipv4 iptable_mangle iptable_filter ip_tables crc_ccitt compat ledtrig_usbport nf_log_ipv6 nf_log_common ip6table_mangle ip6table_filter ip6_tables ip6t_REJECT x_tables nf_reject_ipv6 leds_gpio xhci_plat_hcd xhci_pci xhci_mtk xhci_hcd gpio_button_hotplug usbcore nls_base usb_common
[126776.074445] CPU: 0 PID: 862 Comm: mt76-tx phy1 Not tainted 5.4.143 #0
[126776.087435] Stack : 00000000 80850000 00000000 8007dec4 00000000 00000000 00000000 00000000
[126776.104236]         00000000 00000000 00000000 00000000 00000000 00000001 86f45c30 8d97c1c8
[126776.121036]         86f45cc8 00000000 00000000 00000000 00000038 805eb4e4 00000018 35206465
[126776.137854]         00000000 000018a3 00000000 746d203a 00000000 86f45c10 00000000 86e57080
[126776.154653]         00000009 00000014 00000004 00000002 00000008 8035c59c 001f382f 00000001
[126776.171454]         ...
[126776.176485] Call Trace:
[126776.176503] [<8007dec4>] 0x8007dec4
[126776.188666] [<805eb4e4>] 0x805eb4e4
[126776.195798] [<86e57080>] 0x86e57080 [mac80211@458b198c+0x7d9d0]
[126776.207746] [<8035c59c>] 0x8035c59c
[126776.214867] [<8000b05c>] 0x8000b05c
[126776.221969] [<8000b064>] 0x8000b064
[126776.229089] [<805d0e1c>] 0x805d0e1c
[126776.236205] [<8007a96c>] 0x8007a96c
[126776.243314] [<8002c140>] 0x8002c140
[126776.250437] [<86e57080>] 0x86e57080 [mac80211@458b198c+0x7d9d0]
[126776.262388] [<8002c1e8>] 0x8002c1e8
[126776.269519] [<86e57080>] 0x86e57080 [mac80211@458b198c+0x7d9d0]
[126776.281481] [<86e036f0>] 0x86e036f0 [mac80211@458b198c+0x7d9d0]
[126776.293440] [<86e57268>] 0x86e57268 [mac80211@458b198c+0x7d9d0]
[126776.305405] [<86e57348>] 0x86e57348 [mac80211@458b198c+0x7d9d0]
[126776.317375] [<8041e234>] 0x8041e234
[126776.324517] [<86cb2a24>] 0x86cb2a24 [mt76x02_lib@acaaf99d+0x9de0]
[126776.336829] [<86cc0530>] 0x86cc0530 [mt76@d5a4097e+0x96c0]
[126776.347926] [<86800101>] 0x86800101 [nf_nat@3d8cac59+0x64b0]
[126776.359383] [<86cc0530>] 0x86cc0530 [mt76@d5a4097e+0x96c0]
[126776.370489] [<86cb5004>] 0x86cb5004 [mt76x02_lib@acaaf99d+0x9de0]
[126776.382802] [<86cc0530>] 0x86cc0530 [mt76@d5a4097e+0x96c0]
[126776.393915] [<86cc05d4>] 0x86cc05d4 [mt76@d5a4097e+0x96c0]
[126776.405011] [<8004b484>] 0x8004b484
[126776.412129] [<805ece7c>] 0x805ece7c
[126776.419255] [<8004b994>] 0x8004b994
[126776.426356] [<8004b854>] 0x8004b854
[126776.433460] [<8004b854>] 0x8004b854
[126776.440563] [<8004b854>] 0x8004b854
[126776.447660] [<80006718>] 0x80006718
[126776.454762]
[126776.458791] ---[ end trace 0adc77ccc65b109a ]---

root@R6220-9:~# uname -a
Linux R6220-9 5.4.143 #0 SMP Tue Aug 31 22:20:08 2021 mips GNU/Linux

root@R6220-9:~# cat /etc/os-release
NAME="OpenWrt"
VERSION="21.02.0"
ID="openwrt"
ID_LIKE="lede openwrt"
PRETTY_NAME="OpenWrt 21.02.0"
VERSION_ID="21.02.0"
HOME_URL="https://openwrt.org/"
BUG_URL="https://bugs.openwrt.org/"
SUPPORT_URL="https://forum.openwrt.org/"
BUILD_ID="r16279-5cc0535800"
OPENWRT_BOARD="ramips/mt7621"
OPENWRT_ARCH="mipsel_24kc"
OPENWRT_TAINTS=""
OPENWRT_DEVICE_MANUFACTURER="OpenWrt"
OPENWRT_DEVICE_MANUFACTURER_URL="https://openwrt.org/"
OPENWRT_DEVICE_PRODUCT="Generic"
OPENWRT_DEVICE_REVISION="v0"
OPENWRT_RELEASE="OpenWrt 21.02.0 r16279-5cc0535800"

root@R6220-9:~# cat /etc/board.json
{
        "model": {
                "id": "netgear,r6220",
                "name": "Netgear R6220"
        },
        "led": {
                "wan": {
                        "name": "wan",
                        "sysfs": "green:wan",
                        "type": "netdev",
                        "device": "wan",
                        "mode": "link tx rx"
                }
        },
        "network": {
                "lan": {
                        "ports": [
                                "lan1",
                                "lan2",
                                "lan3",
                                "lan4"
                        ],
                        "protocol": "static"
                },
                "wan": {
                        "device": "wan",
                        "protocol": "dhcp"
                }
        },
        "system": {
                "compat_version": "1.1"
        }
}

root@R6220-9:~# cat /etc/config/wireless

config wifi-device 'radio0'
        option type 'mac80211'
        option channel '11'
        option hwmode '11g'
        option path '1e140000.pcie/pci0000:00/0000:00:01.0/0000:02:00.0'
        option htmode 'HT20'
        option disabled '0'
        option country 'US'

config wifi-iface 'default_radio0'
        option device 'radio0'
        option network 'lan'
        option mode 'ap'
        option ssid 'trident'
        option encryption 'psk2+aes'
        option key 'xxxxxxxx'
        option disabled '0'

config wifi-device 'radio1'
        option type 'mac80211'
        option channel '60'
        option hwmode '11a'
        option path '1e140000.pcie/pci0000:00/0000:00:00.0/0000:01:00.0'
        option htmode 'VHT80'
        option disabled '0'
        option country 'US'

config wifi-iface 'default_radio1'
        option device 'radio1'
        option network 'lan'
        option mode 'ap'
        option ssid 'trident'
        option encryption 'psk2+aes'
        option key 'xxxxxxxx'
        option disabled '0'

config wifi-iface
        option device 'radio0'
        option network 'lan'
        option mode 'ap'
        option ssid 'tridentG'
        option encryption 'psk2+aes'
        option key 'yyyyyyyy'
        option disabled '0'

Can you describe your issue ?
Did it happened once, or is it on a regular basis ?

it happened on a regular basis, every time after reboot, within several hours to a few days.
after that, it cannot provide WIFI service, and dmesg only showed one event
reboot will make it back

Did you install from scratch or upgrade from 19.07 ?
The wifi crashes, with other functions work ? If not you wouldn't be able to provide a log.

apparently it is known ...

it once run the 19.07 and upgraded to 21.02.0 using sysupgrade with firstboot.

others looked normal, wired network, ssh worked normal, as you pointed out

yeah, happened in same line of source code.
I remembered it worked fine with 19.07. introduced in 21.02?

When upgrading to 21.02, did you follow the instructions for converting network settings ? (swconfig to DSA)
Save your settings (to restore if you need), than perfom a reset within openWrt.
You will start with a fresh 21.02. Reconfigure the device, and so wifi.
When I went from 19.07, I started from scratch. Reconfiguring a device is quite fast.

I used sysupgrade to update the firmware to 21.02.0 with not keeping config and partition. after reboot, I also did firstboot command to wipe out the setting and reconfigure the system after another reboot

Sounds reasonable, so your device started from scratch.
Did you try to lower the power transmission on wifi ?

yes, I don't think the previous setting will be inherited to 21.02.0

I never used adjusting the transmission power on the wifi.

Do you think this would be related to transmission power?

If you choose "not keeping settings", well they are not kept :slight_smile:
Yes try to lower transmission power. Using at 100%, 24/7 may heat/overload the wifi unit. It is also adviced for sanitary reason. Try several values and check onto your wifi devices. you'll see that you don't need 100% power to have a high quality bandwith.

thanks for the advice I'll try it soon.

Reducing TX power did not work. I have reduced both radio's power by 3dBm (half the power)
I also found that both radio still worked but reconnecting needed 20~30 seconds to restore after this happened

[83786.018381] ------------[ cut here ]------------
[83786.027850] WARNING: CPU: 1 PID: 877 at backports-5.10.42-1/net/mac80211/airtime.c:457 0x86dd7080 [mac80211@e94de6cd+0x7d9d0]
[83786.050605] Modules linked in: pppoe ppp_async iptable_nat xt_state xt_nat xt_conntrack xt_REDIRECT xt_MASQUERADE xt_FLOWOFFLOAD xt_CT pppox ppp_generic nf_nat nf_flow_table_hw nf_flow_table nf_conntrack mt76x2e mt76x2_common mt76x02_lib mt7603e mt76 mac80211 ipt_REJECT cfg80211 xt_time xt_tcpudp xt_multiport xt_mark xt_mac xt_limit xt_comment xt_TCPMSS xt_LOG slhc nf_reject_ipv4 nf_log_ipv4 nf_defrag_ipv6 nf_defrag_ipv4 iptable_mangle iptable_filter ip_tables crc_ccitt compat ledtrig_usbport nf_log_ipv6 nf_log_common ip6table_mangle ip6table_filter ip6_tables ip6t_REJECT x_tables nf_reject_ipv6 leds_gpio xhci_plat_hcd xhci_pci xhci_mtk xhci_hcd gpio_button_hotplug usbcore nls_base usb_common
[83786.173305] CPU: 1 PID: 877 Comm: mt76-tx phy1 Not tainted 5.4.143 #0
[83786.186133] Stack : 00000000 80850000 00000000 8007dec4 00000000 00000000 00000000 00000000
[83786.202772]         00000000 00000000 00000000 00000000 00000000 00000001 86f4fc30 3c02485b
[83786.219391]         86f4fcc8 00000000 00000000 00000000 00000038 805eb4e4 65746e69 2e352064
[83786.236011]         00000000 00001048 00000000 0002a4f9 00000000 86f4fc10 00000000 86dd7080
[83786.252642]         00000009 00000014 00000004 00000002 00000000 8035c59c 00000004 80820004
[83786.269261]         ...
[83786.274115] Call Trace:
[83786.274132] [<8007dec4>] 0x8007dec4
[83786.285944] [<805eb4e4>] 0x805eb4e4
[83786.292894] [<86dd7080>] 0x86dd7080 [mac80211@e94de6cd+0x7d9d0]
[83786.304666] [<8035c59c>] 0x8035c59c
[83786.311604] [<8000b05c>] 0x8000b05c
[83786.318536] [<8000b064>] 0x8000b064
[83786.325467] [<805d0e1c>] 0x805d0e1c
[83786.332399] [<8007a96c>] 0x8007a96c
[83786.339328] [<8002c140>] 0x8002c140
[83786.346269] [<86dd7080>] 0x86dd7080 [mac80211@e94de6cd+0x7d9d0]
[83786.358040] [<8002c1e8>] 0x8002c1e8
[83786.364986] [<86dd7080>] 0x86dd7080 [mac80211@e94de6cd+0x7d9d0]
[83786.376762] [<86d836f0>] 0x86d836f0 [mac80211@e94de6cd+0x7d9d0]
[83786.388539] [<86dd7268>] 0x86dd7268 [mac80211@e94de6cd+0x7d9d0]
[83786.400319] [<86dd7348>] 0x86dd7348 [mac80211@e94de6cd+0x7d9d0]
[83786.412097] [<8041e234>] 0x8041e234
[83786.419047] [<86e82a24>] 0x86e82a24 [mt76x02_lib@74b3de72+0x9de0]
[83786.431171] [<86d60530>] 0x86d60530 [mt76@8b5cdea3+0x96c0]
[83786.442100] [<86d60530>] 0x86d60530 [mt76@8b5cdea3+0x96c0]
[83786.453013] [<86e85004>] 0x86e85004 [mt76x02_lib@74b3de72+0x9de0]
[83786.465135] [<86d60530>] 0x86d60530 [mt76@8b5cdea3+0x96c0]
[83786.476062] [<86d605d4>] 0x86d605d4 [mt76@8b5cdea3+0x96c0]
[83786.486973] [<8004b484>] 0x8004b484
[83786.493904] [<805ece7c>] 0x805ece7c
[83786.500843] [<8004b994>] 0x8004b994
[83786.507772] [<8004b854>] 0x8004b854
[83786.514706] [<8004b854>] 0x8004b854
[83786.521638] [<8004b854>] 0x8004b854
[83786.528569] [<80006718>] 0x80006718
[83786.535502]
[83786.539521] ---[ end trace 5131cd901e2b5d34 ]---

iwinfo wlan0 txp
   0 dBm (   1 mW)
   1 dBm (   1 mW)
   2 dBm (   1 mW)
   3 dBm (   1 mW)
   4 dBm (   2 mW)
   5 dBm (   3 mW)
   6 dBm (   3 mW)
   7 dBm (   5 mW)
   8 dBm (   6 mW)
   9 dBm (   7 mW)
  10 dBm (  10 mW)
  11 dBm (  12 mW)
  12 dBm (  15 mW)
  13 dBm (  19 mW)
  14 dBm (  25 mW)
  15 dBm (  31 mW)
  16 dBm (  39 mW)
  17 dBm (  50 mW)
  18 dBm (  63 mW)
  19 dBm (  79 mW)
  20 dBm ( 100 mW)
  21 dBm ( 125 mW)
* 22 dBm ( 158 mW)
  23 dBm ( 199 mW)
  24 dBm ( 251 mW)
  25 dBm ( 316 mW)

iwinfo wlan1 txp
   0 dBm (   1 mW)
   1 dBm (   1 mW)
   2 dBm (   1 mW)
   3 dBm (   1 mW)
   4 dBm (   2 mW)
   5 dBm (   3 mW)
   6 dBm (   3 mW)
   7 dBm (   5 mW)
   8 dBm (   6 mW)
   9 dBm (   7 mW)
  10 dBm (  10 mW)
  11 dBm (  12 mW)
  12 dBm (  15 mW)
  13 dBm (  19 mW)
  14 dBm (  25 mW)
  15 dBm (  31 mW)
  16 dBm (  39 mW)
  17 dBm (  50 mW)
  18 dBm (  63 mW)
  19 dBm (  79 mW)
* 20 dBm ( 100 mW)
  21 dBm ( 125 mW)
  22 dBm ( 158 mW)
  23 dBm ( 199 mW)

Hi folks.
Any update on this issue? I see the github issue is also open
I have a r6220 with openwrt 19.07.5. I am putting off the upgrade to v20 after seeing this issue.

Hi.
I have upgraded three R6220 without any issue. Meanwhile consider starting from scratch, so don't keep settings while upgrading.

looks like my hardware has some issues since I have done fresh update several times and the bug is still there within a day.

I have downgraded it to 19.07.4 and run for over three weeks without any issue

Hi.
If your device works fine with 19.07, than it has no hardware issue, and so should also works fine with 21.02.
I reckon that sometimes these devices can be capricious. I have already upgraded two 6220 to 21.02.1 and they run smoothly.

you are right, might not the hardware ...
it should be a race condition, depending on the use case and load patter according to mt76-tx phy1 Tainted, you posted before