Hello Team,
I would like to learn, how can i interpret the crash logs and learn how to debug them and find the root cause of the crash.
I am using ar71xx based qsdk (QCA) device running openwrt12.09.
The crash logs are stored at /sys/kernel/debug/crashlog.
I have attached below the sample crashlogs which i recently found, and i want to know is there any method or tool to debug and troubleshoot the crashlogs generated.
root@EB_1F_B_AP07_QN_00:AC:56:~# cat /sys/kernel/debug/crashlog
Time: 1656327096.902073
Modules: shortcut_fe_cm@83182000+1920 smart_antenna@85390000+883e ath_pktlog@84fc0000+41d0 qca_ol@85700000+ee603 qca_da@85800000+31990 ath_dev@85b80000+46d74 hst_tx99@86b34000+1e84 ath_rate_atheros@86980000+af46 ath_hal@86a00000+dd75d umac@85a00000+176005 ath_spectral@86b68000+7e9b ath_dfs@86b70000+ff01 qdf@86b50000+6196 asf@86b44000+1d11 mem_manager@86846000+171c iptable_nat@86b42000+350 ip6table_nat@86b3b000+350 rndis_host@862e2000+13e9 qmi_wwan@86b3c000+29f0 pppoe@86b30000+205d nf_nat_ipv6@86b2e000+11b3 nf_nat_ipv4@86b04000+1111 nf_conntrack_ipv6@86b12000+1730 nf_conntrack_ipv4@86b20000+1550 ipt_REJECT@86b0d000+410 ipt_MASQUERADE@862f6000+2f0 huawei_cdc_ncm@86b0b000+650 cdc_ncm@86b1c000+3008 cdc_ether@862f0000+ebc xt_u32@86b09000+410 xt_time@86b01000+6d0 xt_tcpudp@86b0e000+710 xt_tcpmss@86b07000+470 xt_string@862f4000+330 xt_statistic@862ef000+370 xt_state@862f2000+310 xt_recent@86b02000+1b90 xt_quota2@862f9000+c00 xt_quota@862ee000+350 xt_policy@862ed000+850 xt_pkttype@862ea000+2d0 xt_physdev@862e8000+570 xt_owner@862bc000+330 xt_nat@862e1000+4d0 xt_multiport@862d1000+4f0 xt_mark@862df000+2f0 xt_mac@862dc000+2d0 xt_limit@862da000+450 xt_length@862ce000+310 xt_iprange@862d7000+450 xt_iface@862cb000+3f0 xt_id@862d4000+250 xt_hl@862d2000+370 xt_helper@8632e000+3d0 xt_hashlimit@862cc000+1820 xt_esp@862c8000+350 xt_ecn@862c1000+570 xt_dscp@862b6000+430 xt_conntrack@862c3000+8d0 xt_connmark@8631d000+4f0 xt_connlimit@863e6000+eb0 xt_connbytes@862be000+630 xt_condition@862b8000+7a0 xt_comment@862af000+250 xt_addrtype@863c5000+970 xt_TCPMSS@862ae000+af0 xt_REDIRECT@8626e000+2f0 xt_NETMAP@86248000+530 xt_LOG@863c6000+350 xt_HL@86252000+570 xt_DSCP@86262000+630 xt_CT@86380000+a30 xt_CLASSIFY@8625b000+290 usbnet@862b0000+42a6 ts_kmp@86259000+590 ts_fsm@86bd6000+a30 ts_bm@86270000+5d0 pppox@86393000+5aa ppp_async@862a6000+18d0 nf_reject_ipv4@862a5000+903 nf_nat_redirect@863ee000+43b nf_nat_masquerade_ipv4@862ab000+5fc nf_log_ipv4@8639b000+cf0 nf_defrag_ipv6@862a8000+24f7 nf_defrag_ipv4@8639e000+3d6 nf_conntrack_rtcache@863f0000+a90 nf_conntrack_netlink@862a0000+4e10 lzo_decompress@863c7000+5d7 lzo_compress@86247000+8b2 lz4_decompress@863e0000+801 lz4_compress@863be000+a8e iptable_raw@86be5000+2f0 iptable_mangle@863d6000+410 iptable_filter@86399000+310 ipt_ah@86290000+310 ipt_ECN@863ff000+5d0 ip_tables@86294000+254d crc_ccitt@86291000+44b cdc_wdm@86292000+1f35 em_cmp@86287000+330 sch_teql@863fe000+e30 em_nbyte@863fa000+330 sch_htb@86280000+33e0 sch_prio@863f6000+d30 sch_tbf@863f8000+14b0 sch_dsmark@863e9000+f50 sch_pie@863de000+1090 act_ipt@86395000+a50 sch_gred@863e4000+1ab0 cls_basic@863d8000+bf0 em_text@86388000+5d0 sch_red@8638c000+1290 em_meta@863d0000+1550 act_police@86394000+d10 sch_codel@8636e000+12d0 sch_sfq@863c0000+20d0 sch_fq@8638e000+1830 act_connmark@8689f000+6b0 act_skbedit@86ba7000+6d0 act_mirred@86336000+ad0 em_u32@8631b000+290 cls_u32@86386000+1a30 cls_tcindex@86bf6000+1170 cls_flow@863bc000+1630 cls_route@86bfa000+1290 cls_fw@86383000+e50 sch_hfsc@86bc4000+34f0 sch_ingress@86363000+370 hyfi_bridging@863a0000+14ea2 ledtrig_usbdev@8631c000+7f0 ledtrig_oneshot@86333000+5f0 xt_LED@86326000+5f0 ledtrig_morse@86302000+770 ledtrig_heartbeat@86bfd000+5c0 xt_set@86bfe000+18f0 ip_set_list_set@86316000+18f0 ip_set_hash_netiface@86bf0000+5f50 ip_set_hash_netport@86be8000+5f30 ip_set_hash_netnet@86bd8000+6690 ip_set_hash_net@86bd0000+59d0 ip_set_hash_netportnet@86bc8000+6a50 ip_set_hash_mac@86bc0000+2630 ip_set_hash_ipportnet@86338000+6490 ip_set_hash_ipportip@86328000+50f0 ip_set_hash_ipport@86320000+4d50 ip_set_hash_ipmark@86310000+4b30 ip_set_hash_ip@86308000+4af0 ip_set_bitmap_port@86300000+1550 ip_set_bitmap_ipmac@86278000+1810 ip_set_bitmap_ip@86274000+1830 ip_set@86268000+568a nfnetlink@8624e000+102b ip6t_NPT@8625d000+690 ip6t_MASQUERADE@86246000+290 nf_nat_masquerade_ipv6@8624a000+6dc nf_nat@86358000+29ac ip6t_rt@86b8a000+6d0 ip6t_frag@86355000+3b0 ip6t_hbh@86353000+570 ip6t_eui64@86345000+350 ip6t_mh@86350000+330 ip6t_ah@86349000+390 ip6t_ipv6header@8634c000+4b0 ip6t_REJECT@86bb4000+410 nf_reject_ipv6@86348000+9e8 nf_log_ipv6@86346000+d90 nf_log_common@86bb9000+a4f ip6table_raw@86b91000+2b0 ip6table_mangle@86b95000+4b0 ip6table_filter@86bba000+2d0 ip6_tables@86340000+2481 x_tables@86bbc000+2d87 hyfi_qdisc@86b97000+a80 ifb@868f7000+c70 ipcomp6@868f4000+650 xfrm6_tunnel@868f2000+8ff xfrm6_mode_tunnel@86376000+550 xfrm6_mode_transport@8637a000+370 xfrm6_mode_beet@86369000+530 esp6@8636a000+1330 ah6@86b86000+1290 ipcomp@868cf000+650 xfrm4_tunnel@86368000+4b0 xfrm4_mode_tunnel@8636c000+590 xfrm4_mode_transport@86365000+330 xfrm4_mode_beet@868be000+650 esp4@86372000+14f0 ah4@8687c000+12d0 tunnel6@868cd000+75e tunnel4@868ee000+71e veth@86361000+c50 tun@86ba0000+3aef qca_ssdk@86000000+23ef20 ppp_generic@868f8000+5abf slhc@868de000+115b nf_conntrack@868e0000+d52f af_key@868d0000+5de0 xfrm_user@868c8000+4b50 xfrm_ipcomp@868af000+b5d xfrm_algo@86871000+f55 shortcut_fe_ipv6@868b0000+cf59 shortcut_fe@868a0000+ca69 sha256_generic@86894000+273a sha1_generic@8687a000+895 md5@86872000+750 hmac@86885000+950 des_generic@86888000+490b deflate@86873000+5d0 zlib_inflate@86874000+3122 zlib_deflate@86880000+4898 cbc@86870000+790 authenc@86842000+d7b usb_storage@87240000+963b ledtrig_transient@8718d000+770 ehci_platform@871aa000+1150 ehci_ath79@87206000+1170 ehci_hcd@87218000+7ef7sd_mod@87210000+60e0 scsi_mod@87220000+16450 gpio_button_hotplug@871a6000+1910 usbcore@871e0000+1dc13 nls_base@871a8000+13c0 usb_common@871a0000+8ed crc16@8719c000+447 mii@8719a000+e38 aead@8716f000+de1 crypto_null@8718e000+a42 crypto_hash@87194000+2762
<4>[17370.645146] ieee80211_ioctl_kickmac[10944]
<4>[17370.660705] ieee80211_ioctl_kickmac[10944]
<4>[17371.612382] Sending SCAN START cmd
<4>[17372.049336] ieee80211_ioctl_kickmac[10944]
<4>[17372.064138] ieee80211_ioctl_kickmac[10944]
<4>[17372.792118] [wifi1] FWLOG: [17744960] WAL_DBGID_TX_BA_SETUP ( 0x44f4c0, 0x44390006, 0x0, 0x2, 0x0 )
<4>[17375.197986] Warning:-QCN_IE attribute 2 invalid or not defined yet
<4>[17377.115422] ieee80211_ioctl_kickmac[10944]
<4>[17377.129717] ieee80211_ioctl_kickmac[10944]
<4>[17378.090782] ieee80211_ioctl_kickmac[10944]
<4>[17378.127648] ieee80211_ioctl_kickmac[10944]
<4>[17378.262166] ieee80211_ioctl_kickmac[10944]
<4>[17378.276341] ieee80211_ioctl_kickmac[10944]
<4>[17381.142430] ieee80211_ioctl_kickmac[10944]
<3>[17381.147952] Assertion failed! qdf_atomic_read(&tid->paused) > 0:ath_tx_resume_tid /home/builder/workdir/qsdk-6.2.1-611B1/build_dir/target-mips_74kc_uClibc-1.0.14/linux-ar71xx_generic/qca-wifi-g33d9df7-dirty-unified-perf/qca-wifi-g33d9df7-dirty/os/linux/../../lmac/ath_dev/ath_xmit_ht.c:475
<4>[17381.174510] CPU: 0 PID: 28567 Comm: iwpriv Tainted: P 4.4.60 #190
<4>[17381.182168] Stack : 804964a2 00000044 00000000 00000001 803ce12c 8043f383 831bc7a4 00006f97
<4>[17381.182168] 804936f0 001a1053 00000008 85bd0420 00000001 800aed4c 00000000 00000003
<4>[17381.182168] 00000006 00000000 803d2960 827312cc 00000000 800acd00 804964a2 00000115
<4>[17381.182168] 8043e0f0 80208a00 00000000 00000000 00000000 00000000 00000000 00000000
<4>[17381.182168] 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
<4>[17381.182168] ...
<4>[17381.219312] Call Trace:
<4>[17381.221866] [<80076640>] show_stack+0x50/0x84
<4>[17381.226450] [<85ba5d9c>] ath_tx_resume_tid+0x1c0/0x1c4 [ath_dev]
<4>[17381.232704] [<85baaf94>] ath_tx_node_resume+0xd4/0x110 [ath_dev]
<4>[17381.239397] [<85a5e494>] ieee80211_mlme_node_pwrsave_ap+0x11c/0x2d8 [umac]
<4>[17381.247022] [<858042e0>] 0x858042e0
<4>[17381.250638]
<4>[17381.252197] Kernel bug detected[#1]:
<4>[17381.255889] CPU: 0 PID: 28567 Comm: iwpriv Tainted: P 4.4.60 #190
<4>[17381.263519] task: 831bc520 ti: 82730000 task.ti: 82730000
<4>[17381.269088] $ 0 : 00000000 00000001 00000000 00000000
<4>[17381.274518] $ 4 : 8043d644 8043d644 8043e0f0 80208aac
<4>[17381.279947] $ 8 : 00000000 00000003 00000001 00000000
<4>[17381.285378] $12 : 00000000 03bf0000 00000000 bc000000
<4>[17381.290806] $16 : 8584bfe4 85848000 8584c928 80440000
<4>[17381.296237] $20 : 84616af8 001a1053 00000008 85bd0420
<4>[17381.301667] $24 : 00000002 80207fb8
<4>[17381.307096] $28 : 82730000 82731320 00000001 85ba5d9c
<4>[17381.312527] Hi : 00000000
<4>[17381.315501] Lo : ec4e4000
<4>[17381.318542] epc : 85ba5d9c ath_tx_resume_tid+0x1c0/0x1c4 [ath_dev]
<4>[17381.325120] ra : 85ba5d9c ath_tx_resume_tid+0x1c0/0x1c4 [ath_dev]
<4>[17381.331674] Status: 1100dc03 KERNEL EXL IE
<4>[17381.336021] Cause : 00800024 (ExcCode 09)
<4>[17381.340161] PrId : 00019750 (MIPS 74Kc)
<4>[17381.344208] Modules linked in: shortcut_fe_cm smart_antenna(P) ath_pktlog(P) qca_ol qca_da ath_dev(P) hst_tx99(P) ath_rate_atheros(P) ath_hal(P) umac ath_spectral(P) ath_dfs(P) qdf asf(P) mem_manager(P) iptable_nat ip6table_nat rndis_host qmi_wwan pppoe nf_nat_ipv6 nf_nat_ipv4 nf_conntrack_ipv6 nf_conntrack_ipv4 ipt_REJECT ipt_MASQUERADE huawei_cdc_ncm cdc_ncm cdc_ether xt_u32 xt_time xt_tcpudp xt_tcpmss xt_string xt_statistic xt_state xt_recent xt_quota2 xt_quota xt_policy xt_pkttype xt_physdev xt_owner xt_nat xt_multiport xt_mark xt_mac xt_limit xt_length xt_iprange xt_iface xt_id xt_hl xt_helper xt_hashlimit xt_esp xt_ecn xt_dscp xt_conntrack xt_connmark xt_connlimit xt_connbytes xt_condition xt_comment xt_addrtype xt_TCPMSS xt_REDIRECT xt_NETMAP xt_LOG xt_HL xt_DSCP xt_CT xt_CLASSIFY usbnet ts_kmp ts_fsm ts_bm pppox ppp_async nf_reject_ipv4 nf_nat_redirect nf_nat_masquerade_ipv4 nf_log_ipv4 nf_defrag_ipv6 nf_defrag_ipv4 nf_conntrack_rtcache nf_conntrack_netlink lzo_decompress lzo_compress lz4_decompress lz4_compress iptable_raw iptable_mangle iptable_filter ipt_ah ipt_ECN ip_tables crc_ccitt cdc_wdm em_cmp sch_teql em_nbyte sch_htb sch_prio sch_tbf sch_dsmark sch_pie act_ipt sch_gred cls_basic em_text sch_red em_meta act_police sch_codel sch_sfq sch_fq act_connmark act_skbedit act_mirred em_u32 cls_u32 cls_tcindex cls_flow cls_route cls_fw sch_hfsc sch_ingress hyfi_bridging ledtrig_usbdev ledtrig_oneshot xt_LED ledtrig_morse ledtrig_heartbeat xt_set ip_set_list_set ip_set_hash_netiface ip_set_hash_netport ip_set_hash_netnet ip_set_hash_net ip_set_hash_netportnet ip_set_hash_mac ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_hash_ipport ip_set_hash_ipmark ip_set_hash_ip ip_set_bitmap_port ip_set_bitmap_ipmac ip_set_bitmap_ip ip_set nfnetlink ip6t_NPT ip6t_MASQUERADE nf_nat_masquerade_ipv6 nf_nat ip6t_rt ip6t_frag ip6t_hbh ip6t_eui64 ip6t_mh ip6t_ah ip6t_ipv6header ip6t_REJECT nf_reject_ipv6 nf_log_ipv6 nf_log_common ip6table_raw ip6table_mangle ip6table_filter ip6_tables x_tables hyfi_qdisc ifb ipcomp6 xfrm6_tunnel xfrm6_mode_tunnel xfrm6_mode_transport xfrm6_mode_beet esp6 ah6 ipcomp xfrm4_tunnel xfrm4_mode_tunnel xfrm4_mode_transport xfrm4_mode_beet esp4 ah4 tunnel6 tunnel4 veth tun qca_ssdk ppp_generic slhc nf_conntrack af_key xfrm_user xfrm_ipcomp xfrm_algo shortcut_fe_ipv6 shortcut_fe sha256_generic sha1_generic md5 hmac des_generic deflate zlib_inflate zlib_deflate cbc authenc usb_storage ledtrig_transient ehci_platform ehci_ath79 ehci_hcd sd_mod scsi_mod gpio_button_hotplug usbcore nls_base usb_common crc16 mii aead crypto_null crypto_hash
<4>[17381.579504] Process iwpriv (pid: 28567, threadinfo=82730000, task=831bc520, tls=77023460)
<4>[17381.587938] Stack : 85bd0420 85bc0d58 85bba9a0 85bc0d7c 000001db 8273134c 846160f8 85baaf94
<4>[17381.587938] 00000000 84616000 00000001 85b89abc 00000000 851aa000 83440000 85bd0420
<4>[17381.587938] 84812400 00000088 8310d030 85a5e494 00000000 8273138c 00000008 8581cdc0
<4>[17381.587938] 85848000 84616000 00000000 85b89abc 00000000 02090220 ffffffff 851aa000
<4>[17381.587938] 83440000 833e1300 80440000 858042e0 82731520 80440000 84808300 02080020
<4>[17381.587938] ...
<4>[17381.624928] Call Trace:
<4>[17381.627499] [<85ba5d9c>] ath_tx_resume_tid+0x1c0/0x1c4 [ath_dev]
<4>[17381.633738] [<85baaf94>] ath_tx_node_resume+0xd4/0x110 [ath_dev]
<4>[17381.640422] [<85a5e494>] ieee80211_mlme_node_pwrsave_ap+0x11c/0x2d8 [umac]
<4>[17381.648017] [<858042e0>] 0x858042e0
<4>[17381.651626]
<4>[17381.653159]
<4>[17381.653159] Code: afa20010 0c06eec4 00000000 <000c000d> 90e20001 27bdffd0 afb30020 7c421880 00021940
<4>[17381.664029] ---[ end trace 368edd9fac1c1c57 ]---