# [Linux Kernel] DUT Kernel oops - debug

http://awaysu.logdown.com/posts/780594
https://blog.csdn.net/weixin_41944449/article/details/95367769
https://hackmd.io/@iST40ExoQtubds5LhuuaAw/HJtq-qQbQ?type=view#

https://training.ti.com/debugging-embedded-linux-kernel-oops-logs

https://www.itread01.com/content/1547160677.html

PC is at dhd_pkt_unlink+0x14/0x94 [dhd]
LR is at dhd_prot_rxpost_reclaim_pkt+0x5c/0x118 [dhd]

PC is at bpm_alloc_buf_skb_attach+0x7c/0x160
LR is at 0x26f8

https://training.ti.com/sites/default/files/docs/Kernel-Debug-Series-Part6-understand-kernel-oops.pdf

hostsacd_daily_proc(145):HSAC-DBG:DATE[2021/8/2 21:2:15] 1627905735 1082 1082985(4) loggerdb:1406
client6_send***********
Unable to handle kernel NULL pointer dereference at virtual address 00000000
pgd = c0014000
[00000000] *pgd=00000000
Internal error: Oops: 817 [#1] PREEMPT SMP ARM
Modules linked in: nd_proxy(P) init_addr( (null) - (null)), core_addr(bf00b000 - bf00bacc)
ip6t_REJECT init_addr( (null) - (null)), core_addr(bfc7c000 - bfc7c198)
nf_reject_ipv6 init_addr( (null) - (null)), core_addr(bfc78000 - bfc78614)
ip6table_filter init_addr( (null) - (null)), core_addr(bfc74000 - bfc74050)
ip6table_mangle init_addr( (null) - (null)), core_addr(bfc70000 - bfc701b0)
nf_log_ipv6 init_addr( (null) - (null)), core_addr(bfc6c000 - bfc6ca14)
ip6_tables init_addr( (null) - (null)), core_addr(bfc65000 - bfc66e84)
nf_conntrack_ipv6 init_addr( (null) - (null)), core_addr(bfc5f000 - bfc5fe08)
nf_defrag_ipv6 init_addr( (null) - (null)), core_addr(bfc58000 - bfc58e50)
nf_nat_ipsec init_addr( (null) - (null)), core_addr(bfc54000 - bfc540ec)
arpt_mangle init_addr( (null) - (null)), core_addr(bfc50000 - bfc5016c)
arptable_filter init_addr( (null) - (null)), core_addr(bfc4c000 - bfc4c050)
arp_tables init_addr( (null) - (null)), core_addr(bfc46000 - bfc47b38)
ipt_REJECT init_addr( (null) - (null)), core_addr(bfc42000 - bfc42138)
nf_reject_ipv4 init_addr( (null) - (null)), core_addr(bfc3e000 - bfc3e504)
iptable_nat init_addr( (null) - (null)), core_addr(bfc3a000 - bfc3a078)
nf_nat_ipv4 init_addr( (null) - (null)), core_addr(bfc35000 - bfc35c10)
ipt_MASQUERADE init_addr( (null) - (null)), core_addr(bfc31000 - bfc31090)
iptable_filter init_addr( (null) - (null)), core_addr(bfc2d000 - bfc2d08c)
iptable_mangle init_addr( (null) - (null)), core_addr(bfc29000 - bfc29138)
ip_tables init_addr( (null) - (null)), core_addr(bfc22000 - bfc23da8)
nf_conntrack_ipv4 init_addr( (null) - (null)), core_addr(bfc1b000 - bfc1c72c)
nf_defrag_ipv4 init_addr( (null) - (null)), core_addr(bfc17000 - bfc17134)
nf_log_ipv4 init_addr( (null) - (null)), core_addr(bfc13000 - bfc13868)
nf_nat_h323 init_addr( (null) - (null)), core_addr(bfc0e000 - bfc0ef50)
nf_nat_masquerade_ipv4 init_addr( (null) - (null)), core_addr(bfc0a000 - bfc0a6f0)
nf_nat_pptp init_addr( (null) - (null)), core_addr(bfc06000 - bfc06410)
nf_nat_proto_gre init_addr( (null) - (null)), core_addr(bfc02000 - bfc02134)
nf_nat_pt init_addr( (null) - (null)), core_addr(bfbfd000 - bfbfdde8)
nf_nat_rtsp init_addr( (null) - (null)), core_addr(bfbf9000 - bfbf9830)
nf_conntrack_safety_proc init_addr( (null) - (null)), core_addr(bfbf4000 - bfbf4ccc)
xt_schedule(P) init_addr( (null) - (null)), core_addr(bfbf0000 - bfbf02e8)
xt_applications(P) init_addr( (null) - (null)), core_addr(bfbec000 - bfbec9dc)
xt_physdev init_addr( (null) - (null)), core_addr(bfbe8000 - bfbe82b8)
br_netfilter init_addr( (null) - (null)), core_addr(bfbe2000 - bfbe40f8)
xt_NFQUEUE init_addr( (null) - (null)), core_addr(bfbde000 - bfbde434)
xt_statistic init_addr( (null) - (null)), core_addr(bfbda000 - bfbda11c)
xt_multiport init_addr( (null) - (null)), core_addr(bfbd6000 - bfbd63d0)
xt_REDIRECT init_addr( (null) - (null)), core_addr(bfbd2000 - bfbd206c)
nf_nat_ftp init_addr( (null) - (null)), core_addr(bfbce000 - bfbce374)
nf_nat_irc init_addr( (null) - (null)), core_addr(bfbca000 - bfbca168)
nf_nat_redirect init_addr( (null) - (null)), core_addr(bfbc6000 - bfbc6198)
nf_nat_tftp init_addr( (null) - (null)), core_addr(bfbc2000 - bfbc207c)
nfnetlink_log init_addr( (null) - (null)), core_addr(bfbbc000 - bfbbd52c)
nfnetlink_queue init_addr( (null) - (null)), core_addr(bfbb5000 - bfbb6a54)
nf_conntrack_ftp init_addr( (null) - (null)), core_addr(bfbb0000 - bfbb0b88)
nf_conntrack_h323 init_addr( (null) - (null)), core_addr(bfba1000 - bfba48d4)
nf_conntrack_irc init_addr( (null) - (null)), core_addr(bfb9d000 - bfb9d4ac)
nf_conntrack_pptp init_addr( (null) - (null)), core_addr(bfb99000 - bfb99934)
nf_conntrack_proto_gre init_addr( (null) - (null)), core_addr(bfb95000 - bfb956fc)
nf_conntrack_rtsp init_addr( (null) - (null)), core_addr(bfb8a000 - bfb8bf3c)
nf_conntrack_tftp init_addr( (null) - (null)), core_addr(bfb86000 - bfb861b4)
nf_conntrack_netlink init_addr( (null) - (null)), core_addr(bfb7c000 - bfb806c4)
nf_dyndscp init_addr( (null) - (null)), core_addr(bfb77000 - bfb77c00)
nf_log_common init_addr( (null) - (null)), core_addr(bfb73000 - bfb73588)
nf_conntrack_ipsec init_addr( (null) - (null)), core_addr(bfb6f000 - bfb6f3fc)
nfnetlink init_addr( (null) - (null)), core_addr(bfb6a000 - bfb6a9e0)
xt_CLASSIFY init_addr( (null) - (null)), core_addr(bfb66000 - bfb66024)
xt_HL init_addr( (null) - (null)), core_addr(bfb62000 - bfb62224)
xt_connmark init_addr( (null) - (null)), core_addr(bfb5e000 - bfb5e1ac)
xt_helper init_addr( (null) - (null)), core_addr(bfb5a000 - bfb5a0f8)
xt_iprange init_addr( (null) - (null)), core_addr(bfb56000 - bfb56228)
xt_length init_addr( (null) - (null)), core_addr(bfb52000 - bfb520b4)
xt_tcpmss init_addr( (null) - (null)), core_addr(bfb4e000 - bfb4e1a0)
xt_time init_addr( (null) - (null)), core_addr(bfb4a000 - bfb4a31c)
xt_pkttype init_addr( (null) - (null)), core_addr(bfb46000 - bfb460a8)
xt_DSCP init_addr( (null) - (null)), core_addr(bfb42000 - bfb4226c)
xt_LOG init_addr( (null) - (null)), core_addr(bfb3e000 - bfb3e0c8)
xt_SKIPLOG init_addr( (null) - (null)), core_addr(bfb3a000 - bfb3a020)
xt_TCPMSS init_addr( (null) - (null)), core_addr(bfb36000 - bfb366fc)
xt_addrtype init_addr( (null) - (null)), core_addr(bfb32000 - bfb32450)
xt_blog init_addr( (null) - (null)), core_addr(bfb2e000 - bfb2e054)
xt_conntrack init_addr( (null) - (null)), core_addr(bfb2a000 - bfb2a644)
xt_dscp init_addr( (null) - (null)), core_addr(bfb26000 - bfb26140)
xt_flowlabel init_addr( (null) - (null)), core_addr(bfb22000 - bfb22078)
xt_limit init_addr( (null) - (null)), core_addr(bfb1e000 - bfb1e280)
xt_mac init_addr( (null) - (null)), core_addr(bfb1a000 - bfb1a094)
xt_mac_extend init_addr( (null) - (null)), core_addr(bfb16000 - bfb160e0)
xt_mark init_addr( (null) - (null)), core_addr(bfb12000 - bfb12074)
xt_nat init_addr( (null) - (null)), core_addr(bfb0e000 - bfb0e138)
xt_state init_addr( (null) - (null)), core_addr(bfb0a000 - bfb0a0b0)
xt_u32 init_addr( (null) - (null)), core_addr(bfb06000 - bfb061c0)
nf_nat init_addr( (null) - (null)), core_addr(bfaff000 - bfb00cd0)
nf_conntrack init_addr( (null) - (null)), core_addr(bfae3000 - bfaed868)
ip_gre init_addr( (null) - (null)), core_addr(bfadc000 - bfadd694)
gre init_addr( (null) - (null)), core_addr(bfad8000 - bfad8850)
wl(P) init_addr( (null) - (null)), core_addr(bf361000 - bf6b7180)
dhd init_addr( (null) - (null)), core_addr(bf277000 - bf2f5d1c)
igs(P) init_addr( (null) - (null)), core_addr(bf26e000 - bf270ed0)
emf(P) init_addr( (null) - (null)), core_addr(bf266000 - bf268740)
hnd init_addr( (null) - (null)), core_addr(bf201000 - bf22c6a0)
cfg80211 init_addr( (null) - (null)), core_addr(bf1c7000 - bf1e9348)
otp(P) init_addr( (null) - (null)), core_addr(bf1c3000 - bf1c3504)
bcm_thermal init_addr( (null) - (null)), core_addr(bf1be000 - bf1be7b4)
pwrmngtd(P) init_addr( (null) - (null)), core_addr(bf1ba000 - bf1ba480)
uas init_addr( (null) - (null)), core_addr(bf1b3000 - bf1b4d48)
usb_storage init_addr( (null) - (null)), core_addr(bf1a0000 - bf1a3824)
usblp init_addr( (null) - (null)), core_addr(bf199000 - bf19a87c)
bcm_usb init_addr( (null) - (null)), core_addr(bf194000 - bf1940b8)
xhci_plat_hcd init_addr( (null) - (null)), core_addr(bf190000 - bf1904a0)
xhci_hcd init_addr( (null) - (null)), core_addr(bf174000 - bf183234)
ohci_pci init_addr( (null) - (null)), core_addr(bf170000 - bf1701f4)
ohci_platform init_addr( (null) - (null)), core_addr(bf16c000 - bf16c6c0)
ohci_hcd init_addr( (null) - (null)), core_addr(bf161000 - bf165b58)
ehci_pci init_addr( (null) - (null)), core_addr(bf15d000 - bf15d584)
ehci_platform init_addr( (null) - (null)), core_addr(bf158000 - bf15873c)
ehci_hcd init_addr( (null) - (null)), core_addr(bf14b000 - bf152224)
bcm_pcie_hcd init_addr( (null) - (null)), core_addr(bf13e000 - bf142fb4)
bcmmcast init_addr( (null) - (null)), core_addr(bf12a000 - bf13351c)
bcm_enet init_addr( (null) - (null)), core_addr(bf0ff000 - bf114754)
archer(P) init_addr( (null) - (null)), core_addr(bf0c7000 - bf0e0e60)
cmdlist(P) init_addr( (null) - (null)), core_addr(bf0b1000 - bf0bc1dc)
pktflow(P) init_addr( (null) - (null)), core_addr(bf06b000 - bf090414)
bcm_ingqos(P) init_addr( (null) - (null)), core_addr(bf031000 - bf034ac8)
chipinfo(P) init_addr( (null) - (null)), core_addr(bf02d000 - bf02d104)
bcmvlan(P) init_addr( (null) - (null)), core_addr(bf00f000 - bf01ca10)
bcmlibs(P) init_addr( (null) - (null)), core_addr(bf005000 - bf007554)
wlcsm(P) init_addr( (null) - (null)), core_addr(bf000000 - bf000dd8)

CPU: 1 PID: 0 Comm: swapper/1 Tainted: P 4.1.52 #1
Hardware name: Generic DT based system
task: df42dc00 ti: df45a000 task.ti: df45a000
PC is at dhd_pkt_unlink+0x14/0x94 [dhd]
LR is at dhd_prot_rxpost_reclaim_pkt+0x5c/0x118 [dhd]
pc : [<bf278820>] lr : [<bf2965c4>] psr: 60030113
sp : df45bcf0 ip : 00000000 fp : 00000048
r10: 00000048 r9 : 00000000 r8 : 00000000
r7 : d6fe6100 r6 : d620000c r5 : bf3149e8 r4 : d94c39c0
r3 : d94c3a40 r2 : 00000000 r1 : d94c39c0 r0 : 00000000
Flags: nZCv IRQs on FIQs on Mode SVC_32 ISA ARM Segment kernel
Control: 10c5387d Table: 10fe404a DAC: 00000015
Process swapper/1 (pid: 0, stack limit = 0xdf45a210)
Stack: (0xdf45bcf0 to 0xdf45c000)
bce0: d629d300 d61f2000 d629df00 d620000c
bd00: 000003e0 000003e0 00000048 bf29da10 00000400 d620100c 0000be60 d976c040
bd20: 00000000 d620000c df45bdbc 000000b5 d976c088 00000048 00000048 bf27ffb8
bd40: 0000004a 00000000 d6200000 00000018 00000000 c035d87c 00000006 d70ce400
bd60: d620000c d620000c d70ce400 00000000 d368f500 00000000 d6200000 d368f624
bd80: 0000004a bf28efdc df45bdc8 df45bdc4 00000000 bf2986a8 00000000 bf3149e8
bda0: 00000001 d6203000 00000000 df45a000 00000000 d976c040 d368f500 00000000
bdc0: dfec3220 d976c088 18000200 b5000000 00000000 00000000 00000000 00000000
bde0: 00000000 6c770000 00000030 00000000 00000000 00000000 d6591338 d620000c
be00: 00000018 00000030 d6591338 0000000e d6591338 bf3149e8 d61f20dc bf2a1db4
be20: 00000000 d6591338 bf3149e8 bf29b310 bf3149e8 bf29b0a8 00000001 bf2f76b8
be40: d6591320 d61f2000 00000002 00000030 00000000 00010000 bf3149e8 d9340000
be60: d9340000 df45a000 c062b154 c0632080 40000006 bf29e6e8 00010000 bf3149e8
be80: d9340000 00000000 df45a000 bf2a3f78 bf3149e8 d9340000 00000000 00000038
bea0: df45beb4 d6200000 d6203690 00000000 df45bec0 bf284600 d620368c c00313e0
bec0: 00000000 00000006 df45a000 c0632098 c0632080 00000101 c0632080 c00315d8
bee0: df4e7d00 df4e7d64 df45bee0 c0660600 0000000a 000ccb35 c0632100 00200040
bf00: df4e7d00 c062c854 00000000 00000000 00000001 df405000 00000109 c0654444
bf20: c062e200 c0031970 c062c854 c0063cc8 e080400c c063271c df45bf60 e0804000
bf40: 10fc0f98 c00193e4 c02ed6c4 20030013 ffffffff df45bf94 10fc0f98 c00234c0
bf60: df45bfa8 00000018 10fc0f98 00000109 dfbdc910 00000001 10fa8f24 00000109
bf80: 10fc0f98 00000109 c0654444 c062e200 14000000 df45bfa8 fffffff8 c02ed6c4
bfa0: 20030013 ffffffff 10fc0f98 00000109 c062b220 df45a000 c06324b0 c04956f0
bfc0: dfbdc910 df45bfd8 c062b200 c0654444 c062e200 c005d230 c062d908 c065fc41
bfe0: c062e200 00000000 1f44004a c066007c 00000000 000194ac eaabeffb dce74e7c
[<bf278820>] (dhd_pkt_unlink [dhd]) from [<bf2965c4>] (dhd_prot_rxpost_reclaim_pkt+0x5c/0x118 [dhd])
[<bf2965c4>] (dhd_prot_rxpost_reclaim_pkt [dhd]) from [<bf29da10>] (dhd_prot_process_hwa_event+0x254/0x344 [dhd])
[<bf29da10>] (dhd_prot_process_hwa_event [dhd]) from [<bf27ffb8>] (wl_host_event+0x1b4/0x6c0 [dhd])
[<bf27ffb8>] (wl_host_event [dhd]) from [<bf28efdc>] (dhd_rx_frame+0xba4/0xea4 [dhd])
[<bf28efdc>] (dhd_rx_frame [dhd]) from [<bf2a1db4>] (dhd_bus_rx_frame+0x24/0x2c [dhd])
[<bf2a1db4>] (dhd_bus_rx_frame [dhd]) from [<bf29b310>] (dhd_prot_process_ctrlbuf+0xa8/0x25c [dhd])
[<bf29b310>] (dhd_prot_process_ctrlbuf [dhd]) from [<bf29e6e8>] (dhdpci_bus_read_frames+0x18/0xc8 [dhd])
[<bf29e6e8>] (dhdpci_bus_read_frames [dhd]) from [<bf2a3f78>] (dhd_bus_dpc+0xbc/0x384 [dhd])
[<bf2a3f78>] (dhd_bus_dpc [dhd]) from [<bf284600>] (dhd_dpc+0x1c/0x58 [dhd])
[<bf284600>] (dhd_dpc [dhd]) from [<c00313e0>] (tasklet_action+0x74/0x110)
[<c00313e0>] (tasklet_action) from [<c00315d8>] (__do_softirq+0x108/0x220)
[<c00315d8>] (__do_softirq) from [<c0031970>] (irq_exit+0x78/0xb0)
[<c0031970>] (irq_exit) from [<c0063cc8>] (__handle_domain_irq+0x60/0xb4)
[<c0063cc8>] (__handle_domain_irq) from [<c00193e4>] (gic_handle_irq+0x24/0x60)
[<c00193e4>] (gic_handle_irq) from [<c00234c0>] (__irq_svc+0x40/0x74)
Exception stack(0xdf45bf60 to 0xdf45bfa8)
bf60: df45bfa8 00000018 10fc0f98 00000109 dfbdc910 00000001 10fa8f24 00000109
bf80: 10fc0f98 00000109 c0654444 c062e200 14000000 df45bfa8 fffffff8 c02ed6c4
bfa0: 20030013 ffffffff
[<c00234c0>] (__irq_svc) from [<c02ed6c4>] (cpuidle_enter_state+0xd8/0x20c)
[<c02ed6c4>] (cpuidle_enter_state) from [<c005d230>] (cpu_startup_entry+0x204/0x264)
[<c005d230>] (cpu_startup_entry) from [<000194ac>] (0x194ac)
Code: 1a000007 e5912084 e2813080 e5910080 (e5820000)
---[ end trace 5e932127ecbec5b1 ]---
Kernel panic - not syncing: Fatal exception in interrupt
CPU0: stopping
CPU: 0 PID: 849 Comm: dhd_watchdog_th Tainted: P D 4.1.52 #1
Hardware name: Generic DT based system
[<c0026be0>] (unwind_backtrace) from [<c00229c8>] (show_stack+0x10/0x14)
[<c00229c8>] (show_stack) from [<c048ff98>] (dump_stack+0x8c/0xa0)
[<c048ff98>] (dump_stack) from [<c00253a4>] (handle_IPI+0x148/0x158)
[<c00253a4>] (handle_IPI) from [<c001941c>] (gic_handle_irq+0x5c/0x60)
[<c001941c>] (gic_handle_irq) from [<c00234c0>] (__irq_svc+0x40/0x74)
Exception stack(0xd6595ed0 to 0xd6595f18)
5ec0: bf31ac6c 00000000 0000c66b 0000c66a
5ee0: d61f2000 d620000c d6200000 d6203710 bf3149e8 c0632100 000ccb36 d620000c
5f00: 00000000 d6595f18 bf29adcc c049427c 20080013 ffffffff
[<c00234c0>] (__irq_svc) from [<c049427c>] (_raw_spin_lock_bh+0x48/0x5c)
[<c049427c>] (_raw_spin_lock_bh) from [<bf29adcc>] (dhd_msgbuf_watchdog+0x10/0x78 [dhd])
[<bf29adcc>] (dhd_msgbuf_watchdog [dhd]) from [<bf2a0674>] (dhd_bus_watchdog+0x64/0xac [dhd])
[<bf2a0674>] (dhd_bus_watchdog [dhd]) from [<bf2846f4>] (dhd_watchdog_thread+0xb8/0x158 [dhd])
[<bf2846f4>] (dhd_watchdog_thread [dhd]) from [<c0046dd8>] (kthread+0xdc/0xf4)
[<c0046dd8>] (kthread) from [<c001f4e8>] (ret_from_fork+0x14/0x2c)
CPU2: stopping
CPU: 2 PID: 0 Comm: swapper/2 Tainted: P D 4.1.52 #1
Hardware name: Generic DT based system
[<c0026be0>] (unwind_backtrace) from [<c00229c8>] (show_stack+0x10/0x14)
[<c00229c8>] (show_stack) from [<c048ff98>] (dump_stack+0x8c/0xa0)
[<c048ff98>] (dump_stack) from [<c00253a4>] (handle_IPI+0x148/0x158)
[<c00253a4>] (handle_IPI) from [<c001941c>] (gic_handle_irq+0x5c/0x60)
[<c001941c>] (gic_handle_irq) from [<c00234c0>] (__irq_svc+0x40/0x74)
Exception stack(0xdf45df60 to 0xdf45dfa8)
df60: df45dfa8 00000018 592f2174 00000109 dfbe7910 00000001 59261d2c 00000109
df80: 00090448 00000000 c0654444 c02ed7f0 14000000 df45dfa8 00000000 c02ed6d4
dfa0: 20000113 ffffffff
[<c00234c0>] (__irq_svc) from [<c02ed6d4>] (cpuidle_enter_state+0xe8/0x20c)
[<c02ed6d4>] (cpuidle_enter_state) from [<c005d230>] (cpu_startup_entry+0x204/0x264)
[<c005d230>] (cpu_startup_entry) from [<000194ac>] (0x194ac)
Rebooting in 5 seconds..
kerSysSoftReset: called on cpu 1