机器内核崩溃求助 kernel BUG at net/ipv4/inetpeer.c:382!

[Kathr1ne]

最近服务器会挂掉 然后磁盘直接损坏无法进入系统 只能进入单用户模式才修复文件系统后才能正常重启

各位大佬有排查思路吗？

下面贴 kdump 信息

      KERNEL: /usr/lib/debug/usr/lib/modules/3.10.0-514.el7.x86_64/vmlinux
    DUMPFILE: vmcore  [PARTIAL DUMP]
        CPUS: 2
        DATE: Fri Jul 26 14:52:41 2019
      UPTIME: 90 days, 16:58:02
LOAD AVERAGE: 0.25, 0.28, 0.31
       TASKS: 153
    NODENAME: ustack-taicangct55.ustack.qeeyou.com
     RELEASE: 3.10.0-514.el7.x86_64
     VERSION: #1 SMP Tue Nov 22 16:42:41 UTC 2016
     MACHINE: x86_64  (2194 Mhz)
      MEMORY: 4 GB
       PANIC: "kernel BUG at net/ipv4/inetpeer.c:382!"
         PID: 0
     COMMAND: "swapper/1"
        TASK: ffff880139b81f60  (1 of 2)  [THREAD_INFO: ffff880139b8c000]
         CPU: 1
       STATE: TASK_RUNNING (PANIC)


crash> bt
PID: 0      TASK: ffff880139b81f60  CPU: 1   COMMAND: "swapper/1"
 #0 [ffff88013fd03370] machine_kexec at ffffffff81059cdb
 #1 [ffff88013fd033d0] __crash_kexec at ffffffff81105182
 #2 [ffff88013fd034a0] crash_kexec at ffffffff81105270
 #3 [ffff88013fd034b8] oops_end at ffffffff8168ee88
 #4 [ffff88013fd034e0] die at ffffffff8102e93b
 #5 [ffff88013fd03510] do_trap at ffffffff8168e540
 #6 [ffff88013fd03560] do_invalid_op at ffffffff8102b144
 #7 [ffff88013fd03610] invalid_op at ffffffff81697e5e
    [exception RIP: inet_getpeer+1079]
    RIP: ffffffff815afe77  RSP: ffff88013fd036c0  RFLAGS: 00010287
    RAX: ffff880138055e40  RBX: 0000000000000001  RCX: 0000000000000001
    RDX: 0000000000000001  RSI: 00000000060606df  RDI: 00000000060606df
    RBP: ffff88013fd03850   R8: ffff880138055e50   R9: ffff88013fd036f8
    R10: ffff8800b4d22250  R11: 0000000000000000  R12: ffff8801395fa780
    R13: 0000000000000000  R14: ffff88013fd038e0  R15: ffff8800b4d22240
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
 #8 [ffff88013fd03738] xfrm_lookup at ffffffff8160f39b
 #9 [ffff88013fd037a8] icmp_route_lookup at ffffffff815e613d
#10 [ffff88013fd03888] icmp_send at ffffffff815e6760
#11 [ffff88013fd03a00] nf_send_unreach at ffffffffa032a282 [nf_reject_ipv4]
#12 [ffff88013fd03a20] reject_tg at ffffffffa032f0ca [ipt_REJECT]
#13 [ffff88013fd03a30] ipt_do_table at ffffffffa009fcb0 [ip_tables]
#14 [ffff88013fd03b78] iptable_filter_hook at ffffffffa02ec036 [iptable_filter]
#15 [ffff88013fd03b88] nf_iterate at ffffffff815a6460
#16 [ffff88013fd03bc8] nf_hook_slow at ffffffff815a6548
#17 [ffff88013fd03c00] ip_local_deliver at ffffffff815b0862
#18 [ffff88013fd03c58] ip_rcv_finish at ffffffff815b01aa
#19 [ffff88013fd03c80] ip_rcv at ffffffff815b0b36
#20 [ffff88013fd03ce8] __netif_receive_skb_core at ffffffff8156f582
#21 [ffff88013fd03d50] __netif_receive_skb at ffffffff8156f818
#22 [ffff88013fd03d70] netif_receive_skb_internal at ffffffff8156f8a0
#23 [ffff88013fd03da0] napi_gro_receive at ffffffff81570a28
#24 [ffff88013fd03dc8] virtnet_poll at ffffffffa008d9d5 [virtio_net]
#25 [ffff88013fd03e38] net_rx_action at ffffffff815700b0
#26 [ffff88013fd03eb8] __do_softirq at ffffffff8108f0ff
#27 [ffff88013fd03f28] call_softirq at ffffffff8169805c
#28 [ffff88013fd03f40] do_softirq at ffffffff8102d365
#29 [ffff88013fd03f60] irq_exit at ffffffff8108f495
#30 [ffff88013fd03f78] do_IRQ at ffffffff81698bf8
--- <IRQ stack> ---
#31 [ffff880139b8fde8] ret_from_intr at ffffffff8168dd6d
    [exception RIP: native_safe_halt+6]
    RIP: ffffffff81060eb6  RSP: ffff880139b8fe98  RFLAGS: 00000286
    RAX: 00000000ffffffed  RBX: ffff880139b825f8  RCX: 0100000000000000
    RDX: 0000000000000000  RSI: 0000000000000000  RDI: 0000000000000046
    RBP: ffff880139b8fe98   R8: 0000000000000000   R9: 0000000000000000
    R10: 0000000000000000  R11: 0000000000000000  R12: ffffffff81029569
    R13: ffff880139b8fe48  R14: ffffffff810cbcdc  R15: ffff880139b8fe10
    ORIG_RAX: ffffffffffffffdd  CS: 0010  SS: 0018
#32 [ffff880139b8fea0] default_idle at ffffffff8103483f
#33 [ffff880139b8fec0] arch_cpu_idle at ffffffff81035186
#34 [ffff880139b8fed0] cpu_startup_entry at ffffffff810e7c95
#35 [ffff880139b8ff28] start_secondary at ffffffff8104f12a



[7763789.273323] XFS (vda1): page discard on page ffffea00027cf840, inode 0x209215, offset 25886720.
[7763799.300905] XFS (vda1): Internal error XFS_WANT_CORRUPTED_RETURN at line 366 of file fs/xfs/libxfs/xfs_alloc.c.  Caller xfs_alloc_ag_vextent_near+0x570/0xab0 [xfs]
[7763799.313824] CPU: 1 PID: 13724 Comm: kworker/u4:2 Tainted: G           OE  ------------   3.10.0-514.el7.x86_64 #1
[7763799.313827] Hardware name: RDO OpenStack Compute, BIOS 1.10.2-3.el7_4.1 04/01/2014
[7763799.313838] Workqueue: writeback bdi_writeback_workfn (flush-253:0)
[7763799.313841]  ffff8800bbb79000 000000009410b28c ffff8800b4e33460 ffffffff81685fac
[7763799.313844]  ffff8800b4e33478 ffffffffa01e115b ffffffffa019eeb0 ffff8800b4e334d0
[7763799.313846]  ffffffffa019c764 4c0000b800000020 ffff880035d1d000 00000000a01a0b41
[7763799.313849] Call Trace:
[7763799.313856]  [<ffffffff81685fac>] dump_stack+0x19/0x1b
[7763799.313904]  [<ffffffffa01e115b>] xfs_error_report+0x3b/0x40 [xfs]
[7763799.313914]  [<ffffffffa019eeb0>] ? xfs_alloc_ag_vextent_near+0x570/0xab0 [xfs]
[7763799.313923]  [<ffffffffa019c764>] xfs_alloc_fixup_trees+0x2c4/0x370 [xfs]
[7763799.313932]  [<ffffffffa019eeb0>] xfs_alloc_ag_vextent_near+0x570/0xab0 [xfs]
[7763799.313951]  [<ffffffffa019f4fd>] xfs_alloc_ag_vextent+0x10d/0x150 [xfs]
[7763799.313960]  [<ffffffffa01a01c6>] xfs_alloc_vextent+0x446/0x5f0 [xfs]
[7763799.313970]  [<ffffffffa01b0663>] xfs_bmap_btalloc+0x3f3/0x780 [xfs]
[7763799.313991]  [<ffffffffa01b09fe>] xfs_bmap_alloc+0xe/0x10 [xfs]
[7763799.314002]  [<ffffffffa01b1439>] xfs_bmapi_write+0x499/0xab0 [xfs]
[7763799.314018]  [<ffffffffa01ecab7>] xfs_iomap_write_allocate+0x177/0x390 [xfs]
[7763799.314032]  [<ffffffffa01d68c6>] xfs_map_blocks+0x186/0x240 [xfs]
[7763799.314045]  [<ffffffffa01d7f03>] xfs_vm_writepage+0x193/0x5d0 [xfs]
[7763799.314051]  [<ffffffff8118b3b3>] __writepage+0x13/0x50
[7763799.314053]  [<ffffffff8118bed1>] write_cache_pages+0x251/0x4d0
[7763799.314055]  [<ffffffff8118b3a0>] ? global_dirtyable_memory+0x70/0x70
[7763799.314060]  [<ffffffff812efe59>] ? blk_flush_plug_list+0xc9/0x230
[7763799.314062]  [<ffffffff8118c19d>] generic_writepages+0x4d/0x80
[7763799.314074]  [<ffffffffa01d7063>] xfs_vm_writepages+0x53/0x90 [xfs]
[7763799.314077]  [<ffffffff8118d24e>] do_writepages+0x1e/0x40
[7763799.314079]  [<ffffffff81228730>] __writeback_single_inode+0x40/0x210
[7763799.314082]  [<ffffffff8122941e>] writeback_sb_inodes+0x25e/0x420
[7763799.314084]  [<ffffffff8122967f>] __writeback_inodes_wb+0x9f/0xd0
[7763799.314086]  [<ffffffff81229ec3>] wb_writeback+0x263/0x2f0
[7763799.314088]  [<ffffffff8118c8f0>] ? bdi_dirty_limit+0x40/0xe0
[7763799.314091]  [<ffffffff8122bdbc>] bdi_writeback_workfn+0x1cc/0x460
[7763799.314095]  [<ffffffff810a7f3b>] process_one_work+0x17b/0x470
[7763799.314097]  [<ffffffff810a8d76>] worker_thread+0x126/0x410
[7763799.314098]  [<ffffffff810a8c50>] ? rescuer_thread+0x460/0x460
[7763799.314103]  [<ffffffff810b052f>] kthread+0xcf/0xe0
[7763799.314106]  [<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
[7763799.314113]  [<ffffffff81696518>] ret_from_fork+0x58/0x90
[7763799.314115]  [<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
[7763799.314137] XFS (vda1): page discard on page ffffea00048ae1c0, inode 0x209215, offset 25890816.
[7836803.460242] ------------[ cut here ]------------
[7836803.469540] kernel BUG at net/ipv4/inetpeer.c:382!
[7836803.473564] invalid opcode: 0000 [#1] SMP 
[7836803.477139] Modules linked in: ip_set_hash_ipport xt_multiport xt_FULLCONENAT(OE) xt_mark ip_set_hash_net sch_sfq act_mirred cls_u32 sch_ingress sch_htb ifb ppp_async crc_ccitt ppp_generic slhc tun xfrm4_tunnel tunnel4 ipcomp xfrm_ipcomp esp4 ah4 af_key ip_gre ip_tunnel gre xt_REDIRECT nf_nat_redirect ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_set ip_set ipt_REJECT nf_reject_ipv4 xt_TCPMSS nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter iptable_mangle iptable_raw nfnetlink iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ppdev cryptd pcspkr virtio_balloon parport_pc parport i2c_piix4 ip_tables xfs libcrc32c virtio_console virtio_blk virtio_net ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel
[7836803.509040]  cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm serio_raw drm virtio_pci virtio_ring i2c_core virtio ata_piix floppy libata
[7836803.516803] CPU: 1 PID: 0 Comm: swapper/1 Tainted: G           OE  ------------   3.10.0-514.el7.x86_64 #1
[7836803.520704] Hardware name: RDO OpenStack Compute, BIOS 1.10.2-3.el7_4.1 04/01/2014
[7836803.524236] task: ffff880139b81f60 ti: ffff880139b8c000 task.ti: ffff880139b8c000
[7836803.527963] RIP: 0010:[<ffffffff815afe77>]  [<ffffffff815afe77>] inet_getpeer.part.3+0x437/0x440
[7836803.531743] RSP: 0018:ffff88013fd036c0  EFLAGS: 00010287
[7836803.535390] RAX: ffff880138055e40 RBX: 0000000000000001 RCX: 0000000000000001
[7836803.539277] RDX: 0000000000000001 RSI: 00000000060606df RDI: 00000000060606df
[7836803.543293] RBP: ffff88013fd03850 R08: ffff880138055e50 R09: ffff88013fd036f8
[7836803.547342] R10: ffff8800b4d22250 R11: 0000000000000000 R12: ffff8801395fa780
[7836803.551371] R13: 0000000000000000 R14: ffff88013fd038e0 R15: ffff8800b4d22240
[7836803.555256] FS:  0000000000000000(0000) GS:ffff88013fd00000(0000) knlGS:0000000000000000
[7836803.559316] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[7836803.562981] CR2: 00007efe2c6d0000 CR3: 00000000b2ae7000 CR4: 00000000003406e0
[7836803.566692] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[7836803.570206] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[7836803.573753] Stack:
[7836803.576108]  ffff8801395fa78c 000077fec02fc927 0000000181a9cd80 ffff88013fd03718
[7836803.580009]  ffff8801395fa780 ffff8800879ff0c0 ffff8800a13cb900 ffff880138055e48
[7836803.583751]  ffff8800b4d22248 ffff880087a146c0 ffff880137be3cc8 ffff88013fd037c0
[7836803.587139] Call Trace:
[7836803.589651]  <IRQ> 
[7836803.590179] 
[7836803.592561]  [<ffffffff8160f39b>] ? xfrm_lookup+0x30b/0x3f0
[7836803.595548]  [<ffffffff815e613d>] ? icmp_route_lookup.isra.15+0x2ed/0x450
[7836803.598860]  [<ffffffff815aff79>] inet_getpeer+0xf9/0x170
[7836803.602460]  [<ffffffff815e6760>] icmp_send+0x4c0/0x510
[7836803.606946]  [<ffffffff815fd976>] ? fib4_rule_action+0x66/0x70
[7836803.611019]  [<ffffffff8158ea1d>] ? fib_rules_lookup+0x10d/0x1c0
[7836803.614040]  [<ffffffffa032a282>] nf_send_unreach+0xa2/0xc0 [nf_reject_ipv4]
[7836803.617604]  [<ffffffffa032f0ca>] reject_tg+0x6a/0xa0 [ipt_REJECT]
[7836803.620901]  [<ffffffffa009fcb0>] ipt_do_table+0x2e0/0x710 [ip_tables]
[7836803.624164]  [<ffffffffa02ec036>] iptable_filter_hook+0x36/0x80 [iptable_filter]
[7836803.627421]  [<ffffffff815a6460>] nf_iterate+0x70/0xb0
[7836803.630297]  [<ffffffff815a6548>] nf_hook_slow+0xa8/0x110
[7836803.633082]  [<ffffffff815b0862>] ip_local_deliver+0xb2/0xd0
[7836803.635622]  [<ffffffff815b0470>] ? ip_rcv_finish+0x350/0x350
[7836803.638377]  [<ffffffff815b01aa>] ip_rcv_finish+0x8a/0x350
[7836803.641076]  [<ffffffff815b0b36>] ip_rcv+0x2b6/0x410
[7836803.643314]  [<ffffffff815b0120>] ? inet_del_offload+0x40/0x40
[7836803.645869]  [<ffffffff8156f582>] __netif_receive_skb_core+0x582/0x800
[7836803.648532]  [<ffffffff8156f818>] __netif_receive_skb+0x18/0x60
[7836803.650987]  [<ffffffff8156f8a0>] netif_receive_skb_internal+0x40/0xc0
[7836803.653469]  [<ffffffff81570a28>] napi_gro_receive+0xd8/0x130
[7836803.655899]  [<ffffffffa008d9d5>] virtnet_poll+0x265/0x750 [virtio_net]
[7836803.658469]  [<ffffffff815700b0>] net_rx_action+0x170/0x380
[7836803.661079]  [<ffffffff8108f0ff>] __do_softirq+0xef/0x280
[7836803.663320]  [<ffffffff8169805c>] call_softirq+0x1c/0x30
[7836803.665441]  [<ffffffff8102d365>] do_softirq+0x65/0xa0
[7836803.667409]  [<ffffffff8108f495>] irq_exit+0x115/0x120
[7836803.669445]  [<ffffffff81698bf8>] do_IRQ+0x58/0xf0
[7836803.671524]  [<ffffffff8168dd6d>] common_interrupt+0x6d/0x6d
[7836803.673487]  <EOI> 
[7836803.673861] 
[7836803.675162]  [<ffffffff81060eb6>] ? native_safe_halt+0x6/0x10
[7836803.677111]  [<ffffffff8103483f>] default_idle+0x1f/0xc0
[7836803.679039]  [<ffffffff81035186>] arch_cpu_idle+0x26/0x30
[7836803.680913]  [<ffffffff810e7c95>] cpu_startup_entry+0x245/0x290
[7836803.682963]  [<ffffffff8104f12a>] start_secondary+0x1ba/0x230
[7836803.685076] Code: 90 fe ff ff 4c 89 38 e8 a8 fa ff ff 41 83 44 24 14 01 e9 bb fc ff ff 0f 1f 44 00 00 4c 89 fa e9 92 fe ff ff 0f 0b e8 49 57 ad ff <0f> 0b 0f 1f 80 00 00 00 00 0f 1f 44 00 00 55 48 c7 c0 e0 0e f9 
[7836803.693794] RIP  [<ffffffff815afe77>] inet_getpeer.part.3+0x437/0x440
[7836803.696787]  RSP <ffff88013fd036c0>

[defunct9]

回退到 ext4 文件系统

[stern123]

[7836803.473564] invalid opcode: 0000 [#1] SMP
看到这一条
要不要检查下内存？

[Kathr1ne]

补充一下： 我们的机器都是用的云服务器