From mboxrd@z Thu Jan 1 00:00:00 1970 From: "Srivatsa S. Bhat" Subject: Re: mod_timer: list_add corruption: WARNING: CPU: 1 PID: 0 at lib/list_debug.c:33 __list_add+0xbe/0xd0() Date: Fri, 19 Jul 2013 17:17:53 +0530 Message-ID: <51E92769.7090106@linux.vnet.ibm.com> References: <51E557C0.2060602@linux.vnet.ibm.com> Mime-Version: 1.0 Content-Type: text/plain; charset=ISO-8859-1 Content-Transfer-Encoding: 7bit Cc: "linux-kernel@vger.kernel.org" , Frederic Weisbecker , Ingo Molnar , "Paul E. McKenney" , netdev@vger.kernel.org To: Thomas Gleixner Return-path: In-Reply-To: Sender: linux-kernel-owner@vger.kernel.org List-Id: netdev.vger.kernel.org On 07/19/2013 04:55 PM, Thomas Gleixner wrote: > On Tue, 16 Jul 2013, Srivatsa S. Bhat wrote: >> ------------[ cut here ]------------ >> WARNING: CPU: 1 PID: 0 at lib/list_debug.c:33 __list_add+0xbe/0xd0() >> list_add corruption. prev->next should be next (ffff8810396b5568), but was (null). (prev=ffff88102c1344c0). > > Can you please enable debugobjects? > Sure Thomas, please find the new traces below, with debug objects enabled. Regards, Srivatsa S. Bhat ------------[ cut here ]------------ WARNING: CPU: 1 PID: 0 at lib/debugobjects.c:260 debug_print_object+0x8e/0xb0() ODEBUG: init active (active state 0) object type: timer_list hint: br_multicast_group_expired+0x0/0x110 [bridge] Modules linked in: ebtable_nat ebtables nfs fscache nf_conntrack_ipv4 nf_defrag_ipv4 ipt_REJECT xt_CHECKSUM iptable_mangle iptable_filter ip_tables nfsd lockd nfs_acl exportfs auth_rpcgss autofs4 sunrpc 8021q garp bridge stp llc ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 xt_state nf_conntrack ip6table_filter ip6_tables ipv6 vfat fat vhost_net macvtap macvlan vhost tun kvm_intel kvm uinput iTCO_wdt iTCO_vendor_support cdc_ether usbnet mii microcode i2c_i801 i2c_core lpc_ich mfd_core shpchp mlx4_core ioatdma dca be2net wmi acpi_cpufreq mperf ext4 jbd2 mbcache dm_mirror dm_region_hash dm_log dm_mod CPU: 1 PID: 0 Comm: swapper/1 Not tainted 3.11.0-rc1-dbg-a #16 Hardware name: IBM -[8737R2A]-/00Y2738, BIOS -[B2E120RUS-1.20]- 11/30/2012 0000000000000104 ffff88103fc43908 ffffffff8161b3c9 ffffffff81a14151 ffff88103fc43958 ffff88103fc43948 ffffffff8104e6bc 0000000000000000 ffffffff81a2cfae ffff88101b21cfd8 ffffffff81c37ba0 ffffffff828f2ca8 Call Trace: [] dump_stack+0x59/0x80 [] warn_slowpath_common+0x8c/0xc0 [] warn_slowpath_fmt+0x46/0x50 [] debug_print_object+0x8e/0xb0 [] ? br_multicast_free_pg+0x20/0x20 [bridge] [] ? __debug_object_init+0x42/0x3f0 [] __debug_object_init+0x21f/0x3f0 [] debug_object_init+0x1f/0x30 [] init_timer_key+0x39/0x100 [] br_ip4_multicast_query+0x155/0x380 [bridge] [] br_multicast_ipv4_rcv+0x2cf/0x3d0 [bridge] [] ? _raw_spin_unlock+0x2b/0x50 [] ? br_fdb_update+0x1db/0x2b0 [bridge] [] br_multicast_rcv+0x45/0x60 [bridge] [] br_handle_frame_finish+0x16e/0x3c0 [bridge] [] br_handle_frame+0x238/0x400 [bridge] [] ? br_del_bridge+0x80/0x80 [bridge] [] __netif_receive_skb_core+0x237/0x960 [] ? __netif_receive_skb_core+0x6e/0x960 [] __netif_receive_skb+0x27/0x70 [] netif_receive_skb+0x2d/0x210 [] ? __netdev_alloc_skb+0xa5/0x110 [] be_rx_compl_process+0xef/0x140 [be2net] [] be_process_rx+0xe2/0x1a0 [be2net] [] be_poll+0x13d/0x1d0 [be2net] [] net_rx_action+0xd8/0x2a0 [] __do_softirq+0x149/0x400 [] irq_exit+0xed/0x100 [] do_IRQ+0x66/0xe0 [] common_interrupt+0x6f/0x6f [] ? cpuidle_enter_state+0x5b/0xe0 [] ? cpuidle_enter_state+0x57/0xe0 [] ? notifier_call_chain+0x120/0x120 [] cpuidle_idle_call+0xcf/0x160 [] arch_cpu_idle+0xe/0x30 [] cpu_idle_loop+0x81/0x3a0 [] cpu_startup_entry+0x70/0x80 [] start_secondary+0xdc/0xe0 ---[ end trace 68b46b9f74e14585 ]--- ------------[ cut here ]------------ WARNING: CPU: 1 PID: 0 at lib/debugobjects.c:260 debug_print_object+0x8e/0xb0() ODEBUG: init active (active state 0) object type: timer_list hint: br_multicast_group_expired+0x0/0x110 [bridge] Modules linked in: ebtable_nat ebtables nfs fscache nf_conntrack_ipv4 nf_defrag_ipv4 ipt_REJECT xt_CHECKSUM iptable_mangle iptable_filter ip_tables nfsd lockd nfs_acl exportfs auth_rpcgss autofs4 sunrpc 8021q garp bridge stp llc ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 xt_state nf_conntrack ip6table_filter ip6_tables ipv6 vfat fat vhost_net macvtap macvlan vhost tun kvm_intel kvm uinput iTCO_wdt iTCO_vendor_support cdc_ether usbnet mii microcode i2c_i801 i2c_core lpc_ich mfd_core shpchp mlx4_core ioatdma dca be2net wmi acpi_cpufreq mperf ext4 jbd2 mbcache dm_mirror dm_region_hash dm_log dm_mod CPU: 1 PID: 0 Comm: swapper/1 Tainted: G W 3.11.0-rc1-dbg-a #16 Hardware name: IBM -[8737R2A]-/00Y2738, BIOS -[B2E120RUS-1.20]- 11/30/2012 0000000000000104 ffff88103fc43908 ffffffff8161b3c9 ffffffff81a14151 ffff88103fc43958 ffff88103fc43948 ffffffff8104e6bc 0000000000000000 ffffffff81a2cfae ffff88101b21cfd8 ffffffff81c37ba0 ffffffff828f2ca8 Call Trace: [] dump_stack+0x59/0x80 [] warn_slowpath_common+0x8c/0xc0 [] warn_slowpath_fmt+0x46/0x50 [] debug_print_object+0x8e/0xb0 [] ? br_multicast_free_pg+0x20/0x20 [bridge] [] ? __debug_object_init+0x42/0x3f0 [] __debug_object_init+0x21f/0x3f0 [] debug_object_init+0x1f/0x30 [] init_timer_key+0x39/0x100 [] br_ip4_multicast_query+0x155/0x380 [bridge] [] br_multicast_ipv4_rcv+0x2cf/0x3d0 [bridge] [] ? nf_hook_slow+0x16e/0x2a0 [] ? br_fdb_update+0x147/0x2b0 [bridge] [] br_multicast_rcv+0x45/0x60 [bridge] [] br_handle_frame_finish+0x16e/0x3c0 [bridge] [] br_handle_frame+0x238/0x400 [bridge] [] ? br_del_bridge+0x80/0x80 [bridge] [] __netif_receive_skb_core+0x237/0x960 [] ? __netif_receive_skb_core+0x6e/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] __netif_receive_skb+0x27/0x70 [] netif_receive_skb+0x2d/0x210 [] ? __netdev_alloc_skb+0xa5/0x110 [] be_rx_compl_process+0xef/0x140 [be2net] [] be_process_rx+0xe2/0x1a0 [be2net] [] be_poll+0x13d/0x1d0 [be2net] [] net_rx_action+0xd8/0x2a0 [] __do_softirq+0x149/0x400 [] irq_exit+0xed/0x100 [] do_IRQ+0x66/0xe0 [] common_interrupt+0x6f/0x6f [] ? cpuidle_enter_state+0x5b/0xe0 [] ? cpuidle_enter_state+0x57/0xe0 [] ? notifier_call_chain+0x120/0x120 [] cpuidle_idle_call+0xcf/0x160 [] arch_cpu_idle+0xe/0x30 [] cpu_idle_loop+0x81/0x3a0 [] cpu_startup_entry+0x70/0x80 [] start_secondary+0xdc/0xe0 ---[ end trace 68b46b9f74e14586 ]--- ------------[ cut here ]------------ WARNING: CPU: 1 PID: 0 at lib/debugobjects.c:260 debug_print_object+0x8e/0xb0() ODEBUG: init active (active state 0) object type: timer_list hint: br_multicast_group_expired+0x0/0x110 [bridge] Modules linked in: ebtable_nat ebtables nfs fscache nf_conntrack_ipv4 nf_defrag_ipv4 ipt_REJECT xt_CHECKSUM iptable_mangle iptable_filter ip_tables nfsd lockd nfs_acl exportfs auth_rpcgss autofs4 sunrpc 8021q garp bridge stp llc ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 xt_state nf_conntrack ip6table_filter ip6_tables ipv6 vfat fat vhost_net macvtap macvlan vhost tun kvm_intel kvm uinput iTCO_wdt iTCO_vendor_support cdc_ether usbnet mii microcode i2c_i801 i2c_core lpc_ich mfd_core shpchp mlx4_core ioatdma dca be2net wmi acpi_cpufreq mperf ext4 jbd2 mbcache dm_mirror dm_region_hash dm_log dm_mod CPU: 1 PID: 0 Comm: swapper/1 Tainted: G W 3.11.0-rc1-dbg-a #16 Hardware name: IBM -[8737R2A]-/00Y2738, BIOS -[B2E120RUS-1.20]- 11/30/2012 0000000000000104 ffff88103fc43908 ffffffff8161b3c9 ffffffff81a14151 ffff88103fc43958 ffff88103fc43948 ffffffff8104e6bc 0000000000000000 ffffffff81a2cfae ffff88101b21cfd8 ffffffff81c37ba0 ffffffff828f2ca8 Call Trace: [] dump_stack+0x59/0x80 [] warn_slowpath_common+0x8c/0xc0 [] warn_slowpath_fmt+0x46/0x50 [] debug_print_object+0x8e/0xb0 [] ? br_multicast_free_pg+0x20/0x20 [bridge] [] ? __debug_object_init+0x42/0x3f0 [] __debug_object_init+0x21f/0x3f0 [] ? trace_hardirqs_on+0xd/0x10 [] debug_object_init+0x1f/0x30 [] init_timer_key+0x39/0x100 [] br_ip4_multicast_query+0x155/0x380 [bridge] [] br_multicast_ipv4_rcv+0x2cf/0x3d0 [bridge] [] ? nf_hook_slow+0x16e/0x2a0 [] ? br_fdb_update+0xe4/0x2b0 [bridge] [] br_multicast_rcv+0x45/0x60 [bridge] [] br_handle_frame_finish+0x16e/0x3c0 [bridge] [] br_handle_frame+0x238/0x400 [bridge] [] ? br_del_bridge+0x80/0x80 [bridge] [] __netif_receive_skb_core+0x237/0x960 [] ? __netif_receive_skb_core+0x6e/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] __netif_receive_skb+0x27/0x70 [] netif_receive_skb+0x2d/0x210 [] ? __netdev_alloc_skb+0xa5/0x110 [] be_rx_compl_process+0xef/0x140 [be2net] [] be_process_rx+0xe2/0x1a0 [be2net] [] be_poll+0x13d/0x1d0 [be2net] [] net_rx_action+0xd8/0x2a0 [] __do_softirq+0x149/0x400 [] irq_exit+0xed/0x100 [] do_IRQ+0x66/0xe0 [] common_interrupt+0x6f/0x6f [] ? cpuidle_enter_state+0x5b/0xe0 [] ? cpuidle_enter_state+0x57/0xe0 [] ? notifier_call_chain+0x120/0x120 [] cpuidle_idle_call+0xcf/0x160 [] arch_cpu_idle+0xe/0x30 [] cpu_idle_loop+0x81/0x3a0 [] cpu_startup_entry+0x70/0x80 [] start_secondary+0xdc/0xe0 ---[ end trace 68b46b9f74e14587 ]--- ------------[ cut here ]------------ WARNING: CPU: 1 PID: 0 at lib/debugobjects.c:260 debug_print_object+0x8e/0xb0() ODEBUG: init active (active state 0) object type: timer_list hint: br_multicast_group_expired+0x0/0x110 [bridge] Modules linked in: ebtable_nat ebtables nfs fscache nf_conntrack_ipv4 nf_defrag_ipv4 ipt_REJECT xt_CHECKSUM iptable_mangle iptable_filter ip_tables nfsd lockd nfs_acl exportfs auth_rpcgss autofs4 sunrpc 8021q garp bridge stp llc ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 xt_state nf_conntrack ip6table_filter ip6_tables ipv6 vfat fat vhost_net macvtap macvlan vhost tun kvm_intel kvm uinput iTCO_wdt iTCO_vendor_support cdc_ether usbnet mii microcode i2c_i801 i2c_core lpc_ich mfd_core shpchp mlx4_core ioatdma dca be2net wmi acpi_cpufreq mperf ext4 jbd2 mbcache dm_mirror dm_region_hash dm_log dm_mod CPU: 1 PID: 0 Comm: swapper/1 Tainted: G W 3.11.0-rc1-dbg-a #16 Hardware name: IBM -[8737R2A]-/00Y2738, BIOS -[B2E120RUS-1.20]- 11/30/2012 0000000000000104 ffff88103fc43908 ffffffff8161b3c9 ffffffff81a14151 ffff88103fc43958 ffff88103fc43948 ffffffff8104e6bc 0000000000000000 ffffffff81a2cfae ffff88101b21cfd8 ffffffff81c37ba0 ffffffff828f2ca8 Call Trace: [] dump_stack+0x59/0x80 [] warn_slowpath_common+0x8c/0xc0 [] warn_slowpath_fmt+0x46/0x50 [] debug_print_object+0x8e/0xb0 [] ? br_multicast_free_pg+0x20/0x20 [bridge] [] ? __debug_object_init+0x42/0x3f0 [] __debug_object_init+0x21f/0x3f0 [] ? trace_hardirqs_on+0xd/0x10 [] debug_object_init+0x1f/0x30 [] init_timer_key+0x39/0x100 [] br_ip4_multicast_query+0x155/0x380 [bridge] [] br_multicast_ipv4_rcv+0x2cf/0x3d0 [bridge] [] ? nf_hook_slow+0x16e/0x2a0 [] ? br_fdb_update+0xe4/0x2b0 [bridge] [] br_multicast_rcv+0x45/0x60 [bridge] [] br_handle_frame_finish+0x16e/0x3c0 [bridge] [] br_handle_frame+0x238/0x400 [bridge] [] ? br_del_bridge+0x80/0x80 [bridge] [] __netif_receive_skb_core+0x237/0x960 [] ? __netif_receive_skb_core+0x6e/0x960 [] __netif_receive_skb+0x27/0x70 [] netif_receive_skb+0x2d/0x210 [] ? __netdev_alloc_skb+0xa5/0x110 [] be_rx_compl_process+0xef/0x140 [be2net] [] be_process_rx+0xe2/0x1a0 [be2net] [] be_poll+0x13d/0x1d0 [be2net] [] net_rx_action+0xd8/0x2a0 [] __do_softirq+0x149/0x400 [] irq_exit+0xed/0x100 [] do_IRQ+0x66/0xe0 [] common_interrupt+0x6f/0x6f [] ? cpuidle_enter_state+0x5b/0xe0 [] ? cpuidle_enter_state+0x57/0xe0 [] ? notifier_call_chain+0x120/0x120 [] cpuidle_idle_call+0xcf/0x160 [] arch_cpu_idle+0xe/0x30 [] cpu_idle_loop+0x81/0x3a0 [] cpu_startup_entry+0x70/0x80 [] start_secondary+0xdc/0xe0 ---[ end trace 68b46b9f74e14588 ]---