Hi,

After hitting the pvclock-related issue, I recompiled my 2.6.27-rc6
kernel without CONFIG_KVM_CLOCK.  It stays up far longer, but I see
the following guest crash when I stress it (with a source build):

  BUG: unable to handle kernel paging request at d97b8000
  IP: [<c0496f64>] __slab_alloc+0x1cd/0x3a1
  Oops: 0002 [#1] SMP DEBUG_PAGEALLOC
  Modules linked in: ipt_MASQUERADE iptable_nat nf_nat nf_conntrack_ipv4 ipt_REJECT iptable_filter ip_tables bridge stp ib_iser rdma_cm ib_cm iw_cm ib_sa ib_mad ib_core ib_addr iscsi_tcp libiscsi scsi_transport_iscsi nfs lockd nfs_acl sunrpc ip6t_REJECT xt_tcpudp nf_conntrack_ipv6 xt_state nf_conntrack ip6table_filter ip6_tables x_tables ipv6 dm_mirror dm_log dm_multipath dm_mod virtio_net floppy pcspkr virtio_pci i2c_piix4 i2c_core sr_mod cdrom ata_piix pata_acpi ata_generic ext3 jbd mbcache [last unloaded: microcode]
  
  Pid: 3020, comm: sed Not tainted (2.6.27-rc6 #1)
  EIP: 0060:[<c0496f64>] EFLAGS: 00210006 CPU: 0
  EIP is at __slab_alloc+0x1cd/0x3a1
  EAX: 5a5a5a5a EBX: 00000009 ECX: 00000800 EDX: 00002000
  ESI: c15f9f20 EDI: d97b8000 EBP: c096ed0c ESP: c096ece0
   DS: 007b ES: 007b FS: 00d8 GS: 0000 SS: 0068
  Process sed (pid: 3020, ti=c096e000 task=d93d14e0 task.ti=d9024000)
  Stack: d97b8000 c096ecec ffffffff 00000020 df83cbe0 00000000 d93d14e0 00000020 
         00200292 000000b8 00000000 c096ed3c c049738d c06196a1 c18abd58 00000000 
         c06196a1 00000020 df83cbe0 00200286 000005fa 00000020 d8ed4700 c096ed60 
  Call Trace:
   [<c049738d>] ? kmem_cache_alloc+0x63/0xd1
   [<c06196a1>] ? __alloc_skb+0x2e/0x10c
   [<c06196a1>] ? __alloc_skb+0x2e/0x10c
   [<c06196a1>] ? __alloc_skb+0x2e/0x10c
   [<c061979b>] ? __netdev_alloc_skb+0x1c/0x39
   [<e08572ac>] ? try_fill_recv+0x37/0x153 [virtio_net]
   [<c044ab9a>] ? lock_release_holdtime+0x43/0x48
   [<e08579b4>] ? virtnet_poll+0x239/0x2f7 [virtio_net]
   [<c06209a5>] ? net_rx_action+0xde/0x204
   [<c0432f93>] ? __do_softirq+0x89/0xf1
   [<c0432f0a>] ? __do_softirq+0x0/0xf1
   [<c0406e51>] ? do_softirq+0x7e/0xdf
   [<c046b8e2>] ? handle_fasteoi_irq+0x0/0xbd
   [<c0432e6c>] ? irq_exit+0x4c/0x8b
   [<c0406f57>] ? do_IRQ+0xa5/0xbe
   [<c040558c>] ? common_interrupt+0x28/0x30
   [<c0499b45>] ? css_put+0x20/0x23
   [<c0499bfa>] ? __mem_cgroup_uncharge_common+0xb2/0xce
   [<c0499d4d>] ? mem_cgroup_uncharge_page+0x12/0x14
   [<c048aa06>] ? page_remove_rmap+0xe4/0xfd
   [<c0484a8b>] ? unmap_vmas+0x36d/0x50d
   [<c0487d23>] ? exit_mmap+0x57/0xa2
   [<c042c60f>] ? mmput+0x3f/0x90
   [<c043012d>] ? exit_mm+0xed/0xf5
   [<c0431375>] ? do_exit+0x1cc/0x734
   [<c044354b>] ? up_read+0x1b/0x2e
   [<c0431963>] ? sys_exit_group+0x0/0x16
   [<c0431977>] ? sys_exit_group+0x14/0x16
   [<c0404b96>] ? syscall_call+0x7/0xb
   =======================
  Code: fe ff 89 45 d4 8b 45 e4 f6 40 01 08 74 2c 89 f0 e8 54 e5 ff ff ba 00 10 00 00 8b 7d d4 89 c1 b8 5a 5a 5a 5a d3 e2 89 d1 c1 e9 02 <f3> ab f6 c2 02 74 02 66 ab f6 c2 01 74 01 aa 8b 5d d4 89 5d f0 
  EIP: [<c0496f64>] __slab_alloc+0x1cd/0x3a1 SS:ESP 0068:c096ece0
  Kernel panic - not syncing: Fatal exception in interrupt
  ------------[ cut here ]------------
  WARNING: at kernel/smp.c:332 smp_call_function_mask+0x38/0x18a()
  Modules linked in: ipt_MASQUERADE iptable_nat nf_nat nf_conntrack_ipv4 ipt_REJECT iptable_filter ip_tables bridge stp ib_iser rdma_cm ib_cm iw_cm ib_sa ib_mad ib_core ib_addr iscsi_tcp libiscsi scsi_transport_iscsi nfs lockd nfs_acl sunrpc ip6t_REJECT xt_tcpudp nf_conntrack_ipv6 xt_state nf_conntrack ip6table_filter ip6_tables x_tables ipv6 dm_mirror dm_log dm_multipath dm_mod virtio_net floppy pcspkr virtio_pci i2c_piix4 i2c_core sr_mod cdrom ata_piix pata_acpi ata_generic ext3 jbd mbcache [last unloaded: microcode]
  Pid: 3020, comm: sed Tainted: G      D   2.6.27-rc6 #1
   [<c042e716>] warn_on_slowpath+0x46/0x6a
   [<c0518b58>] ? __delay+0xe/0x10
   [<c04757e0>] ? time_hardirqs_off+0xe/0x1f
   [<c044b056>] ? trace_hardirqs_off_caller+0x15/0x97
   [<c044b0e3>] ? trace_hardirqs_off+0xb/0xd
   [<c04757e0>] ? time_hardirqs_off+0xe/0x1f
   [<c04757e0>] ? time_hardirqs_off+0xe/0x1f
   [<c044b056>] ? trace_hardirqs_off_caller+0x15/0x97
   [<c044b0e3>] ? trace_hardirqs_off+0xb/0xd
   [<c042ecff>] ? release_console_sem+0x1be/0x1c6
   [<c0451826>] smp_call_function_mask+0x38/0x18a
   [<c041512a>] ? stop_this_cpu+0x0/0x4f
   [<c04757e0>] ? time_hardirqs_off+0xe/0x1f
   [<c044b056>] ? trace_hardirqs_off_caller+0x15/0x97
   [<c044b0e3>] ? trace_hardirqs_off+0xb/0xd
   [<c0692018>] ? _spin_unlock_irqrestore+0x3e/0x55
   [<c04152d2>] ? native_smp_send_stop+0x4/0x6e
   [<c04714f1>] ? ftrace_record_ip+0x1b3/0x1cc
   [<c0692018>] ? _spin_unlock_irqrestore+0x3e/0x55
   [<c045198f>] smp_call_function+0x17/0x19
   [<c04152ee>] native_smp_send_stop+0x20/0x6e
   [<c042e62f>] panic+0x53/0xf4
   [<c06924e1>] oops_end+0x87/0x9b
   [<c040608e>] die+0x5c/0x64
   [<c0693ec9>] do_page_fault+0x4f5/0x5af
   [<c06939d4>] ? do_page_fault+0x0/0x5af
   [<c06922b2>] error_code+0x72/0x78
   [<c048007b>] ? shrink_page_list+0x3ba/0x560
   [<c0496f64>] ? __slab_alloc+0x1cd/0x3a1
   [<c049738d>] kmem_cache_alloc+0x63/0xd1
   [<c06196a1>] ? __alloc_skb+0x2e/0x10c
   [<c06196a1>] ? __alloc_skb+0x2e/0x10c
   [<c06196a1>] __alloc_skb+0x2e/0x10c
   [<c061979b>] __netdev_alloc_skb+0x1c/0x39
   [<e08572ac>] try_fill_recv+0x37/0x153 [virtio_net]
   [<c044ab9a>] ? lock_release_holdtime+0x43/0x48
   [<e08579b4>] virtnet_poll+0x239/0x2f7 [virtio_net]
   [<c06209a5>] net_rx_action+0xde/0x204
   [<c0432f93>] __do_softirq+0x89/0xf1
   [<c0432f0a>] ? __do_softirq+0x0/0xf1
   [<c0406e51>] do_softirq+0x7e/0xdf
   [<c046b8e2>] ? handle_fasteoi_irq+0x0/0xbd
   [<c0432e6c>] irq_exit+0x4c/0x8b
   [<c0406f57>] do_IRQ+0xa5/0xbe
   [<c040558c>] common_interrupt+0x28/0x30
   [<c0499b45>] ? css_put+0x20/0x23
   [<c0499bfa>] __mem_cgroup_uncharge_common+0xb2/0xce
   [<c0499d4d>] mem_cgroup_uncharge_page+0x12/0x14
   [<c048aa06>] page_remove_rmap+0xe4/0xfd
   [<c0484a8b>] unmap_vmas+0x36d/0x50d
   [<c0487d23>] exit_mmap+0x57/0xa2
   [<c042c60f>] mmput+0x3f/0x90
   [<c043012d>] exit_mm+0xed/0xf5
   [<c0431375>] do_exit+0x1cc/0x734
   [<c044354b>] ? up_read+0x1b/0x2e
   [<c0431963>] sys_exit_group+0x0/0x16
   [<c0431977>] sys_exit_group+0x14/0x16
   [<c0404b96>] syscall_call+0x7/0xb
   =======================
  ---[ end trace 443e075b33442f93 ]---

I initially thought this might be virtio-related, because the crash
ended up near virtio_net.  To be sure, I tried again without the
virtio NIC, and got a different oops which may be more indicative of
the root cause:

  =========================
  [ BUG: held lock freed! ]
  -------------------------
  init/1 is freeing memory d6193000-d6193fff, with a lock still held there!
   (&anon_vma->lock){--..}, at: [<c048a69e>] page_lock_anon_vma+0x3e/0x5d
  3 locks held by init/1:
   #0:  (&mm->mmap_sem){----}, at: [<c0693ba0>] do_page_fault+0x1cc/0x5af
   #1:  (rcu_read_lock){..--}, at: [<c048a660>] page_lock_anon_vma+0x0/0x5d
   #2:  (&anon_vma->lock){--..}, at: [<c048a69e>] page_lock_anon_vma+0x3e/0x5d
  
  stack backtrace:
  Pid: 1, comm: init Not tainted 2.6.27-rc6 #1
   [<c044c96d>] debug_check_no_locks_freed+0xea/0x13d
   [<c047b23f>] free_hot_cold_page+0x56/0x14a
   [<c047b381>] free_hot_page+0xf/0x11
   [<c047b4d7>] __free_pages+0x2a/0x35
   [<c04969d9>] __free_slab+0xa3/0xab
   [<c0496b3b>] rcu_free_slab+0x13/0x15
   [<c046c97e>] __rcu_process_callbacks+0x112/0x177
   [<c046ca03>] rcu_process_callbacks+0x20/0x3a
   [<c0432f93>] __do_softirq+0x89/0xf1
   [<c0432f0a>] ? __do_softirq+0x0/0xf1
   [<c0406e51>] do_softirq+0x7e/0xdf
   [<c0432e6c>] irq_exit+0x4c/0x8b
   [<c041689f>] smp_apic_timer_interrupt+0x73/0x84
   [<c0405691>] apic_timer_interrupt+0x2d/0x34
   [<c044d77e>] ? lock_acquire+0x6f/0x81
   [<c048a69e>] ? page_lock_anon_vma+0x3e/0x5d
   [<c0691b21>] _spin_lock+0x23/0x50
   [<c048a69e>] ? page_lock_anon_vma+0x3e/0x5d
   [<c048a69e>] page_lock_anon_vma+0x3e/0x5d
   [<c048ad98>] page_referenced+0x47/0xfd
   [<c04218e7>] ? need_resched+0x14/0x1e
   [<c047fe00>] shrink_page_list+0x13f/0x560
   [<c044b0e3>] ? trace_hardirqs_off+0xb/0xd
   [<c04099c0>] ? native_sched_clock+0x97/0xb5
   [<c044c413>] ? mark_lock+0x20/0x2fa
   [<c044c732>] ? mark_held_locks+0x45/0x5c
   [<c044c881>] ? trace_hardirqs_on+0xb/0xd
   [<c044c841>] ? trace_hardirqs_on_caller+0xf8/0x12d
   [<c0480327>] shrink_inactive_list+0xdd/0x299
   [<c04805ca>] shrink_zone+0xe7/0x106
   [<c0480958>] do_try_to_free_pages+0x17b/0x2b9
   [<c0480b7a>] try_to_free_pages+0x6c/0x74
   [<c047f72b>] ? isolate_pages_global+0x0/0x43
   [<c047bda7>] __alloc_pages_internal+0x22e/0x38a
   [<c047dbe9>] __do_page_cache_readahead+0xd5/0x19e
   [<c047dce2>] do_page_cache_readahead+0x30/0x3e
   [<c047783a>] filemap_fault+0x13b/0x2db
   [<c0483d1b>] __do_fault+0x40/0x2ff
   [<c044b056>] ? trace_hardirqs_off_caller+0x15/0x97
   [<c04099c0>] ? native_sched_clock+0x97/0xb5
   [<c0482f11>] ? page_address+0x1a/0x87
   [<c044c413>] ? mark_lock+0x20/0x2fa
   [<c04854b3>] handle_mm_fault+0x342/0x6f9
   [<c0693ba0>] ? do_page_fault+0x1cc/0x5af
   [<c044363f>] ? down_read_trylock+0x3e/0x48
   [<c0693c5a>] do_page_fault+0x286/0x5af
   [<c06939d4>] ? do_page_fault+0x0/0x5af
   [<c06922b2>] error_code+0x72/0x78

Thanks!

-- 
Dan Smith
IBM Linux Technology Center
Open Hypervisor Team
email: danms@us.ibm.com