Re: [BUG] unable to handle kernel NULL pointer dereference

Re: [BUG] unable to handle kernel NULL pointer dereference

[John]

> When booting into linux v3.13.3, I am unable to mount an nfs share on this 

> particular hardware.  I get the same problem using v3.12.11.  Only the 3.10.x 
> series allows normal operation.  Partial dmesg output shown inline, additional 
> logs available upon request.
> 
> PLEASE cc me on my replies as I am not subscribed to lkml.
> 
> Hardware: Athlon XP 3200+ on an NVIDIA nForce2 Ultra 400 motherboard.
> Distro: Arch Linux i686.
> 
> % dmesg
> ...
> [ 137.616014] NFS: Registering the id_resolver key type
> [ 137.616036] Key type id_resolver registered
> [ 137.616038] Key type id_legacy registered
> [ 137.686758] BUG: unable to handle kernel NULL pointer dereference at 00000858
> [ 137.689996] IP: [<f8aa2d99>] put_pipe_version+0x19/0x60 [auth_rpcgss]
> [ 137.689996] *pde = 00000000 
> [ 137.689996] Oops: 0000 [#1] PREEMPT SMP 
> [ 137.689996] Modules linked in: rpcsec_gss_krb5 auth_rpcgss oid_registry nfsv4 
> asb100 hwmon_vid snd_wavefront ir_mce_kbd_decoder ir_lirc_codec 
> ir_rc5_sz_decoder ir_sony_decoder lirc_dev ir_rc5_decoder ir_jvc_decoder 
> ir_sanyo_decoder ir_rc6_decoder ir_nec_decoder rc_streamzap streamzap mousedev 
> snd_cs4236 rc_core snd_intel8x0 snd_wss_lib snd_opl3_lib snd_hwdep 
> snd_ac97_codec evdev snd_mpu401 ac97_bus snd_mpu401_uart snd_pcm snd_rawmidi 
> snd_page_alloc snd_seq_device snd_timer snd pcspkr skge shpchp i2c_nforce2 
> i2c_core soundcore ns558 gameport processor button nvidia_agp agpgart nfs lockd 
> sunrpc fscache ext4 crc16 mbcache jbd2 hid_generic usbhid hid sr_mod cdrom 
> sd_mod ata_generic pata_acpi sata_sil pata_amd libata ehci_pci ohci_pci ohci_hcd 
> ehci_hcd scsi_mod usbcore usb_common
> [ 137.689996] CPU: 0 PID: 534 Comm: rpc.gssd Not tainted 3.13.3-1-ARCH #1
> [ 137.689996] Hardware name: ASUSTeK Computer INC. A7N8X-E/A7N8X-E, BIOS ASUS 
> A7N8X-E Deluxe ACPI BIOS Rev 1013 11/12/2004
> [ 137.689996] task: f4633210 ti: f568e000 task.ti: f568e000
> [ 137.689996] EIP: 0060:[<f8aa2d99>] EFLAGS: 00010202 CPU: 0
> [ 137.689996] EIP is at put_pipe_version+0x19/0x60 [auth_rpcgss]
> [ 137.689996] EAX: f4633210 EBX: 00000001 ECX: f56efca8 EDX: 00000296
> [ 137.689996] ESI: 00000000 EDI: f56efc00 EBP: f568fee8 ESP: f568fee0
> [ 137.689996] DS: 007b ES: 007b FS: 00d8 GS: 00e0 SS: 0068
> [ 137.689996] CR0: 8005003b CR2: 00000858 CR3: 34523000 CR4: 000007d0
> [ 137.689996] Stack:
> [ 137.689996] f56efc00 f6c64f78 f568fef4 f8aa2e05 00000010 f568ff40 f8aa3b38 
> 00000374
> [ 137.689996] 00000080 b74dde40 f4644a80 f568ff30 00000246 f8ac1080 ffff41c9 
> f6c64f78
> [ 137.689996] fffffff3 00000010 f4460140 f44d5820 f44d5810 f53df7ec f57595a0 
> f8aa93e8
> [ 137.689996] Call Trace:
> [ 137.689996] [<f8aa2e05>] gss_release_msg+0x25/0x70 [auth_rpcgss]
> [ 137.689996] [<f8aa3b38>] gss_pipe_downcall+0x208/0x4b0 [auth_rpcgss]
> [ 137.689996] [<f8a2f9ab>] rpc_pipe_write+0x3b/0x60 [sunrpc]
> [ 137.689996] [<f8a2f970>] ? rpc_pipe_poll+0x90/0x90 [sunrpc]
> [ 137.689996] [<c1156bd5>] vfs_write+0x95/0x1c0
> [ 137.689996] [<c11572a1>] SyS_write+0x51/0x90
> [ 137.689996] [<c145cc0d>] sysenter_do_call+0x12/0x28
> [ 137.689996] Code: f8 e8 4f b8 9a c8 31 c0 eb c6 90 8d b4 26 00 00 00 00 55 89 
> e5 56 53 3e 8d 74 26 00 8b 1d 28 e9 a3 f8 89 c6 e8 59 64 5f c8 85 db <8b> 
> 86 58 08 00 00 74 3a 3b 18 77 36 8b 5c 98 08 e8 32 66 5f c8
> [ 137.689996] EIP: [<f8aa2d99>] put_pipe_version+0x19/0x60 [auth_rpcgss] 
> SS:ESP 0068:f568fee0
> [ 137.689996] CR2: 0000000000000858
> [ 138.578433] ---[ end trace 3dcb8d5c35b64fbd ]---
> [ 142.979263] type=1006 audit(1392415950.632:4): pid=540 uid=0 old 
> auid=4294967295 new auid=1000 old ses=4294967295 new ses=3 res=1


I should add that if I test the same kernel version (v3.13.3 compiled for i686) on a similar machine of the same vintage, there is not a problem.  When I looked into the `lspci -v` output on the machine that has the problems, I found that it seems to be related to the skge driver as shown below; the similar machine that does not have the problem is using the forcedeth driver so I am hypothesizing that the error is with the skge driver.

01:04.0 Ethernet controller: Marvell Technology Group Ltd. 88E8001 Gigabit Ethernet Controller (rev 13)
        Subsystem: ASUSTeK Computer Inc. Marvell 88E8001 Gigabit Ethernet Controller (Asus)
        Flags: bus master, 66MHz, medium devsel, latency 32, IRQ 17
        Memory at d5000000 (32-bit, non-prefetchable) [size=16K]
        I/O ports at a000 [size=256]
        [virtual] Expansion ROM at 80080000 [disabled] [size=128K]
        Capabilities: [48] Power Management version 2
        Capabilities: [50] Vital Product Data
        Kernel driver in use: skge
        Kernel modules: skge

Re: [BUG] unable to handle kernel NULL pointer dereference

[Borislav Petkov]

If I'd have to guess, that's trying to rcu deref that struct net_generic
*ng in net_generic() but this is only guesswork as I don't have your
.config.

Anyway, adding some more people to CC.

[ 137.689996] Code: f8 e8 4f b8 9a c8 31 c0 eb c6 90 8d b4 26 00 00 00 00 55 89 e5 56 53 3e 8d 74 26 00 8b 1d 28 e9 a3 f8 89 c6 e8 59 64 5f c8 85 db <8b> 86 58 08 00 00 74 3a 3b 18 77 36 8b 5c 98 08 e8 32 66 5f c8
All code
========
   0:   f8                      clc    
   1:   e8 4f b8 9a c8          call   0xc89ab855
   6:   31 c0                   xor    %eax,%eax
   8:   eb c6                   jmp    0xffffffd0
   a:   90                      nop
   b:   8d b4 26 00 00 00 00    lea    0x0(%esi,%eiz,1),%esi
  12:   55                      push   %ebp
  13:   89 e5                   mov    %esp,%ebp
  15:   56                      push   %esi
  16:   53                      push   %ebx
  17:   3e 8d 74 26 00          lea    %ds:0x0(%esi,%eiz,1),%esi
  1c:   8b 1d 28 e9 a3 f8       mov    0xf8a3e928,%ebx
  22:   89 c6                   mov    %eax,%esi
  24:   e8 59 64 5f c8          call   0xc85f6482
  29:   85 db                   test   %ebx,%ebx
  2b:*  8b 86 58 08 00 00       mov    0x858(%esi),%eax         <-- trapping instruction
  31:   74 3a                   je     0x6d
  33:   3b 18                   cmp    (%eax),%ebx
  35:   77 36                   ja     0x6d
  37:   8b 5c 98 08             mov    0x8(%eax,%ebx,4),%ebx
  3b:   e8 32 66 5f c8          call   0xc85f6672

Code starting with the faulting instruction
===========================================
   0:   8b 86 58 08 00 00       mov    0x858(%esi),%eax
   6:   74 3a                   je     0x42
   8:   3b 18                   cmp    (%eax),%ebx
   a:   77 36                   ja     0x42
   c:   8b 5c 98 08             mov    0x8(%eax,%ebx,4),%ebx
  10:   e8 32 66 5f c8          call   0xc85f6647


On Sat, Feb 15, 2014 at 12:08:37PM -0800, John wrote:
> > When booting into linux v3.13.3, I am unable to mount an nfs share on this 
> 
> > particular hardware.  I get the same problem using v3.12.11.  Only the 3.10.x 
> > series allows normal operation.  Partial dmesg output shown inline, additional 
> > logs available upon request.
> > 
> > PLEASE cc me on my replies as I am not subscribed to lkml.
> > 
> > Hardware: Athlon XP 3200+ on an NVIDIA nForce2 Ultra 400 motherboard.
> > Distro: Arch Linux i686.
> > 
> > % dmesg
> > ...
> > [ 137.616014] NFS: Registering the id_resolver key type
> > [ 137.616036] Key type id_resolver registered
> > [ 137.616038] Key type id_legacy registered
> > [ 137.686758] BUG: unable to handle kernel NULL pointer dereference at 00000858
> > [ 137.689996] IP: [<f8aa2d99>] put_pipe_version+0x19/0x60 [auth_rpcgss]
> > [ 137.689996] *pde = 00000000 
> > [ 137.689996] Oops: 0000 [#1] PREEMPT SMP 
> > [ 137.689996] Modules linked in: rpcsec_gss_krb5 auth_rpcgss oid_registry nfsv4 
> > asb100 hwmon_vid snd_wavefront ir_mce_kbd_decoder ir_lirc_codec 
> > ir_rc5_sz_decoder ir_sony_decoder lirc_dev ir_rc5_decoder ir_jvc_decoder 
> > ir_sanyo_decoder ir_rc6_decoder ir_nec_decoder rc_streamzap streamzap mousedev 
> > snd_cs4236 rc_core snd_intel8x0 snd_wss_lib snd_opl3_lib snd_hwdep 
> > snd_ac97_codec evdev snd_mpu401 ac97_bus snd_mpu401_uart snd_pcm snd_rawmidi 
> > snd_page_alloc snd_seq_device snd_timer snd pcspkr skge shpchp i2c_nforce2 
> > i2c_core soundcore ns558 gameport processor button nvidia_agp agpgart nfs lockd 
> > sunrpc fscache ext4 crc16 mbcache jbd2 hid_generic usbhid hid sr_mod cdrom 
> > sd_mod ata_generic pata_acpi sata_sil pata_amd libata ehci_pci ohci_pci ohci_hcd 
> > ehci_hcd scsi_mod usbcore usb_common
> > [ 137.689996] CPU: 0 PID: 534 Comm: rpc.gssd Not tainted 3.13.3-1-ARCH #1
> > [ 137.689996] Hardware name: ASUSTeK Computer INC. A7N8X-E/A7N8X-E, BIOS ASUS 
> > A7N8X-E Deluxe ACPI BIOS Rev 1013 11/12/2004
> > [ 137.689996] task: f4633210 ti: f568e000 task.ti: f568e000
> > [ 137.689996] EIP: 0060:[<f8aa2d99>] EFLAGS: 00010202 CPU: 0
> > [ 137.689996] EIP is at put_pipe_version+0x19/0x60 [auth_rpcgss]
> > [ 137.689996] EAX: f4633210 EBX: 00000001 ECX: f56efca8 EDX: 00000296
> > [ 137.689996] ESI: 00000000 EDI: f56efc00 EBP: f568fee8 ESP: f568fee0
> > [ 137.689996] DS: 007b ES: 007b FS: 00d8 GS: 00e0 SS: 0068
> > [ 137.689996] CR0: 8005003b CR2: 00000858 CR3: 34523000 CR4: 000007d0
> > [ 137.689996] Stack:
> > [ 137.689996] f56efc00 f6c64f78 f568fef4 f8aa2e05 00000010 f568ff40 f8aa3b38 
> > 00000374
> > [ 137.689996] 00000080 b74dde40 f4644a80 f568ff30 00000246 f8ac1080 ffff41c9 
> > f6c64f78
> > [ 137.689996] fffffff3 00000010 f4460140 f44d5820 f44d5810 f53df7ec f57595a0 
> > f8aa93e8
> > [ 137.689996] Call Trace:
> > [ 137.689996] [<f8aa2e05>] gss_release_msg+0x25/0x70 [auth_rpcgss]
> > [ 137.689996] [<f8aa3b38>] gss_pipe_downcall+0x208/0x4b0 [auth_rpcgss]
> > [ 137.689996] [<f8a2f9ab>] rpc_pipe_write+0x3b/0x60 [sunrpc]
> > [ 137.689996] [<f8a2f970>] ? rpc_pipe_poll+0x90/0x90 [sunrpc]
> > [ 137.689996] [<c1156bd5>] vfs_write+0x95/0x1c0
> > [ 137.689996] [<c11572a1>] SyS_write+0x51/0x90
> > [ 137.689996] [<c145cc0d>] sysenter_do_call+0x12/0x28
> > [ 137.689996] Code: f8 e8 4f b8 9a c8 31 c0 eb c6 90 8d b4 26 00 00 00 00 55 89 
> > e5 56 53 3e 8d 74 26 00 8b 1d 28 e9 a3 f8 89 c6 e8 59 64 5f c8 85 db <8b> 
> > 86 58 08 00 00 74 3a 3b 18 77 36 8b 5c 98 08 e8 32 66 5f c8
> > [ 137.689996] EIP: [<f8aa2d99>] put_pipe_version+0x19/0x60 [auth_rpcgss] 
> > SS:ESP 0068:f568fee0
> > [ 137.689996] CR2: 0000000000000858
> > [ 138.578433] ---[ end trace 3dcb8d5c35b64fbd ]---
> > [ 142.979263] type=1006 audit(1392415950.632:4): pid=540 uid=0 old 
> > auid=4294967295 new auid=1000 old ses=4294967295 new ses=3 res=1
> 
> 
> I should add that if I test the same kernel version (v3.13.3 compiled for i686) on a similar machine of the same vintage, there is not a problem.  When I looked into the `lspci -v` output on the machine that has the problems, I found that it seems to be related to the skge driver as shown below; the similar machine that does not have the problem is using the forcedeth driver so I am hypothesizing that the error is with the skge driver.
> 
> 01:04.0 Ethernet controller: Marvell Technology Group Ltd. 88E8001 Gigabit Ethernet Controller (rev 13)
>         Subsystem: ASUSTeK Computer Inc. Marvell 88E8001 Gigabit Ethernet Controller (Asus)
>         Flags: bus master, 66MHz, medium devsel, latency 32, IRQ 17
>         Memory at d5000000 (32-bit, non-prefetchable) [size=16K]
>         I/O ports at a000 [size=256]
>         [virtual] Expansion ROM at 80080000 [disabled] [size=128K]
>         Capabilities: [48] Power Management version 2
>         Capabilities: [50] Vital Product Data
>         Kernel driver in use: skge
>         Kernel modules: skge
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/
> 

-- 
Regards/Gruss,
    Boris.

Sent from a fat crate under my desk. Formatting is fine.
--

Re: [BUG] unable to handle kernel NULL pointer dereference

[John]

----- Original Message -----
> From: Borislav Petkov <>
> Sent: Saturday, February 15, 2014 3:30 PM
> Subject: Re: [BUG] unable to handle kernel NULL pointer dereference
> 
> If I'd have to guess, that's trying to rcu deref that struct net_generic
> *ng in net_generic() but this is only guesswork as I don't have your
> .config.
> 
> Anyway, adding some more people to CC.
> 


Thanks for the reply, Boris.  The .config is unmodified from the Arch Distro default for 3.13.3-1 which can be found here: http://pastebin.com/LPGZ8ZqA

Re: [BUG] unable to handle kernel NULL pointer dereference

[Borislav Petkov]

On Sat, Feb 15, 2014 at 01:04:22PM -0800, John wrote:
> Thanks for the reply, Boris.  The .config is unmodified
> from the Arch Distro default for 3.13.3-1 which can be found
> here: http://pastebin.com/LPGZ8ZqA

Yep, it is that struct net *net argument to put_pipe_version() which is NULL:

  12:   55                      push   %ebp
  13:   89 e5                   mov    %esp,%ebp
  15:   56                      push   %esi
  16:   53                      push   %ebx
  17:   3e 8d 74 26 00          lea    %ds:0x0(%esi,%eiz,1),%esi
  1c:   8b 1d 28 e9 a3 f8       mov    0xf8a3e928,%ebx
  22:   89 c6                   mov    %eax,%esi
  24:   e8 59 64 5f c8          call   0xc85f6482
  29:   85 db                   test   %ebx,%ebx
  2b:*  8b 86 58 08 00 00       mov    0x858(%esi),%eax         <-- trapping instruction

put_pipe_version:
	pushl	%ebp	#
	movl	%esp, %ebp	#,
	pushl	%esi	#
	pushl	%ebx	#
	call	mcount
	movl	sunrpc_net_id, %ebx	# sunrpc_net_id, sunrpc_net_id.130
	movl	%eax, %esi	# net, net
	call	__rcu_read_lock	#
	testl	%ebx, %ebx	# sunrpc_net_id.130
	movl	2136(%esi), %eax	# MEM[(struct net_generic * const *)net_4(D) + 2136B], ng <-- trapping insn


	[ 137.689996] ESI: 00000000 EDI: f56efc00 EBP: f568fee8 ESP: f568fee0
			   ^^^^^^^^

Here's the c/asm interleaved version:

static void put_pipe_version(struct net *net)
{
     d80:       55                      push   %ebp
     d81:       89 e5                   mov    %esp,%ebp
     d83:       56                      push   %esi
     d84:       53                      push   %ebx
     d85:       e8 fc ff ff ff          call   d86 <put_pipe_version+0x6>
                        d86: R_386_PC32 mcount
        struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);
     d8a:       8b 1d 00 00 00 00       mov    0x0,%ebx
                        d8c: R_386_32   sunrpc_net_id
        spin_unlock(&pipe_version_lock);
        return ret;
}

static void put_pipe_version(struct net *net)
{
     d90:       89 c6                   mov    %eax,%esi
 * block, but only when acquiring spinlocks that are subject to priority
 * inheritance.
 */
static inline void rcu_read_lock(void)
{
        __rcu_read_lock();
     d92:       e8 fc ff ff ff          call   d93 <put_pipe_version+0x13>
                        d93: R_386_PC32 __rcu_read_lock
        struct net_generic *ng;
        void *ptr;

        rcu_read_lock();
        ng = rcu_dereference(net->gen);
        BUG_ON(id == 0 || id > ng->len);
     d97:       85 db                   test   %ebx,%ebx
{
        struct net_generic *ng;
        void *ptr;

        rcu_read_lock();
        ng = rcu_dereference(net->gen);
     d99:       8b 86 58 08 00 00       mov    0x858(%esi),%eax			<-- trapping insn


I guess you could avoid the crash if you did

	if (!net)
		return;

in put_pipe_version() but this hardly is the right solution. Someone
else has to make sense of this thing, not me. :-)

HTH.

-- 
Regards/Gruss,
    Boris.

Sent from a fat crate under my desk. Formatting is fine.
--

Re: [BUG] unable to handle kernel NULL pointer dereference

[John]

----- Original Message -----
> From: Borislav Petkov <>
> Sent: Saturday, February 15, 2014 6:25 PM
> Subject: Re: [BUG] unable to handle kernel NULL pointer dereference
> 
> On Sat, Feb 15, 2014 at 01:04:22PM -0800, John wrote:
>>  Thanks for the reply, Boris.  The .config is unmodified
>>  from the Arch Distro default for 3.13.3-1 which can be found
>>  here: http://pastebin.com/LPGZ8ZqA
> 
> Yep, it is that struct net *net argument to put_pipe_version() which is NULL:
> 
>   12:   55                      push   %ebp
>   13:   89 e5                   mov    %esp,%ebp
>   15:   56                      push   %esi
>   16:   53                      push   %ebx
>   17:   3e 8d 74 26 00          lea    %ds:0x0(%esi,%eiz,1),%esi
>   1c:   8b 1d 28 e9 a3 f8       mov    0xf8a3e928,%ebx
>   22:   89 c6                   mov    %eax,%esi
>   24:   e8 59 64 5f c8          call   0xc85f6482
>   29:   85 db                   test   %ebx,%ebx
>   2b:*  8b 86 58 08 00 00       mov    0x858(%esi),%eax         <-- trapping 
> instruction
> 
> put_pipe_version:
>     pushl    %ebp    #
>     movl    %esp, %ebp    #,
>     pushl    %esi    #
>     pushl    %ebx    #
>     call    mcount
>     movl    sunrpc_net_id, %ebx    # sunrpc_net_id, sunrpc_net_id.130
>     movl    %eax, %esi    # net, net
>     call    __rcu_read_lock    #
>     testl    %ebx, %ebx    # sunrpc_net_id.130
>     movl    2136(%esi), %eax    # MEM[(struct net_generic * const *)net_4(D) + 
> 2136B], ng <-- trapping insn
> 
> 
>     [ 137.689996] ESI: 00000000 EDI: f56efc00 EBP: f568fee8 ESP: f568fee0
>                ^^^^^^^^
> 
> Here's the c/asm interleaved version:
> 
> static void put_pipe_version(struct net *net)
> {
>      d80:       55                      push   %ebp
>      d81:       89 e5                   mov    %esp,%ebp
>      d83:       56                      push   %esi
>      d84:       53                      push   %ebx
>      d85:       e8 fc ff ff ff          call   d86 <put_pipe_version+0x6>
>                         d86: R_386_PC32 mcount
>         struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);
>      d8a:       8b 1d 00 00 00 00       mov    0x0,%ebx
>                         d8c: R_386_32   sunrpc_net_id
>         spin_unlock(&pipe_version_lock);
>         return ret;
> }
> 
> static void put_pipe_version(struct net *net)
> {
>      d90:       89 c6                   mov    %eax,%esi
> * block, but only when acquiring spinlocks that are subject to priority
> * inheritance.
> */
> static inline void rcu_read_lock(void)
> {
>         __rcu_read_lock();
>      d92:       e8 fc ff ff ff          call   d93 <put_pipe_version+0x13>
>                         d93: R_386_PC32 __rcu_read_lock
>         struct net_generic *ng;
>         void *ptr;
> 
>         rcu_read_lock();
>         ng = rcu_dereference(net->gen);
>         BUG_ON(id == 0 || id > ng->len);
>      d97:       85 db                   test   %ebx,%ebx
> {
>         struct net_generic *ng;
>         void *ptr;
> 
>         rcu_read_lock();
>         ng = rcu_dereference(net->gen);
>      d99:       8b 86 58 08 00 00       mov    0x858(%esi),%eax            
> <-- trapping insn
> 
> 
> I guess you could avoid the crash if you did
> 
>     if (!net)
>         return;
> 
> in put_pipe_version() but this hardly is the right solution. Someone
> else has to make sense of this thing, not me. :-)
> 
> HTH.


I copy someone you cc'ed on this understands it.  I have no idea what you wrote :)

Re: [BUG] unable to handle kernel NULL pointer dereference

[Trond Myklebust]

Please ensure that you post to the linux-nfs@vger.kernel.org when
reporting NFS and RPC related bugs.

On Sun, 2014-02-16 at 00:25 +0100, Borislav Petkov wrote:
> On Sat, Feb 15, 2014 at 01:04:22PM -0800, John wrote:
> > Thanks for the reply, Boris.  The .config is unmodified
> > from the Arch Distro default for 3.13.3-1 which can be found
> > here: http://pastebin.com/LPGZ8ZqA
> 
> Yep, it is that struct net *net argument to put_pipe_version() which is NULL:
> 
>   12:   55                      push   %ebp
>   13:   89 e5                   mov    %esp,%ebp
>   15:   56                      push   %esi
>   16:   53                      push   %ebx
>   17:   3e 8d 74 26 00          lea    %ds:0x0(%esi,%eiz,1),%esi
>   1c:   8b 1d 28 e9 a3 f8       mov    0xf8a3e928,%ebx
>   22:   89 c6                   mov    %eax,%esi
>   24:   e8 59 64 5f c8          call   0xc85f6482
>   29:   85 db                   test   %ebx,%ebx
>   2b:*  8b 86 58 08 00 00       mov    0x858(%esi),%eax         <-- trapping instruction
> 
> put_pipe_version:
> 	pushl	%ebp	#
> 	movl	%esp, %ebp	#,
> 	pushl	%esi	#
> 	pushl	%ebx	#
> 	call	mcount
> 	movl	sunrpc_net_id, %ebx	# sunrpc_net_id, sunrpc_net_id.130
> 	movl	%eax, %esi	# net, net
> 	call	__rcu_read_lock	#
> 	testl	%ebx, %ebx	# sunrpc_net_id.130
> 	movl	2136(%esi), %eax	# MEM[(struct net_generic * const *)net_4(D) + 2136B], ng <-- trapping insn
> 
> 
> 	[ 137.689996] ESI: 00000000 EDI: f56efc00 EBP: f568fee8 ESP: f568fee0
> 			   ^^^^^^^^
> 
> Here's the c/asm interleaved version:
> 
> static void put_pipe_version(struct net *net)
> {
>      d80:       55                      push   %ebp
>      d81:       89 e5                   mov    %esp,%ebp
>      d83:       56                      push   %esi
>      d84:       53                      push   %ebx
>      d85:       e8 fc ff ff ff          call   d86 <put_pipe_version+0x6>
>                         d86: R_386_PC32 mcount
>         struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);
>      d8a:       8b 1d 00 00 00 00       mov    0x0,%ebx
>                         d8c: R_386_32   sunrpc_net_id
>         spin_unlock(&pipe_version_lock);
>         return ret;
> }
> 
> static void put_pipe_version(struct net *net)
> {
>      d90:       89 c6                   mov    %eax,%esi
>  * block, but only when acquiring spinlocks that are subject to priority
>  * inheritance.
>  */
> static inline void rcu_read_lock(void)
> {
>         __rcu_read_lock();
>      d92:       e8 fc ff ff ff          call   d93 <put_pipe_version+0x13>
>                         d93: R_386_PC32 __rcu_read_lock
>         struct net_generic *ng;
>         void *ptr;
> 
>         rcu_read_lock();
>         ng = rcu_dereference(net->gen);
>         BUG_ON(id == 0 || id > ng->len);
>      d97:       85 db                   test   %ebx,%ebx
> {
>         struct net_generic *ng;
>         void *ptr;
> 
>         rcu_read_lock();
>         ng = rcu_dereference(net->gen);
>      d99:       8b 86 58 08 00 00       mov    0x858(%esi),%eax			<-- trapping insn
> 
> 
> I guess you could avoid the crash if you did
> 
> 	if (!net)
> 		return;
> 
> in put_pipe_version() but this hardly is the right solution. Someone
> else has to make sense of this thing, not me. :-)

Does the following patch help?

8<-------------------------------------------------------------------
>From 0e57b109cd7b17d6e6f16c3454427372a583b18a Mon Sep 17 00:00:00 2001
From: Trond Myklebust <trond.myklebust@primarydata.com>
Date: Sun, 16 Feb 2014 12:14:13 -0500
Subject: [PATCH] SUNRPC: Ensure that gss_auth isn't freed before its upcall
 messages

Fix a race in which the RPC client is shutting down while the
gss daemon is processing a downcall. If the RPC client manages to
shut down before the gss daemon is done, then the struct gss_auth
used in gss_release_msg() may have already been freed.

Link: http://lkml.kernel.org/r/1392494917.71728.YahooMailNeo@web140002.mail.bf1.yahoo.com
Reported-by: John <da_audiophile@yahoo.com>
Reported-by: Borislav Petkov <bp@alien8.de>
Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>
---
 net/sunrpc/auth_gss/auth_gss.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c
index 44a61e8fda6f..1ba1fd114912 100644
--- a/net/sunrpc/auth_gss/auth_gss.c
+++ b/net/sunrpc/auth_gss/auth_gss.c
@@ -108,6 +108,7 @@ struct gss_auth {
 static DEFINE_SPINLOCK(pipe_version_lock);
 static struct rpc_wait_queue pipe_version_rpc_waitqueue;
 static DECLARE_WAIT_QUEUE_HEAD(pipe_version_waitqueue);
+static void gss_put_auth(struct gss_auth *gss_auth);
 
 static void gss_free_ctx(struct gss_cl_ctx *);
 static const struct rpc_pipe_ops gss_upcall_ops_v0;
@@ -320,6 +321,7 @@ gss_release_msg(struct gss_upcall_msg *gss_msg)
 	if (gss_msg->ctx != NULL)
 		gss_put_ctx(gss_msg->ctx);
 	rpc_destroy_wait_queue(&gss_msg->rpc_waitqueue);
+	gss_put_auth(gss_msg->auth);
 	kfree(gss_msg);
 }
 
@@ -500,6 +502,7 @@ gss_alloc_msg(struct gss_auth *gss_auth,
 		if (err)
 			goto err_free_msg;
 	};
+	kref_get(&gss_auth->kref);
 	return gss_msg;
 err_free_msg:
 	kfree(gss_msg);
@@ -1064,6 +1067,12 @@ gss_free_callback(struct kref *kref)
 }
 
 static void
+gss_put_auth(struct gss_auth *gss_auth)
+{
+	kref_put(&gss_auth->kref, gss_free_callback);
+}
+
+static void
 gss_destroy(struct rpc_auth *auth)
 {
 	struct gss_auth *gss_auth = container_of(auth,
@@ -1084,7 +1093,7 @@ gss_destroy(struct rpc_auth *auth)
 	gss_auth->gss_pipe[1] = NULL;
 	rpcauth_destroy_credcache(auth);
 
-	kref_put(&gss_auth->kref, gss_free_callback);
+	gss_put_auth(gss_auth);
 }
 
 /*
@@ -1255,7 +1264,7 @@ gss_destroy_nullcred(struct rpc_cred *cred)
 	call_rcu(&cred->cr_rcu, gss_free_cred_callback);
 	if (ctx)
 		gss_put_ctx(ctx);
-	kref_put(&gss_auth->kref, gss_free_callback);
+	gss_put_auth(gss_auth);
 }
 
 static void
-- 
1.8.5.3


-- 
Trond Myklebust
Linux NFS client maintainer, PrimaryData
trond.myklebust@primarydata.com

Re: [BUG] unable to handle kernel NULL pointer dereference

[Borislav Petkov]

On Sun, Feb 16, 2014 at 12:27:33PM -0500, Trond Myklebust wrote:
> Please ensure that you post to the linux-nfs-u79uwXL29TY76Z2rM5mHXA@public.gmane.org when
> reporting NFS and RPC related bugs.

Sorry, get_maintainer.pl gave it too but far down in an already too
long list and I wasn't sure who to spam so I picked up supporter and
maintainer:

$ ./scripts/get_maintainer.pl -f net/sunrpc/auth_gss/auth_gss.c
"J. Bruce Fields" <bfields-uC3wQj2KruNg9hUCZPvPmw@public.gmane.org> (supporter:KERNEL NFSD, SUNR...,commit_signer:3/26=12%)
Trond Myklebust <trond.myklebust-7I+n7zu2hftEKMMhf/gKZA@public.gmane.org> (maintainer:NFS, SUNRPC, AND...,commit_signer:24/26=92%,authored:14/26=54%,added_lines:394/475=83%,removed_lines:189/215=88%)
"David S. Miller" <davem-fT/PcQaiUtIeIZ0/mPfg9Q@public.gmane.org> (maintainer:NETWORKING [GENERAL])
Andy Adamson <andros-HgOvQuBEEgTQT0dZR+AlfA@public.gmane.org> (commit_signer:3/26=12%,authored:3/26=12%,added_lines:56/475=12%)
Chuck Lever <chuck.lever-QHcLZuEGTsvQT0dZR+AlfA@public.gmane.org> (commit_signer:3/26=12%,authored:3/26=12%)
Jeff Layton <jlayton-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org> (commit_signer:3/26=12%,authored:3/26=12%,removed_lines:19/215=9%)
linux-nfs-u79uwXL29TY76Z2rM5mHXA@public.gmane.org (open list:KERNEL NFSD, SUNR...)
netdev-u79uwXL29TY76Z2rM5mHXA@public.gmane.org (open list:NETWORKING [GENERAL])
linux-kernel-u79uwXL29TY76Z2rM5mHXA@public.gmane.org (open list)

-- 
Regards/Gruss,
    Boris.

Sent from a fat crate under my desk. Formatting is fine.
--
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [BUG] unable to handle kernel NULL pointer dereference

[John]

----- Original Message -----
> From: Trond Myklebust <trond.myklebust-7I+n7zu2hftEKMMhf/gKZA@public.gmane.org>
> To: Borislav Petkov <bp-Gina5bIWoIWzQB+pC5nmwQ@public.gmane.org>; Linux NFS Mailing List <linux-nfs@vger.kernel.org>
> Cc: John <da_audiophile-/E1597aS9LQAvxtiuMwx3w@public.gmane.org>; lkml <linux-kernel-u79uwXL29TaqPxH82wqD4g@public.gmane.orgg>; "netdev-u79uwXL29TY76Z2rM5mHXA@public.gmane.org" <netdev-u79uwXL29TY76Z2rM5mHXA@public.gmane.org>; "stephen@networkplumber.org" <stephen-OTpzqLSitTUnbdJkjeBofR2eb7JE58TQ@public.gmane.org>; "mlindner-eYqpPyKDWXRBDgjK7y7TUQ@public.gmane.org" <mlindner-eYqpPyKDWXRBDgjK7y7TUQ@public.gmane.org>; J. Bruce Fields <bfields-uC3wQj2KruNg9hUCZPvPmw@public.gmane.org>
> Sent: Sunday, February 16, 2014 12:27 PM
> Subject: Re: [BUG] unable to handle kernel NULL pointer dereference
> 
> Please ensure that you post to the linux-nfs-u79uwXL29TY76Z2rM5mHXA@public.gmane.org when
> reporting NFS and RPC related bugs.
>  
> Does the following patch help?
> 


Trond.  Yes, your patch fixes the regression for me; tested on v3.13.3,  I do not know the process by which patches get into the next stable release (minor version).  My hope is that once peer-reviewed, this patch gets into the 3.13.4 and since the 3.12 series is not EOL yet, into 3.12.12 as well.  Thank you for the time and effort!
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [BUG] unable to handle kernel NULL pointer dereference

[Borislav Petkov]

On Mon, Feb 17, 2014 at 12:12:54PM -0800, John wrote:
> Trond.  Yes, your patch fixes the regression for me; tested on
> v3.13.3,  I do not know the process by which patches get into
> the next stable release (minor version).  My hope is that once
> peer-reviewed, this patch gets into the 3.13.4 and since the 3.12
> series is not EOL yet, into 3.12.12 as well.  Thank you for the time
> and effort!

Basically you say

Tested-by: John <da_audiophile-/E1597aS9LQAvxtiuMwx3w@public.gmane.org>

Trond adds stable to CC, sends it to Linus and it trickles down to
3.12.x and 3.13.x stable.

-- 
Regards/Gruss,
    Boris.

Sent from a fat crate under my desk. Formatting is fine.
--
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [BUG] unable to handle kernel NULL pointer dereference

[John]

----- Original Message -----
> From: Trond Myklebust <trond.myklebust@primarydata.com>
> To: Borislav Petkov <bp@alien8.de>; Linux NFS Mailing List <linux-nfs@vger.kernel.org>
> Cc: John <da_audiophile@yahoo.com>; lkml <linux-kernel@vger.kernel.org>; "netdev@vger.kernel.org" <netdev@vger.kernel.org>; "stephen@networkplumber.org" <stephen@networkplumber.org>; "mlindner@marvell.com" <mlindner@marvell.com>; J. Bruce Fields <bfields@fieldses.org>
> Sent: Sunday, February 16, 2014 12:27 PM
> Subject: Re: [BUG] unable to handle kernel NULL pointer dereference
> 
> Please ensure that you post to the linux-nfs@vger.kernel.org when
> reporting NFS and RPC related bugs.
> 
> Does the following patch help?
> 
> 8<-------------------------------------------------------------------
> From 0e57b109cd7b17d6e6f16c3454427372a583b18a Mon Sep 17 00:00:00 2001
> From: Trond Myklebust <trond.myklebust@primarydata.com>
> Date: Sun, 16 Feb 2014 12:14:13 -0500
> Subject: [PATCH] SUNRPC: Ensure that gss_auth isn't freed before its upcall
> messages
> 
> Fix a race in which the RPC client is shutting down while the
> gss daemon is processing a downcall. If the RPC client manages to
> shut down before the gss daemon is done, then the struct gss_auth
> used in gss_release_msg() may have already been freed.
> 
> Link: 
> http://lkml.kernel.org/r/1392494917.71728.YahooMailNeo@web140002.mail.bf1.yahoo.com
> Reported-by: John <da_audiophile@yahoo.com>
> Reported-by: Borislav Petkov <bp@alien8.de>
> Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>
> ---
> net/sunrpc/auth_gss/auth_gss.c | 13 +++++++++++--
> 1 file changed, 11 insertions(+), 2 deletions(-)
> 
> diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c
> index 44a61e8fda6f..1ba1fd114912 100644
> --- a/net/sunrpc/auth_gss/auth_gss.c
> +++ b/net/sunrpc/auth_gss/auth_gss.c
> @@ -108,6 +108,7 @@ struct gss_auth {
> static DEFINE_SPINLOCK(pipe_version_lock);
> static struct rpc_wait_queue pipe_version_rpc_waitqueue;
> static DECLARE_WAIT_QUEUE_HEAD(pipe_version_waitqueue);
> +static void gss_put_auth(struct gss_auth *gss_auth);
> 
> static void gss_free_ctx(struct gss_cl_ctx *);
> static const struct rpc_pipe_ops gss_upcall_ops_v0;
> @@ -320,6 +321,7 @@ gss_release_msg(struct gss_upcall_msg *gss_msg)
>     if (gss_msg->ctx != NULL)
>         gss_put_ctx(gss_msg->ctx);
>     rpc_destroy_wait_queue(&gss_msg->rpc_waitqueue);
> +    gss_put_auth(gss_msg->auth);
>     kfree(gss_msg);
> }
> 
> @@ -500,6 +502,7 @@ gss_alloc_msg(struct gss_auth *gss_auth,
>         if (err)
>             goto err_free_msg;
>     };
> +    kref_get(&gss_auth->kref);
>     return gss_msg;
> err_free_msg:
>     kfree(gss_msg);
> @@ -1064,6 +1067,12 @@ gss_free_callback(struct kref *kref)
> }
> 
> static void
> +gss_put_auth(struct gss_auth *gss_auth)
> +{
> +    kref_put(&gss_auth->kref, gss_free_callback);
> +}
> +
> +static void
> gss_destroy(struct rpc_auth *auth)
> {
>     struct gss_auth *gss_auth = container_of(auth,
> @@ -1084,7 +1093,7 @@ gss_destroy(struct rpc_auth *auth)
>     gss_auth->gss_pipe[1] = NULL;
>     rpcauth_destroy_credcache(auth);
> 
> -    kref_put(&gss_auth->kref, gss_free_callback);
> +    gss_put_auth(gss_auth);
> }
> 
> /*
> @@ -1255,7 +1264,7 @@ gss_destroy_nullcred(struct rpc_cred *cred)
>     call_rcu(&cred->cr_rcu, gss_free_cred_callback);
>     if (ctx)
>         gss_put_ctx(ctx);
> -    kref_put(&gss_auth->kref, gss_free_callback);
> +    gss_put_auth(gss_auth);
> 
> }
> 
> static void
> -- 


Tested-by: John <da_audiophile@yahoo.com>


Fixes the problem on 3.13.3 for me (i686). Thank you.