kgdb test suite failure

kgdb test suite failure

[Alexander Beregalov]

[-- Attachment #1: Type: text/plain, Size: 5968 bytes --]

Hi
I tried to run the latest git kernel and got the following error.

See an attachment for full dmesg.

kgdbts: ERROR PUT: end of test buffer on 'do_fork_test' line 5
expected OK got $E02#a7
------------[ cut here ]------------
WARNING: at drivers/misc/kgdbts.c:721 run_simple_test+0x1de/0x20e()
Modules linked in:
Pid: 6, comm: khelper Not tainted 2.6.26-rc1-00279-g28a4acb #12
[<c011a00a>] warn_on_slowpath+0x41/0x6c
[<c011aa77>] ? vprintk+0x3e7/0x407
[<c0254cf4>] ? number+0x10d/0x1cf
[<c012d3da>] ? sched_clock_cpu+0x70/0x88
[<c011aaac>] ? printk+0x15/0x17
[<c02ae228>] run_simple_test+0x1de/0x20e
[<c02adc15>] kgdbts_put_char+0x14/0x16
[<c013d7ba>] put_packet+0x6b/0xbb
[<c013e8bc>] kgdb_handle_exception+0xa19/0xb21
[<c0119400>] ? do_fork+0x0/0x19e
[<c010e9e3>] kgdb_notify+0x103/0x140
[<c035ac89>] notifier_call_chain+0x2b/0x4a
[<c035accd>] __atomic_notifier_call_chain+0x25/0x36
[<c035acea>] atomic_notifier_call_chain+0xc/0xe
[<c012cd15>] notify_die+0x2d/0x2f
[<c03597fd>] do_int3+0x32/0x6c
[<c0359293>] int3+0x27/0x2c
[<c0119400>] ? do_fork+0x0/0x19e
[<c010176c>] ? kernel_thread+0x73/0x7b
[<c01263d1>] ? ____call_usermodehelper+0x0/0xe7
[<c01263d1>] ? ____call_usermodehelper+0x0/0xe7
[<c0103740>] ? kernel_thread_helper+0x0/0x10
[<c0126236>] __call_usermodehelper+0x33/0x4d
[<c0126686>] run_workqueue+0xba/0x187
[<c012664c>] ? run_workqueue+0x80/0x187
[<c0126203>] ? __call_usermodehelper+0x0/0x4d
[<c0126ea6>] worker_thread+0x80/0x8c
[<c01291bd>] ? autoremove_wake_function+0x0/0x30
[<c0126e26>] ? worker_thread+0x0/0x8c
[<c01290fc>] kthread+0x39/0x61
[<c01290c3>] ? kthread+0x0/0x61
[<c0103747>] kernel_thread_helper+0x7/0x10
=======================
---[ end trace b9faa95810ce9135 ]---
e1000: eth0: e1000_clean_tx_irq: Detected Tx Unit Hang
 Tx Queue             <0>
 TDH                  <2a>
 TDT                  <2a>
 next_to_use          <2a>
 next_to_clean        <14>
buffer_info[next_to_clean]
 time_stamp           <fffeb89e>
 next_to_watch        <14>
 jiffies              <fffebb63>
 next_to_watch.status <1>
khelper used greatest stack depth: 2516 bytes left
BUG: unable to handle kernel NULL pointer dereference at 00000000
IP: [<00000000>]
*pde = 00000000
Oops: 0000 [#1] PREEMPT DEBUG_PAGEALLOC
kgdbts: BP mismatch 0 expected c0119400
------------[ cut here ]------------
WARNING: at drivers/misc/kgdbts.c:302 check_and_rewind_pc+0x9a/0xb4()
Modules linked in:
Pid: 6, comm: khelper Tainted: G        W 2.6.26-rc1-00279-g28a4acb #12
[<c011a00a>] warn_on_slowpath+0x41/0x6c
[<c011aa77>] ? vprintk+0x3e7/0x407
[<c0119400>] ? do_fork+0x0/0x19e
[<c012d3da>] ? sched_clock_cpu+0x70/0x88
[<c0116d4e>] ? cpu_clock+0x115/0x12c
[<c0119400>] ? do_fork+0x0/0x19e
[<c011aaac>] ? printk+0x15/0x17
[<c0119400>] ? do_fork+0x0/0x19e
[<c02ae96d>] check_and_rewind_pc+0x9a/0xb4
[<c0119400>] ? do_fork+0x0/0x19e
[<c02adb7a>] validate_simple_test+0x22/0x69
[<c02ae1eb>] run_simple_test+0x1a1/0x20e
[<c02adc15>] kgdbts_put_char+0x14/0x16
[<c013d7ba>] put_packet+0x6b/0xbb
[<c013e8bc>] kgdb_handle_exception+0xa19/0xb21
[<c010e9e3>] kgdb_notify+0x103/0x140
[<c035ac89>] notifier_call_chain+0x2b/0x4a
[<c035accd>] __atomic_notifier_call_chain+0x25/0x36
[<c035acea>] atomic_notifier_call_chain+0xc/0xe
[<c012cd15>] notify_die+0x2d/0x2f
[<c0359303>] __die+0x61/0xc6
[<c0103dec>] die+0x87/0x106
[<c035aae1>] do_page_fault+0x4e3/0x59d
[<c035a5fe>] ? do_page_fault+0x0/0x59d
[<c010176c>] ? kernel_thread+0x73/0x7b
[<c0359102>] error_code+0x6a/0x70
[<c010176c>] ? kernel_thread+0x73/0x7b
[<c01263d1>] ? ____call_usermodehelper+0x0/0xe7
[<c01263d1>] ? ____call_usermodehelper+0x0/0xe7
[<c0103740>] ? kernel_thread_helper+0x0/0x10
[<c0126236>] ? __call_usermodehelper+0x33/0x4d
[<c0126686>] ? run_workqueue+0xba/0x187
[<c012664c>] ? run_workqueue+0x80/0x187
[<c0126203>] ? __call_usermodehelper+0x0/0x4d
[<c0126ea6>] ? worker_thread+0x80/0x8c
[<c01291bd>] ? autoremove_wake_function+0x0/0x30
[<c0126e26>] ? worker_thread+0x0/0x8c
[<c01290fc>] ? kthread+0x39/0x61
[<c01290c3>] ? kthread+0x0/0x61
[<c0103747>] ? kernel_thread_helper+0x7/0x10
=======================
---[ end trace b9faa95810ce9135 ]---
kgdbts: ERROR PUT: end of test buffer on 'do_fork_test' line 3
expected do_fork got
$e201000000b0c6f70000000000000000000000006c1710c01141800054bfc6f7Б^A
------------[ cut here ]------------
WARNING: at drivers/misc/kgdbts.c:721 run_simple_test+0x1de/0x20e()
Modules linked in:
Pid: 6, comm: khelper Tainted: G        W 2.6.26-rc1-00279-g28a4acb #12
[<c011a00a>] warn_on_slowpath+0x41/0x6c
[<c011aa77>] ? vprintk+0x3e7/0x407
[<c0119400>] ? do_fork+0x0/0x19e
[<c012d3da>] ? sched_clock_cpu+0x70/0x88
[<c0116d4e>] ? cpu_clock+0x115/0x12c
[<c0119400>] ? do_fork+0x0/0x19e
[<c011aaac>] ? printk+0x15/0x17
[<c011aaac>] ? printk+0x15/0x17
[<c02ae228>] run_simple_test+0x1de/0x20e
[<c02adc15>] kgdbts_put_char+0x14/0x16
[<c013d7ba>] put_packet+0x6b/0xbb
[<c013e8bc>] kgdb_handle_exception+0xa19/0xb21
[<c010e9e3>] kgdb_notify+0x103/0x140
[<c035ac89>] notifier_call_chain+0x2b/0x4a
[<c035accd>] __atomic_notifier_call_chain+0x25/0x36
[<c035acea>] atomic_notifier_call_chain+0xc/0xe
[<c012cd15>] notify_die+0x2d/0x2f
[<c0359303>] __die+0x61/0xc6
[<c0103dec>] die+0x87/0x106
[<c035aae1>] do_page_fault+0x4e3/0x59d
[<c035a5fe>] ? do_page_fault+0x0/0x59d
[<c010176c>] ? kernel_thread+0x73/0x7b
[<c0359102>] error_code+0x6a/0x70
[<c010176c>] ? kernel_thread+0x73/0x7b
[<c01263d1>] ? ____call_usermodehelper+0x0/0xe7
[<c01263d1>] ? ____call_usermodehelper+0x0/0xe7
[<c0103740>] ? kernel_thread_helper+0x0/0x10
[<c0126236>] ? __call_usermodehelper+0x33/0x4d
[<c0126686>] ? run_workqueue+0xba/0x187
[<c012664c>] ? run_workqueue+0x80/0x187
[<c0126203>] ? __call_usermodehelper+0x0/0x4d
[<c0126ea6>] ? worker_thread+0x80/0x8c
[<c01291bd>] ? autoremove_wake_function+0x0/0x30
[<c0126e26>] ? worker_thread+0x0/0x8c
[<c01290fc>] ? kthread+0x39/0x61
[<c01290c3>] ? kthread+0x0/0x61
[<c0103747>] ? kernel_thread_helper+0x7/0x10
=======================
---[ end trace b9faa95810ce9135 ]---

[-- Attachment #2: dmesg.tor --]
[-- Type: application/octet-stream, Size: 38410 bytes --]

Re: kgdb test suite failure

[Rafael J. Wysocki]

[Added some CCs.]

On Friday, 9 of May 2008, Alexander Beregalov wrote:
> Hi
> I tried to run the latest git kernel and got the following error.
> 
> See an attachment for full dmesg.
> 
> kgdbts: ERROR PUT: end of test buffer on 'do_fork_test' line 5
> expected OK got $E02#a7
> ------------[ cut here ]------------
> WARNING: at drivers/misc/kgdbts.c:721 run_simple_test+0x1de/0x20e()
> Modules linked in:
> Pid: 6, comm: khelper Not tainted 2.6.26-rc1-00279-g28a4acb #12
> [<c011a00a>] warn_on_slowpath+0x41/0x6c
> [<c011aa77>] ? vprintk+0x3e7/0x407
> [<c0254cf4>] ? number+0x10d/0x1cf
> [<c012d3da>] ? sched_clock_cpu+0x70/0x88
> [<c011aaac>] ? printk+0x15/0x17
> [<c02ae228>] run_simple_test+0x1de/0x20e
> [<c02adc15>] kgdbts_put_char+0x14/0x16
> [<c013d7ba>] put_packet+0x6b/0xbb
> [<c013e8bc>] kgdb_handle_exception+0xa19/0xb21
> [<c0119400>] ? do_fork+0x0/0x19e
> [<c010e9e3>] kgdb_notify+0x103/0x140
> [<c035ac89>] notifier_call_chain+0x2b/0x4a
> [<c035accd>] __atomic_notifier_call_chain+0x25/0x36
> [<c035acea>] atomic_notifier_call_chain+0xc/0xe
> [<c012cd15>] notify_die+0x2d/0x2f
> [<c03597fd>] do_int3+0x32/0x6c
> [<c0359293>] int3+0x27/0x2c
> [<c0119400>] ? do_fork+0x0/0x19e
> [<c010176c>] ? kernel_thread+0x73/0x7b
> [<c01263d1>] ? ____call_usermodehelper+0x0/0xe7
> [<c01263d1>] ? ____call_usermodehelper+0x0/0xe7
> [<c0103740>] ? kernel_thread_helper+0x0/0x10
> [<c0126236>] __call_usermodehelper+0x33/0x4d
> [<c0126686>] run_workqueue+0xba/0x187
> [<c012664c>] ? run_workqueue+0x80/0x187
> [<c0126203>] ? __call_usermodehelper+0x0/0x4d
> [<c0126ea6>] worker_thread+0x80/0x8c
> [<c01291bd>] ? autoremove_wake_function+0x0/0x30
> [<c0126e26>] ? worker_thread+0x0/0x8c
> [<c01290fc>] kthread+0x39/0x61
> [<c01290c3>] ? kthread+0x0/0x61
> [<c0103747>] kernel_thread_helper+0x7/0x10
> =======================
> ---[ end trace b9faa95810ce9135 ]---
> e1000: eth0: e1000_clean_tx_irq: Detected Tx Unit Hang
>  Tx Queue             <0>
>  TDH                  <2a>
>  TDT                  <2a>
>  next_to_use          <2a>
>  next_to_clean        <14>
> buffer_info[next_to_clean]
>  time_stamp           <fffeb89e>
>  next_to_watch        <14>
>  jiffies              <fffebb63>
>  next_to_watch.status <1>
> khelper used greatest stack depth: 2516 bytes left
> BUG: unable to handle kernel NULL pointer dereference at 00000000
> IP: [<00000000>]
> *pde = 00000000
> Oops: 0000 [#1] PREEMPT DEBUG_PAGEALLOC
> kgdbts: BP mismatch 0 expected c0119400
> ------------[ cut here ]------------
> WARNING: at drivers/misc/kgdbts.c:302 check_and_rewind_pc+0x9a/0xb4()
> Modules linked in:
> Pid: 6, comm: khelper Tainted: G        W 2.6.26-rc1-00279-g28a4acb #12
> [<c011a00a>] warn_on_slowpath+0x41/0x6c
> [<c011aa77>] ? vprintk+0x3e7/0x407
> [<c0119400>] ? do_fork+0x0/0x19e
> [<c012d3da>] ? sched_clock_cpu+0x70/0x88
> [<c0116d4e>] ? cpu_clock+0x115/0x12c
> [<c0119400>] ? do_fork+0x0/0x19e
> [<c011aaac>] ? printk+0x15/0x17
> [<c0119400>] ? do_fork+0x0/0x19e
> [<c02ae96d>] check_and_rewind_pc+0x9a/0xb4
> [<c0119400>] ? do_fork+0x0/0x19e
> [<c02adb7a>] validate_simple_test+0x22/0x69
> [<c02ae1eb>] run_simple_test+0x1a1/0x20e
> [<c02adc15>] kgdbts_put_char+0x14/0x16
> [<c013d7ba>] put_packet+0x6b/0xbb
> [<c013e8bc>] kgdb_handle_exception+0xa19/0xb21
> [<c010e9e3>] kgdb_notify+0x103/0x140
> [<c035ac89>] notifier_call_chain+0x2b/0x4a
> [<c035accd>] __atomic_notifier_call_chain+0x25/0x36
> [<c035acea>] atomic_notifier_call_chain+0xc/0xe
> [<c012cd15>] notify_die+0x2d/0x2f
> [<c0359303>] __die+0x61/0xc6
> [<c0103dec>] die+0x87/0x106
> [<c035aae1>] do_page_fault+0x4e3/0x59d
> [<c035a5fe>] ? do_page_fault+0x0/0x59d
> [<c010176c>] ? kernel_thread+0x73/0x7b
> [<c0359102>] error_code+0x6a/0x70
> [<c010176c>] ? kernel_thread+0x73/0x7b
> [<c01263d1>] ? ____call_usermodehelper+0x0/0xe7
> [<c01263d1>] ? ____call_usermodehelper+0x0/0xe7
> [<c0103740>] ? kernel_thread_helper+0x0/0x10
> [<c0126236>] ? __call_usermodehelper+0x33/0x4d
> [<c0126686>] ? run_workqueue+0xba/0x187
> [<c012664c>] ? run_workqueue+0x80/0x187
> [<c0126203>] ? __call_usermodehelper+0x0/0x4d
> [<c0126ea6>] ? worker_thread+0x80/0x8c
> [<c01291bd>] ? autoremove_wake_function+0x0/0x30
> [<c0126e26>] ? worker_thread+0x0/0x8c
> [<c01290fc>] ? kthread+0x39/0x61
> [<c01290c3>] ? kthread+0x0/0x61
> [<c0103747>] ? kernel_thread_helper+0x7/0x10
> =======================
> ---[ end trace b9faa95810ce9135 ]---
> kgdbts: ERROR PUT: end of test buffer on 'do_fork_test' line 3
> expected do_fork got
> $e201000000b0c6f70000000000000000000000006c1710c01141800054bfc6f7Б^A
> ------------[ cut here ]------------
> WARNING: at drivers/misc/kgdbts.c:721 run_simple_test+0x1de/0x20e()
> Modules linked in:
> Pid: 6, comm: khelper Tainted: G        W 2.6.26-rc1-00279-g28a4acb #12
> [<c011a00a>] warn_on_slowpath+0x41/0x6c
> [<c011aa77>] ? vprintk+0x3e7/0x407
> [<c0119400>] ? do_fork+0x0/0x19e
> [<c012d3da>] ? sched_clock_cpu+0x70/0x88
> [<c0116d4e>] ? cpu_clock+0x115/0x12c
> [<c0119400>] ? do_fork+0x0/0x19e
> [<c011aaac>] ? printk+0x15/0x17
> [<c011aaac>] ? printk+0x15/0x17
> [<c02ae228>] run_simple_test+0x1de/0x20e
> [<c02adc15>] kgdbts_put_char+0x14/0x16
> [<c013d7ba>] put_packet+0x6b/0xbb
> [<c013e8bc>] kgdb_handle_exception+0xa19/0xb21
> [<c010e9e3>] kgdb_notify+0x103/0x140
> [<c035ac89>] notifier_call_chain+0x2b/0x4a
> [<c035accd>] __atomic_notifier_call_chain+0x25/0x36
> [<c035acea>] atomic_notifier_call_chain+0xc/0xe
> [<c012cd15>] notify_die+0x2d/0x2f
> [<c0359303>] __die+0x61/0xc6
> [<c0103dec>] die+0x87/0x106
> [<c035aae1>] do_page_fault+0x4e3/0x59d
> [<c035a5fe>] ? do_page_fault+0x0/0x59d
> [<c010176c>] ? kernel_thread+0x73/0x7b
> [<c0359102>] error_code+0x6a/0x70
> [<c010176c>] ? kernel_thread+0x73/0x7b
> [<c01263d1>] ? ____call_usermodehelper+0x0/0xe7
> [<c01263d1>] ? ____call_usermodehelper+0x0/0xe7
> [<c0103740>] ? kernel_thread_helper+0x0/0x10
> [<c0126236>] ? __call_usermodehelper+0x33/0x4d
> [<c0126686>] ? run_workqueue+0xba/0x187
> [<c012664c>] ? run_workqueue+0x80/0x187
> [<c0126203>] ? __call_usermodehelper+0x0/0x4d
> [<c0126ea6>] ? worker_thread+0x80/0x8c
> [<c01291bd>] ? autoremove_wake_function+0x0/0x30
> [<c0126e26>] ? worker_thread+0x0/0x8c
> [<c01290fc>] ? kthread+0x39/0x61
> [<c01290c3>] ? kthread+0x0/0x61
> [<c0103747>] ? kernel_thread_helper+0x7/0x10
> =======================
> ---[ end trace b9faa95810ce9135 ]---
> 



-- 
"Premature optimization is the root of all evil." - Donald Knuth

Re: kgdb test suite failure

[Jason Wessel]

Alexander Beregalov wrote:
> Hi
> I tried to run the latest git kernel and got the following error.
>
> See an attachment for full dmesg.
>
> kgdbts: ERROR PUT: end of test buffer on 'do_fork_test' line 5
> expected OK got $E02#a7
> ------------[ cut here ]------------
> WARNING: at drivers/misc/kgdbts.c:721 run_simple_test+0x1de/0x20e()
> Modules linked in:
> Pid: 6, comm: khelper Not tainted 2.6.26-rc1-00279-g28a4acb #12
> [<c011a00a>] warn_on_slowpath+0x41/0x6c
> [<c011aa77>] ? vprintk+0x3e7/0x407
> [<c0254cf4>] ? number+0x10d/0x1cf
> [<c012d3da>] ? sched_clock_cpu+0x70/0x88
> [<c011aaac>] ? printk+0x15/0x17
>   
Is this a problem you can reproduce every time?  Basically what line 5
of the test does is to write a value into the register struct to rewind
the PC after hitting a breakpoint such that the instruction will be
executed again.   This is stored in memory which will be used for a
context switch back to the process.   The test also replaces the
original instruction in memory.  In this case the memory write failed. 
This will certainly be fatal to the operation of the kernel and
stability would be called into question if the kernel doesn't just crash
and oops in strange ways immediately.

I looked in your dmesg log and noticed:
[   21.027632] Testing CPA: undo c035d000-c044e000
[   21.083029] Testing CPA: write protecting again
[   21.297278] kgdbts: ERROR PUT: end of test buffer on 'do_fork_test'
line 5 expected OK got $E02#a7


I looked through the kernel source in the tip of the tree for the
"Testing CPA: write protecting again".  It seems this code is only used
when CONFIG_DEBUG_RODATA is set.  Perhaps this option is mutually
exclusive of the use of kgdb?  Today kgdb assumes it can write anywhere
without any kind of special concession for page protection bits.

Certainly kgdb's memory write function could be changed based on the
conditional CONFIG_DEBUG_RODATA if there is an api to
check/change/revert the protection bits on the memory pages.  This of
course is assuming that this is the root cause of the problem.

Jason.

Re: kgdb test suite failure

[Alexander Beregalov]

2008/5/20 Jason Wessel <jason.wessel@windriver.com>:
> Alexander Beregalov wrote:
>> Hi
>> I tried to run the latest git kernel and got the following error.
>>
>> See an attachment for full dmesg.
>>
>> kgdbts: ERROR PUT: end of test buffer on 'do_fork_test' line 5
>> expected OK got $E02#a7
>> ------------[ cut here ]------------
>> WARNING: at drivers/misc/kgdbts.c:721 run_simple_test+0x1de/0x20e()
>> Modules linked in:
>> Pid: 6, comm: khelper Not tainted 2.6.26-rc1-00279-g28a4acb #12
>> [<c011a00a>] warn_on_slowpath+0x41/0x6c
>> [<c011aa77>] ? vprintk+0x3e7/0x407
>> [<c0254cf4>] ? number+0x10d/0x1cf
>> [<c012d3da>] ? sched_clock_cpu+0x70/0x88
>> [<c011aaac>] ? printk+0x15/0x17
>>
> Is this a problem you can reproduce every time?  Basically what line 5
Yes, it happened every time when kernel booted, even before init.

> of the test does is to write a value into the register struct to rewind
> the PC after hitting a breakpoint such that the instruction will be
> executed again.   This is stored in memory which will be used for a
> context switch back to the process.   The test also replaces the
> original instruction in memory.  In this case the memory write failed.
> This will certainly be fatal to the operation of the kernel and
> stability would be called into question if the kernel doesn't just crash
> and oops in strange ways immediately.
>
> I looked in your dmesg log and noticed:
> [   21.027632] Testing CPA: undo c035d000-c044e000
> [   21.083029] Testing CPA: write protecting again
> [   21.297278] kgdbts: ERROR PUT: end of test buffer on 'do_fork_test'
> line 5 expected OK got $E02#a7
>
>
> I looked through the kernel source in the tip of the tree for the
> "Testing CPA: write protecting again".  It seems this code is only used
> when CONFIG_DEBUG_RODATA is set.  Perhaps this option is mutually
> exclusive of the use of kgdb?  Today kgdb assumes it can write anywhere
> without any kind of special concession for page protection bits.
It seems like this.
The kernel with the following config can boot.
CONFIG_HAVE_ARCH_KGDB=y
CONFIG_KGDB=y
CONFIG_KGDB_SERIAL_CONSOLE=y
CONFIG_KGDB_TESTS=y
CONFIG_KGDB_TESTS_ON_BOOT=y
CONFIG_KGDB_TESTS_BOOT_STRING="V1F100"
# CONFIG_DEBUG_RODATA is not set

And kgdbts runs successfully:
[    7.579110] kgdb: Registered I/O driver kgdbts.
[    7.633491] kgdbts:RUN plant and detach test
[    7.686505] kgdbts:RUN sw breakpoint test
[    7.734031] kgdbts:RUN bad memory access test
[    7.786258] kgdbts:RUN singlestep test 1000 iterations
[    7.847832] kgdbts:RUN singlestep [0/1000]
[    7.905206] kgdbts:RUN singlestep [100/1000]
[    7.965778] kgdbts:RUN singlestep [200/1000]
[    8.026037] kgdbts:RUN singlestep [300/1000]
[    8.086507] kgdbts:RUN singlestep [400/1000]
[    8.146839] kgdbts:RUN singlestep [500/1000]
[    8.207588] kgdbts:RUN singlestep [600/1000]
[    8.267889] kgdbts:RUN singlestep [700/1000]
[    8.328643] kgdbts:RUN singlestep [800/1000]
[    8.388939] kgdbts:RUN singlestep [900/1000]
[    8.449322] kgdbts:RUN hw breakpoint test
[    8.497369] kgdbts:RUN hw write breakpoint test
[    8.551663] kgdbts:RUN access write breakpoint test
[    8.610117] kgdbts:RUN do_fork for 100 breakpoints

It's 2.6.26-rc3-00154-gc110a2b

>
> Certainly kgdb's memory write function could be changed based on the
> conditional CONFIG_DEBUG_RODATA if there is an api to
> check/change/revert the protection bits on the memory pages.  This of
> course is assuming that this is the root cause of the problem.
>
> Jason.
>
>

Re: kgdb test suite failure

[Jason Wessel]

Alexander Beregalov wrote:
> 2008/5/20 Jason Wessel <jason.wessel@windriver.com>:
>> Alexander Beregalov wrote:
>>> Hi
>>> I tried to run the latest git kernel and got the following error.
>>>
>>> See an attachment for full dmesg.
>>>
>>> kgdbts: ERROR PUT: end of test buffer on 'do_fork_test' line 5
>>> expected OK got $E02#a7
>>> ------------[ cut here ]------------
>>> WARNING: at drivers/misc/kgdbts.c:721 run_simple_test+0x1de/0x20e()
>>> Modules linked in:
>>> Pid: 6, comm: khelper Not tainted 2.6.26-rc1-00279-g28a4acb #12
>>> [<c011a00a>] warn_on_slowpath+0x41/0x6c
>>> [<c011aa77>] ? vprintk+0x3e7/0x407
>>> [<c0254cf4>] ? number+0x10d/0x1cf
>>> [<c012d3da>] ? sched_clock_cpu+0x70/0x88
>>> [<c011aaac>] ? printk+0x15/0x17
>>>
>> Is this a problem you can reproduce every time?  Basically what line 5
> Yes, it happened every time when kernel booted, even before init.
> 
>> of the test does is to write a value into the register struct to rewind
>> the PC after hitting a breakpoint such that the instruction will be
>> executed again.   This is stored in memory which will be used for a
>> context switch back to the process.   The test also replaces the
>> original instruction in memory.  In this case the memory write failed.
>> This will certainly be fatal to the operation of the kernel and
>> stability would be called into question if the kernel doesn't just crash
>> and oops in strange ways immediately.
>>
>> I looked in your dmesg log and noticed:
>> [   21.027632] Testing CPA: undo c035d000-c044e000
>> [   21.083029] Testing CPA: write protecting again
>> [   21.297278] kgdbts: ERROR PUT: end of test buffer on 'do_fork_test'
>> line 5 expected OK got $E02#a7
>>
>>
>> I looked through the kernel source in the tip of the tree for the
>> "Testing CPA: write protecting again".  It seems this code is only used
>> when CONFIG_DEBUG_RODATA is set.  Perhaps this option is mutually
>> exclusive of the use of kgdb?  Today kgdb assumes it can write anywhere
>> without any kind of special concession for page protection bits.


I duplicated the problem by trying the kernel out with the
CONFIG_DEBUG_RODATA, which appears to only be implemented on the x86
architecture.  I am not exactly certain what the right level of fix
should be in this case, or if it should be fixed at all.  It seems
that the probe_kernel_write should have a force option to force
writing to a read-only memory page (meaning it will be
unprotected/re-protected), or there should be another function to deal
with writes that are deemed to be critical which calls the
probe_kernel_write() from with in kgdb.

It is an interesting problem, but the first inclination is not to fix
it at all because debugging read-only text sections is generally a job
for hardware breakpoints.  The kgdb test suite fell into the trap of
trying to debug a function call that was in a read-only page using
software breakpoints with trap instructions.

An RFC type patch follows which addresses the problem, but in my
opinion not in a terribly clean way...  Perhaps other folks will
comment on a different or better way to approach the problem or chime
in on if it should be fixed at all.  Certainly the test suite could be
changed to use HW breakpoints for the case where the text segments are
going to be read only as an example.

Another approach I considered was to add a global option to kgdb which
causes it to force write to read only pages.  This could be used by
the kgdb test suite as well as by a "human" via a debugger to change
the setting as needed.  How often you might want to set a software
breakpoint in a read-only page remains to be seen, but it does seem
like something that is marginally useful.

Jason.


===RFC only to illustrate a work around===

---
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 60bcb5b..e4d0039 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -737,7 +737,7 @@ static int change_page_attr_set_clr(unsigned long addr, int numpages,
 	/*
 	 * Check whether we really changed something:
 	 */
-	if (!cpa.flushtlb)
+	if (!cpa.flushtlb || in_atomic())
 		goto out;
 
 	/*
diff --git a/mm/maccess.c b/mm/maccess.c
index ac40796..7609fb9 100644
--- a/mm/maccess.c
+++ b/mm/maccess.c
@@ -4,6 +4,7 @@
 #include <linux/uaccess.h>
 #include <linux/module.h>
 #include <linux/mm.h>
+#include <asm/cacheflush.h>
 
 /**
  * probe_kernel_read(): safely attempt to read from a location
@@ -42,11 +43,24 @@ EXPORT_SYMBOL_GPL(probe_kernel_read);
 long probe_kernel_write(void *dst, void *src, size_t size)
 {
 	long ret;
+#ifdef CONFIG_X86
+	unsigned int level;
+	pte_t *pte = lookup_address((unsigned long)dst, &level);
+	int unprotect = !pte_write(*pte);
+#endif
 	mm_segment_t old_fs = get_fs();
 
 	set_fs(KERNEL_DS);
 	pagefault_disable();
+#ifdef CONFIG_X86
+	if (unprotect)
+		set_memory_rw(((unsigned long)dst) & PAGE_MASK, 1);
+#endif
 	ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
+#ifdef CONFIG_X86
+	if (unprotect)
+		set_memory_ro(((unsigned long)dst) & PAGE_MASK, 1);
+#endif
 	pagefault_enable();
 	set_fs(old_fs);
 

Re: kgdb test suite failure

[Thomas Gleixner]

On Wed, 21 May 2008, Jason Wessel wrote:
> An RFC type patch follows which addresses the problem, but in my
> opinion not in a terribly clean way...  Perhaps other folks will
> comment on a different or better way to approach the problem or chime
> in on if it should be fixed at all.  Certainly the test suite could be
> changed to use HW breakpoints for the case where the text segments are
> going to be read only as an example.

Please use HW breakpoints in the test suite. The RFC patch looks
horrible :)

Thanks,
	tglx

Re: kgdb test suite failure

[David Miller]

From: Thomas Gleixner <tglx@linutronix.de>
Date: Thu, 22 May 2008 23:14:23 +0200 (CEST)

> Please use HW breakpoints in the test suite. The RFC patch looks
> horrible :)

ftrace needs basically the same kind of facility as kgdb
needs here.  So bypassing the need for kgdb won't get around
the problem in general.

Re: kgdb test suite failure

[Jason Wessel]

David Miller wrote:
> From: Thomas Gleixner <tglx@linutronix.de>
> Date: Thu, 22 May 2008 23:14:23 +0200 (CEST)
>
>> Please use HW breakpoints in the test suite. The RFC patch looks
>> horrible :)
>
> ftrace needs basically the same kind of facility as kgdb
> needs here. So bypassing the need for kgdb won't get around
> the problem in general.

It does seem like there should be a general way to force a controlled
write to a read-only page. That would certainly allow for the use of
software breakpoints with KGDB. At the point in time that such calls
come into existence I would certainly consider adding a toggle which
you can use from the debugger to provide such access.

At least for the current time for kgdb, we will use hardware
breakpoints.

Jason.

Re: kgdb test suite failure

[Jason Wessel]

Thomas Gleixner wrote:
> On Wed, 21 May 2008, Jason Wessel wrote:
>> An RFC type patch follows which addresses the problem, but in my
>> opinion not in a terribly clean way...  Perhaps other folks will
>> comment on a different or better way to approach the problem or chime
>> in on if it should be fixed at all.  Certainly the test suite could be
>> changed to use HW breakpoints for the case where the text segments are
>> going to be read only as an example.
> 
> Please use HW breakpoints in the test suite. The RFC patch looks
> horrible :)
> 
> Thanks,
> 	tglx


I believe the kgdb tests should pass for this configuration, and the
tests have been adjusted to support boards that do and don't have
working hardware breakpoints.

I queued the patch below into the for_linus branch for the 2.6.26
tree.

Jason.


---
From: Jason Wessel <jason.wessel@windriver.com>
Subject: kgdbts: Use HW breakpoints with CONFIG_DEBUG_RODATA

Whenever CONFIG_DEBUG_RODATA is set in the kernel config many kernel
text sections become read-only, and the use of software breakpoints in
the kgdb tests will cause the kernel to fail to complete the start up.

Until such time that there is an official API for modifying read-only
text sections hardware breakpoints must be used to run the do_fork or
sys_open tests or the tests get skipped.

Signed-off-by: Jason Wessel <jason.wessel@windriver.com>

---
 drivers/misc/kgdbts.c |   27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

--- a/drivers/misc/kgdbts.c
+++ b/drivers/misc/kgdbts.c
@@ -130,6 +130,8 @@ static int repeat_test;
 static int test_complete;
 static int send_ack;
 static int final_ack;
+static int force_hwbrks;
+static int hwbreaks_ok;
 static int hw_break_val;
 static int hw_break_val2;
 #if defined(CONFIG_ARM) || defined(CONFIG_MIPS) || defined(CONFIG_SPARC)
@@ -233,12 +235,12 @@ static void break_helper(char *bp_type, 
 
 static void sw_break(char *arg)
 {
-	break_helper("Z0", arg, 0);
+	break_helper(force_hwbrks ? "Z1" : "Z0", arg, 0);
 }
 
 static void sw_rem_break(char *arg)
 {
-	break_helper("z0", arg, 0);
+	break_helper(force_hwbrks ? "z1" : "z0", arg, 0);
 }
 
 static void hw_break(char *arg)
@@ -780,6 +782,8 @@ static void run_breakpoint_test(int is_h
 		return;
 
 	eprintk("kgdbts: ERROR %s test failed\n", ts.name);
+	if (is_hw_breakpoint)
+		hwbreaks_ok = 0;
 }
 
 static void run_hw_break_test(int is_write_test)
@@ -797,9 +801,11 @@ static void run_hw_break_test(int is_wri
 	kgdb_breakpoint();
 	hw_break_val_access();
 	if (is_write_test) {
-		if (test_complete == 2)
+		if (test_complete == 2) {
 			eprintk("kgdbts: ERROR %s broke on access\n",
 				ts.name);
+			hwbreaks_ok = 0;
+		}
 		hw_break_val_write();
 	}
 	kgdb_breakpoint();
@@ -808,6 +814,7 @@ static void run_hw_break_test(int is_wri
 		return;
 
 	eprintk("kgdbts: ERROR %s test failed\n", ts.name);
+	hwbreaks_ok = 0;
 }
 
 static void run_nmi_sleep_test(int nmi_sleep)
@@ -911,6 +918,7 @@ static void kgdbts_run_tests(void)
 
 	/* All HW break point tests */
 	if (arch_kgdb_ops.flags & KGDB_HW_BREAKPOINT) {
+		hwbreaks_ok = 1;
 		v1printk("kgdbts:RUN hw breakpoint test\n");
 		run_breakpoint_test(1);
 		v1printk("kgdbts:RUN hw write breakpoint test\n");
@@ -924,6 +932,19 @@ static void kgdbts_run_tests(void)
 		run_nmi_sleep_test(nmi_sleep);
 	}
 
+#ifdef CONFIG_DEBUG_RODATA
+	/* Until there is an api to write to read-only text segments, use
+	 * HW breakpoints for the remainder of any tests, else print a
+	 * failure message if hw breakpoints do not work.
+	 */
+	if (!(arch_kgdb_ops.flags & KGDB_HW_BREAKPOINT && hwbreaks_ok)) {
+		eprintk("kgdbts: HW breakpoints do not work,"
+			"skipping remaining tests\n");
+		return;
+	}
+	force_hwbrks = 1;
+#endif /* CONFIG_DEBUG_RODATA */
+
 	/* If the do_fork test is run it will be the last test that is
 	 * executed because a kernel thread will be spawned at the very
 	 * end to unregister the debug hooks.