ARM + jprobes/kretprobes SEGV/hangs/OOPS in 2.6.29 kernel

ARM + jprobes/kretprobes SEGV/hangs/OOPS in 2.6.29 kernel

[venki kaps]

Hi,

I have been tracing kernel system call information using
the Kprobes/jprobes/kretprobes implementation in the 2.6.29 kernel on
ARM architecture.
Although the mainline kprobe/jprobe/kretprobe examples are working
fine (do_fork),
I have been facing some issues while running my own jprobe/kretprobe tests.

For instance, when I am planting jprobe at
do_execve/sys_open/sys_close/sys_read/sys_write/,
after registration of probe, segmentation fault is getting generated.
The following is my code snapshot-

static int
jdo_execve(char * filename,
                char __user *__user *argv,
                char __user *__user *envp,
                struct pt_regs * regs)

{
		jprobe_return();
		return 0;
}

static struct jprobe my_jprobe = {
	.entry                          = jdo_execve,
                .kp = {
                         .symbol_name    = "do_execve",
               },

};

static int __init jprobe_init(void) {
	
	if ((register_jprobe(&my_jprobe)) < 0) {
		printk("%s %d,register_jprobe failed, returned\n", __FILE__, __LINE__);
		return -1;
	}
	printk("%s %d,Planted jprobe at %p, handler addr %p\n", __FILE__, __LINE__,
		 my_jprobe.kp.addr, my_jprobe.entry);

	return 0;
}

static void __exit jprobe_exit(void) {
	unregister_jprobe(&my_jprobe);
	printk("%s %d,jprobe unregistered\n", __FILE__, __LINE__);
}

And also system hangs for sys_open/sys_close/sys_read/sys_write with
jprobes/kretprobes
but there are no issues with kprobes.

Query:
=====
	- Are there any limitations for jptobes/kretporbes in mainline kernel for ARM?
	- Why it works for only do_fork and why not for others
(do_execve/sys_open/sys_close/sys_read/sys_write)?
	- Is it required any additional setup to achieve this?

Please give your valuable comments.

Thanks in advance.

Best regards,
Venkappa

Re: ARM + jprobes/kretprobes SEGV/hangs/OOPS in 2.6.29 kernel

[Nicolas Pitre]

On Tue, 25 Aug 2009, venki kaps wrote:

> Hi,
> 
> I have been tracing kernel system call information using
> the Kprobes/jprobes/kretprobes implementation in the 2.6.29 kernel on
> ARM architecture.
> Although the mainline kprobe/jprobe/kretprobe examples are working
> fine (do_fork),
> I have been facing some issues while running my own jprobe/kretprobe tests.
> 
[...]
> 
> Query:
> =====
> 	- Are there any limitations for jptobes/kretporbes in mainline kernel for ARM?

No limitation in particular that I know of.

> 	- Why it works for only do_fork and why not for others
> (do_execve/sys_open/sys_close/sys_read/sys_write)?

I don't know.  Will try to have a look.

> 	- Is it required any additional setup to achieve this?

Not supposed to need anything special.


Nicolas

Re: ARM + jprobes/kretprobes SEGV/hangs/OOPS in 2.6.29 kernel

[venki kaps]

Hi,

I have further investigated with respect to this issue and found the
problem with
'dump_stack()' which calls in my sample kprobe,kretprobe and jprobe
modules to get
the stack dump.

Here it is giving sample example module which covers all the probes.

Sample module test program:
---------------------------
#include <linux/module.h>
#include <linux/init.h>
#include <linux/kprobes.h>

static int k_count1 = 0;
static int k_count2 = 0;
static int k_count3 = 0;
static int k_count4 = 0;

/* Proxy routine having the same arguments as actual sys_open() routine */
long jsys_open(unsigned int fd, char __user * buf, size_t count)
{
        printk("%s %d, Proxy sys_open, arguments are %d, %d\n",
__FILE__, __LINE__, fd, count);
        printk("%s %d, Stack_dump :\n", __FILE__, __LINE__);
        dump_stack();
        /* Always end with a call to jprobe_return(). */
        k_count1++;
        jprobe_return();
        /* NOTREACHED */
        return 0;
}

static struct jprobe my_jprobe = {
        .entry = JPROBE_ENTRY(jsys_open)
};

static const char *probed_func = "sys_open";

/* Return-probe handler: Log the return value from the probed function. */
static int ret_handler(struct kretprobe_instance *ri, struct pt_regs *regs)
{
        int retval = regs_return_value(regs);

        printk("%s %d, %s returns %d\n", __FILE__, __LINE__,
probed_func, retval);
        printk("%s %d, Stack_dump :\n", __FILE__, __LINE__);
        dump_stack();
        k_count2++;
        return 0;
}

static struct kretprobe my_kretprobe = {
        .handler = ret_handler,
        /* Probe up to 20 instances concurrently. */
        .maxactive = 20
};

static struct kprobe k_001_kpr;

static int k_001_before_hook(struct kprobe *k_001_kpr, struct pt_regs *p)
{
        printk("%s %d\nStack dump for the kprobe pre handler for
instruction at %p\n", __FILE__, __LINE__, k_001_kpr->addr);
        dump_stack();
        k_count3++;
        return 0;
}

void k_001_after_hook(struct kprobe *k_001_kpr, struct pt_regs *p,
unsigned long flags)
{
        printk("%s %d\nStack dump for the kprobe post handler at
%p\n", __FILE__, __LINE__, k_001_kpr->addr);
        dump_stack();
        printk("%s %d, The Registers are:\n", __FILE__, __LINE__);
        k_count4++;
}


static int __init k_001_init_probe(void)
{
        int ret;
        int retj;
        printk("%s %d\nInserting the kprobe for sys_open\n", __FILE__,
__LINE__);

        /* Registering a kprobe */
        k_001_kpr.pre_handler = (kprobe_pre_handler_t) k_001_before_hook;
        k_001_kpr.post_handler = (kprobe_post_handler_t) k_001_after_hook;
        k_001_kpr.symbol_name = "sys_open", __FILE__, __LINE__;
        if (register_kprobe(&k_001_kpr) < 0) {
                printk("%s %dk-001.c:register_kprobe is failed\n",
__FILE__, __LINE__);
                return -1;
        }
        printk("%s %d, register_kprobe is successful\n", __FILE__, __LINE__);

        printk("%s %d, Inserting the kretprobe for sys_open\n",
__FILE__, __LINE__);
        my_kretprobe.kp.symbol_name = (char *)probed_func;

        if ((ret = register_kretprobe(&my_kretprobe)) < 0) {
                printk("%s %d, register_kretprobe failed, returned
%d\n", __FILE__, __LINE__, ret);
                return -1;
        }
        printk("%s %d, Planted return probe for sys_open at %p\n",
__FILE__, __LINE__, my_kretprobe.kp.addr);

        my_jprobe.kp.symbol_name = "sys_open";

        if ((retj = register_jprobe(&my_jprobe)) < 0) {
                printk("%s %d,register_jprobe failed, returned %d\n",
__FILE__, __LINE__, ret);
                return -1;
        }
        printk("%s %d,Planted jprobe at %p, handler addr %p\n",
__FILE__, __LINE__, my_jprobe.kp.addr, my_jprobe.entry);

        return 0;
}

static void __exit k_001_exit_probe(void)
{
        unregister_kprobe(&k_001_kpr);
        printk("%s %d\nkprobe unregistered from sys_open \n",
__FILE__, __LINE__);

        unregister_kretprobe(&my_kretprobe);
        printk("%s %dkretprobe unregistered\n", __FILE__, __LINE__);
        /* nmissed > 0 suggests that maxactive was set too low. */
        printk("%s %dMissed probing %d instances of %s\n", __FILE__,
__LINE__, my_kretprobe.nmissed, probed_func);

        unregister_jprobe(&my_jprobe);
        printk("%s %d,jprobe unregistered\n", __FILE__, __LINE__);

        if (k_count1 > 0 && k_count2 > 0 && k_count3 > 0 && k_count4 > 0)
                printk("TEST PASS");
        else
                printk("TEST FAIL");
}

module_init(k_001_init_probe);
module_exit(k_001_exit_probe);

MODULE_DESCRIPTION("Kprobes test module");
MODULE_LICENSE("GPL");

I have tested the above program and got result as system hang.

Disable dump_stack:
------------------------------
I have disabled dump_stack(ARM specific) in the above program
and did not notice any problem.

After disabling the dump_stack(), all the probes are working
fine for 'do_fork,sys_open and sys_close system calls.

ARM dump_stack implementation has been changed in 2.6.29 kernel:
----------------------------------------------------------------------------------------------------
I have found in 2.6.29 kernel the current ARM dump_stack implementation
has been changed.Due to that change the kprobes,jprobes and kretprobes
are failing for 'do_fork,sys_open and sys_close' system calls.

Current dump_stack implementation:

Location: arch/arm/kernel/traps.c

void dump_stack(void)
{
        dump_backtrace(NULL, NULL);
}


I have just reverted back the above source to old kernel implementation.

Index: b/arch/arm/kernel/traps.c
===================================================================
--- a/arch/arm/kernel/traps.c
+++ b/arch/arm/kernel/traps.c
@@ -202,7 +202,11 @@ static void dump_backtrace(struct pt_reg

 void dump_stack(void)
 {
+#if 0
        dump_backtrace(NULL, NULL);
+#else
+       __backtrace();
+#endif
 }

 EXPORT_SYMBOL(dump_stack);


After the above change,All probes are working fine with
enabling of dump_stack() in my sample modules.

I have some queries with repect to the above changes.

Queries:
========
1) Revert back the old kernel implementation might not be a good solution.
	I anticipate it needs to be fixed in 2.6.29 kernel implementations.
	Am i right/wrong.
	
2) Shall i avoid by calling dump_stack() in my sample test modules?

3) Only do_fork, sys_open and sys_close system calls are working fine with
	dump_stack() but still 'SEGV,sysetm hangs' for do_execve,do_gettimeofday,
	sys_gettimeofday,sys_read,sys_write, etc system calls.

I have been further investigating with respect to the above issues
Meanwhile could you please provide the inputs with respect to the above queries?

Thanks in advance.

Best regards,
Venkappa


On Tue, Aug 25, 2009 at 11:07 PM, Nicolas Pitre<nico@cam.org> wrote:
> On Tue, 25 Aug 2009, venki kaps wrote:
>
>> Hi,
>>
>> I have been tracing kernel system call information using
>> the Kprobes/jprobes/kretprobes implementation in the 2.6.29 kernel on
>> ARM architecture.
>> Although the mainline kprobe/jprobe/kretprobe examples are working
>> fine (do_fork),
>> I have been facing some issues while running my own jprobe/kretprobe tests.
>>
> [...]
>>
>> Query:
>> =====
>>       - Are there any limitations for jptobes/kretporbes in mainline kernel for ARM?
>
> No limitation in particular that I know of.
>
>>       - Why it works for only do_fork and why not for others
>> (do_execve/sys_open/sys_close/sys_read/sys_write)?
>
> I don't know.  Will try to have a look.
>
>>       - Is it required any additional setup to achieve this?
>
> Not supposed to need anything special.
>
>
> Nicolas
>

Re: ARM + jprobes/kretprobes SEGV/hangs/OOPS in 2.6.29 kernel

[venki kaps]

Hi,

After dump_stack change, most of the system calls
(do_fork,do_execve,do_gettimeofday,
sys_gettimeofday,sys_open,sys_close, sys_read,sys_write) are working
fine with kprobes
and kretprobes.

But in jprobes, still 'SEGV,sysetm hangs,OOPS' will be getting for 'do_execve,
do_gettimeofday,sys_gettimeofday,sys_close, sys_read,sys_write' system calls.

 I have surprised with jprobes which are working well for
'do_fork,sys_open' system calls.

Best Regards,
Venkappa

On Mon, Aug 31, 2009 at 10:57 AM, venki kaps<venkiece2005@gmail.com> wrote:
> Hi,
>
> I have further investigated with respect to this issue and found the
> problem with
> 'dump_stack()' which calls in my sample kprobe,kretprobe and jprobe
> modules to get
> the stack dump.
>
> Here it is giving sample example module which covers all the probes.
>
> Sample module test program:
> ---------------------------
> #include <linux/module.h>
> #include <linux/init.h>
> #include <linux/kprobes.h>
>
> static int k_count1 = 0;
> static int k_count2 = 0;
> static int k_count3 = 0;
> static int k_count4 = 0;
>
> /* Proxy routine having the same arguments as actual sys_open() routine */
> long jsys_open(unsigned int fd, char __user * buf, size_t count)
> {
>        printk("%s %d, Proxy sys_open, arguments are %d, %d\n",
> __FILE__, __LINE__, fd, count);
>        printk("%s %d, Stack_dump :\n", __FILE__, __LINE__);
>        dump_stack();
>        /* Always end with a call to jprobe_return(). */
>        k_count1++;
>        jprobe_return();
>        /* NOTREACHED */
>        return 0;
> }
>
> static struct jprobe my_jprobe = {
>        .entry = JPROBE_ENTRY(jsys_open)
> };
>
> static const char *probed_func = "sys_open";
>
> /* Return-probe handler: Log the return value from the probed function. */
> static int ret_handler(struct kretprobe_instance *ri, struct pt_regs *regs)
> {
>        int retval = regs_return_value(regs);
>
>        printk("%s %d, %s returns %d\n", __FILE__, __LINE__,
> probed_func, retval);
>        printk("%s %d, Stack_dump :\n", __FILE__, __LINE__);
>        dump_stack();
>        k_count2++;
>        return 0;
> }
>
> static struct kretprobe my_kretprobe = {
>        .handler = ret_handler,
>        /* Probe up to 20 instances concurrently. */
>        .maxactive = 20
> };
>
> static struct kprobe k_001_kpr;
>
> static int k_001_before_hook(struct kprobe *k_001_kpr, struct pt_regs *p)
> {
>        printk("%s %d\nStack dump for the kprobe pre handler for
> instruction at %p\n", __FILE__, __LINE__, k_001_kpr->addr);
>        dump_stack();
>        k_count3++;
>        return 0;
> }
>
> void k_001_after_hook(struct kprobe *k_001_kpr, struct pt_regs *p,
> unsigned long flags)
> {
>        printk("%s %d\nStack dump for the kprobe post handler at
> %p\n", __FILE__, __LINE__, k_001_kpr->addr);
>        dump_stack();
>        printk("%s %d, The Registers are:\n", __FILE__, __LINE__);
>        k_count4++;
> }
>
>
> static int __init k_001_init_probe(void)
> {
>        int ret;
>        int retj;
>        printk("%s %d\nInserting the kprobe for sys_open\n", __FILE__,
> __LINE__);
>
>        /* Registering a kprobe */
>        k_001_kpr.pre_handler = (kprobe_pre_handler_t) k_001_before_hook;
>        k_001_kpr.post_handler = (kprobe_post_handler_t) k_001_after_hook;
>        k_001_kpr.symbol_name = "sys_open", __FILE__, __LINE__;
>        if (register_kprobe(&k_001_kpr) < 0) {
>                printk("%s %dk-001.c:register_kprobe is failed\n",
> __FILE__, __LINE__);
>                return -1;
>        }
>        printk("%s %d, register_kprobe is successful\n", __FILE__, __LINE__);
>
>        printk("%s %d, Inserting the kretprobe for sys_open\n",
> __FILE__, __LINE__);
>        my_kretprobe.kp.symbol_name = (char *)probed_func;
>
>        if ((ret = register_kretprobe(&my_kretprobe)) < 0) {
>                printk("%s %d, register_kretprobe failed, returned
> %d\n", __FILE__, __LINE__, ret);
>                return -1;
>        }
>        printk("%s %d, Planted return probe for sys_open at %p\n",
> __FILE__, __LINE__, my_kretprobe.kp.addr);
>
>        my_jprobe.kp.symbol_name = "sys_open";
>
>        if ((retj = register_jprobe(&my_jprobe)) < 0) {
>                printk("%s %d,register_jprobe failed, returned %d\n",
> __FILE__, __LINE__, ret);
>                return -1;
>        }
>        printk("%s %d,Planted jprobe at %p, handler addr %p\n",
> __FILE__, __LINE__, my_jprobe.kp.addr, my_jprobe.entry);
>
>        return 0;
> }
>
> static void __exit k_001_exit_probe(void)
> {
>        unregister_kprobe(&k_001_kpr);
>        printk("%s %d\nkprobe unregistered from sys_open \n",
> __FILE__, __LINE__);
>
>        unregister_kretprobe(&my_kretprobe);
>        printk("%s %dkretprobe unregistered\n", __FILE__, __LINE__);
>        /* nmissed > 0 suggests that maxactive was set too low. */
>        printk("%s %dMissed probing %d instances of %s\n", __FILE__,
> __LINE__, my_kretprobe.nmissed, probed_func);
>
>        unregister_jprobe(&my_jprobe);
>        printk("%s %d,jprobe unregistered\n", __FILE__, __LINE__);
>
>        if (k_count1 > 0 && k_count2 > 0 && k_count3 > 0 && k_count4 > 0)
>                printk("TEST PASS");
>        else
>                printk("TEST FAIL");
> }
>
> module_init(k_001_init_probe);
> module_exit(k_001_exit_probe);
>
> MODULE_DESCRIPTION("Kprobes test module");
> MODULE_LICENSE("GPL");
>
> I have tested the above program and got result as system hang.
>
> Disable dump_stack:
> ------------------------------
> I have disabled dump_stack(ARM specific) in the above program
> and did not notice any problem.
>
> After disabling the dump_stack(), all the probes are working
> fine for 'do_fork,sys_open and sys_close system calls.
>
> ARM dump_stack implementation has been changed in 2.6.29 kernel:
> ----------------------------------------------------------------------------------------------------
> I have found in 2.6.29 kernel the current ARM dump_stack implementation
> has been changed.Due to that change the kprobes,jprobes and kretprobes
> are failing for 'do_fork,sys_open and sys_close' system calls.
>
> Current dump_stack implementation:
>
> Location: arch/arm/kernel/traps.c
>
> void dump_stack(void)
> {
>        dump_backtrace(NULL, NULL);
> }
>
>
> I have just reverted back the above source to old kernel implementation.
>
> Index: b/arch/arm/kernel/traps.c
> ===================================================================
> --- a/arch/arm/kernel/traps.c
> +++ b/arch/arm/kernel/traps.c
> @@ -202,7 +202,11 @@ static void dump_backtrace(struct pt_reg
>
>  void dump_stack(void)
>  {
> +#if 0
>        dump_backtrace(NULL, NULL);
> +#else
> +       __backtrace();
> +#endif
>  }
>
>  EXPORT_SYMBOL(dump_stack);
>
>
> After the above change,All probes are working fine with
> enabling of dump_stack() in my sample modules.
>
> I have some queries with repect to the above changes.
>
> Queries:
> ========
> 1) Revert back the old kernel implementation might not be a good solution.
>        I anticipate it needs to be fixed in 2.6.29 kernel implementations.
>        Am i right/wrong.
>
> 2) Shall i avoid by calling dump_stack() in my sample test modules?
>
> 3) Only do_fork, sys_open and sys_close system calls are working fine with
>        dump_stack() but still 'SEGV,sysetm hangs' for do_execve,do_gettimeofday,
>        sys_gettimeofday,sys_read,sys_write, etc system calls.
>
> I have been further investigating with respect to the above issues
> Meanwhile could you please provide the inputs with respect to the above queries?
>
> Thanks in advance.
>
> Best regards,
> Venkappa
>
>
> On Tue, Aug 25, 2009 at 11:07 PM, Nicolas Pitre<nico@cam.org> wrote:
>> On Tue, 25 Aug 2009, venki kaps wrote:
>>
>>> Hi,
>>>
>>> I have been tracing kernel system call information using
>>> the Kprobes/jprobes/kretprobes implementation in the 2.6.29 kernel on
>>> ARM architecture.
>>> Although the mainline kprobe/jprobe/kretprobe examples are working
>>> fine (do_fork),
>>> I have been facing some issues while running my own jprobe/kretprobe tests.
>>>
>> [...]
>>>
>>> Query:
>>> =====
>>>       - Are there any limitations for jptobes/kretporbes in mainline kernel for ARM?
>>
>> No limitation in particular that I know of.
>>
>>>       - Why it works for only do_fork and why not for others
>>> (do_execve/sys_open/sys_close/sys_read/sys_write)?
>>
>> I don't know.  Will try to have a look.
>>
>>>       - Is it required any additional setup to achieve this?
>>
>> Not supposed to need anything special.
>>
>>
>> Nicolas
>>
>

Re: ARM + jprobes/kretprobes SEGV/hangs/OOPS in 2.6.29 kernel

[venki kaps]

Hi,

I have found the exact problem with respect to ARM jprobes.

The problem with configure i.e, CONFIG_ARM_UNWIND = y; is enabled.

Due to  the stack unwinding, the jprobe is not able to return since
Where jprobe starts getting 'system call"  is when you have exceptions
or setjmp/longjmp
which simply unwinding the stack without returning.

After disabling (CONFIG_ARM_UNWIND = n;), all probes are working fine.

The dump_stack() is also not an issue after the above change. Please ignore the
previous mail.

Finally there are no issues with ARM kprobes,jprobes and return probes
But 'CONFIG_ARM_UNWIND' should be disabled for jprobes testing.


Best regards,
Venkappa




On Mon, Aug 31, 2009 at 3:41 PM, venki kaps<venkiece2005@gmail.com> wrote:
> Hi,
>
> After dump_stack change, most of the system calls
> (do_fork,do_execve,do_gettimeofday,
> sys_gettimeofday,sys_open,sys_close, sys_read,sys_write) are working
> fine with kprobes
> and kretprobes.
>
> But in jprobes, still 'SEGV,sysetm hangs,OOPS' will be getting for 'do_execve,
> do_gettimeofday,sys_gettimeofday,sys_close, sys_read,sys_write' system calls.
>
>  I have surprised with jprobes which are working well for
> 'do_fork,sys_open' system calls.
>
> Best Regards,
> Venkappa
>
> On Mon, Aug 31, 2009 at 10:57 AM, venki kaps<venkiece2005@gmail.com> wrote:
>> Hi,
>>
>> I have further investigated with respect to this issue and found the
>> problem with
>> 'dump_stack()' which calls in my sample kprobe,kretprobe and jprobe
>> modules to get
>> the stack dump.
>>
>> Here it is giving sample example module which covers all the probes.
>>
>> Sample module test program:
>> ---------------------------
>> #include <linux/module.h>
>> #include <linux/init.h>
>> #include <linux/kprobes.h>
>>
>> static int k_count1 = 0;
>> static int k_count2 = 0;
>> static int k_count3 = 0;
>> static int k_count4 = 0;
>>
>> /* Proxy routine having the same arguments as actual sys_open() routine */
>> long jsys_open(unsigned int fd, char __user * buf, size_t count)
>> {
>>        printk("%s %d, Proxy sys_open, arguments are %d, %d\n",
>> __FILE__, __LINE__, fd, count);
>>        printk("%s %d, Stack_dump :\n", __FILE__, __LINE__);
>>        dump_stack();
>>        /* Always end with a call to jprobe_return(). */
>>        k_count1++;
>>        jprobe_return();
>>        /* NOTREACHED */
>>        return 0;
>> }
>>
>> static struct jprobe my_jprobe = {
>>        .entry = JPROBE_ENTRY(jsys_open)
>> };
>>
>> static const char *probed_func = "sys_open";
>>
>> /* Return-probe handler: Log the return value from the probed function. */
>> static int ret_handler(struct kretprobe_instance *ri, struct pt_regs *regs)
>> {
>>        int retval = regs_return_value(regs);
>>
>>        printk("%s %d, %s returns %d\n", __FILE__, __LINE__,
>> probed_func, retval);
>>        printk("%s %d, Stack_dump :\n", __FILE__, __LINE__);
>>        dump_stack();
>>        k_count2++;
>>        return 0;
>> }
>>
>> static struct kretprobe my_kretprobe = {
>>        .handler = ret_handler,
>>        /* Probe up to 20 instances concurrently. */
>>        .maxactive = 20
>> };
>>
>> static struct kprobe k_001_kpr;
>>
>> static int k_001_before_hook(struct kprobe *k_001_kpr, struct pt_regs *p)
>> {
>>        printk("%s %d\nStack dump for the kprobe pre handler for
>> instruction at %p\n", __FILE__, __LINE__, k_001_kpr->addr);
>>        dump_stack();
>>        k_count3++;
>>        return 0;
>> }
>>
>> void k_001_after_hook(struct kprobe *k_001_kpr, struct pt_regs *p,
>> unsigned long flags)
>> {
>>        printk("%s %d\nStack dump for the kprobe post handler at
>> %p\n", __FILE__, __LINE__, k_001_kpr->addr);
>>        dump_stack();
>>        printk("%s %d, The Registers are:\n", __FILE__, __LINE__);
>>        k_count4++;
>> }
>>
>>
>> static int __init k_001_init_probe(void)
>> {
>>        int ret;
>>        int retj;
>>        printk("%s %d\nInserting the kprobe for sys_open\n", __FILE__,
>> __LINE__);
>>
>>        /* Registering a kprobe */
>>        k_001_kpr.pre_handler = (kprobe_pre_handler_t) k_001_before_hook;
>>        k_001_kpr.post_handler = (kprobe_post_handler_t) k_001_after_hook;
>>        k_001_kpr.symbol_name = "sys_open", __FILE__, __LINE__;
>>        if (register_kprobe(&k_001_kpr) < 0) {
>>                printk("%s %dk-001.c:register_kprobe is failed\n",
>> __FILE__, __LINE__);
>>                return -1;
>>        }
>>        printk("%s %d, register_kprobe is successful\n", __FILE__, __LINE__);
>>
>>        printk("%s %d, Inserting the kretprobe for sys_open\n",
>> __FILE__, __LINE__);
>>        my_kretprobe.kp.symbol_name = (char *)probed_func;
>>
>>        if ((ret = register_kretprobe(&my_kretprobe)) < 0) {
>>                printk("%s %d, register_kretprobe failed, returned
>> %d\n", __FILE__, __LINE__, ret);
>>                return -1;
>>        }
>>        printk("%s %d, Planted return probe for sys_open at %p\n",
>> __FILE__, __LINE__, my_kretprobe.kp.addr);
>>
>>        my_jprobe.kp.symbol_name = "sys_open";
>>
>>        if ((retj = register_jprobe(&my_jprobe)) < 0) {
>>                printk("%s %d,register_jprobe failed, returned %d\n",
>> __FILE__, __LINE__, ret);
>>                return -1;
>>        }
>>        printk("%s %d,Planted jprobe at %p, handler addr %p\n",
>> __FILE__, __LINE__, my_jprobe.kp.addr, my_jprobe.entry);
>>
>>        return 0;
>> }
>>
>> static void __exit k_001_exit_probe(void)
>> {
>>        unregister_kprobe(&k_001_kpr);
>>        printk("%s %d\nkprobe unregistered from sys_open \n",
>> __FILE__, __LINE__);
>>
>>        unregister_kretprobe(&my_kretprobe);
>>        printk("%s %dkretprobe unregistered\n", __FILE__, __LINE__);
>>        /* nmissed > 0 suggests that maxactive was set too low. */
>>        printk("%s %dMissed probing %d instances of %s\n", __FILE__,
>> __LINE__, my_kretprobe.nmissed, probed_func);
>>
>>        unregister_jprobe(&my_jprobe);
>>        printk("%s %d,jprobe unregistered\n", __FILE__, __LINE__);
>>
>>        if (k_count1 > 0 && k_count2 > 0 && k_count3 > 0 && k_count4 > 0)
>>                printk("TEST PASS");
>>        else
>>                printk("TEST FAIL");
>> }
>>
>> module_init(k_001_init_probe);
>> module_exit(k_001_exit_probe);
>>
>> MODULE_DESCRIPTION("Kprobes test module");
>> MODULE_LICENSE("GPL");
>>
>> I have tested the above program and got result as system hang.
>>
>> Disable dump_stack:
>> ------------------------------
>> I have disabled dump_stack(ARM specific) in the above program
>> and did not notice any problem.
>>
>> After disabling the dump_stack(), all the probes are working
>> fine for 'do_fork,sys_open and sys_close system calls.
>>
>> ARM dump_stack implementation has been changed in 2.6.29 kernel:
>> ----------------------------------------------------------------------------------------------------
>> I have found in 2.6.29 kernel the current ARM dump_stack implementation
>> has been changed.Due to that change the kprobes,jprobes and kretprobes
>> are failing for 'do_fork,sys_open and sys_close' system calls.
>>
>> Current dump_stack implementation:
>>
>> Location: arch/arm/kernel/traps.c
>>
>> void dump_stack(void)
>> {
>>        dump_backtrace(NULL, NULL);
>> }
>>
>>
>> I have just reverted back the above source to old kernel implementation.
>>
>> Index: b/arch/arm/kernel/traps.c
>> ===================================================================
>> --- a/arch/arm/kernel/traps.c
>> +++ b/arch/arm/kernel/traps.c
>> @@ -202,7 +202,11 @@ static void dump_backtrace(struct pt_reg
>>
>>  void dump_stack(void)
>>  {
>> +#if 0
>>        dump_backtrace(NULL, NULL);
>> +#else
>> +       __backtrace();
>> +#endif
>>  }
>>
>>  EXPORT_SYMBOL(dump_stack);
>>
>>
>> After the above change,All probes are working fine with
>> enabling of dump_stack() in my sample modules.
>>
>> I have some queries with repect to the above changes.
>>
>> Queries:
>> ========
>> 1) Revert back the old kernel implementation might not be a good solution.
>>        I anticipate it needs to be fixed in 2.6.29 kernel implementations.
>>        Am i right/wrong.
>>
>> 2) Shall i avoid by calling dump_stack() in my sample test modules?
>>
>> 3) Only do_fork, sys_open and sys_close system calls are working fine with
>>        dump_stack() but still 'SEGV,sysetm hangs' for do_execve,do_gettimeofday,
>>        sys_gettimeofday,sys_read,sys_write, etc system calls.
>>
>> I have been further investigating with respect to the above issues
>> Meanwhile could you please provide the inputs with respect to the above queries?
>>
>> Thanks in advance.
>>
>> Best regards,
>> Venkappa
>>
>>
>> On Tue, Aug 25, 2009 at 11:07 PM, Nicolas Pitre<nico@cam.org> wrote:
>>> On Tue, 25 Aug 2009, venki kaps wrote:
>>>
>>>> Hi,
>>>>
>>>> I have been tracing kernel system call information using
>>>> the Kprobes/jprobes/kretprobes implementation in the 2.6.29 kernel on
>>>> ARM architecture.
>>>> Although the mainline kprobe/jprobe/kretprobe examples are working
>>>> fine (do_fork),
>>>> I have been facing some issues while running my own jprobe/kretprobe tests.
>>>>
>>> [...]
>>>>
>>>> Query:
>>>> =====
>>>>       - Are there any limitations for jptobes/kretporbes in mainline kernel for ARM?
>>>
>>> No limitation in particular that I know of.
>>>
>>>>       - Why it works for only do_fork and why not for others
>>>> (do_execve/sys_open/sys_close/sys_read/sys_write)?
>>>
>>> I don't know.  Will try to have a look.
>>>
>>>>       - Is it required any additional setup to achieve this?
>>>
>>> Not supposed to need anything special.
>>>
>>>
>>> Nicolas
>>>
>>
>

Re: ARM + jprobes/kretprobes SEGV/hangs/OOPS in 2.6.29 kernel

[Catalin Marinas]

venki kaps <venkiece2005@gmail.com> wrote:
> I have found the exact problem with respect to ARM jprobes.
>
> The problem with configure i.e, CONFIG_ARM_UNWIND = y; is enabled.

I haven't followed the kprobes implementation for ARM but does it make
any assumptions about the existence of a frame pointer on the stack?
Enabling stack unwinding automatically disables the framepointer.

-- 
Catalin

Re: ARM + jprobes/kretprobes SEGV/hangs/OOPS in 2.6.29 kernel

[Russell King]

On Tue, Sep 01, 2009 at 02:54:54PM +0100, Catalin Marinas wrote:
> venki kaps <venkiece2005@gmail.com> wrote:
> > I have found the exact problem with respect to ARM jprobes.
> >
> > The problem with configure i.e, CONFIG_ARM_UNWIND = y; is enabled.
> 
> I haven't followed the kprobes implementation for ARM but does it make
> any assumptions about the existence of a frame pointer on the stack?
> Enabling stack unwinding automatically disables the framepointer.

If it uses CALLER_ADDRESSx() then it won't work with unwinding enabled.
See 5613/1 (which is pending in the devel branch.)

-- 
Russell King
 Linux kernel    2.6 ARM Linux   - http://www.arm.linux.org.uk/
 maintainer of:

Re: ARM + jprobes/kretprobes SEGV/hangs/OOPS in 2.6.29 kernel

[Catalin Marinas]

On Tue, 2009-09-01 at 15:25 +0100, Russell King wrote:
> On Tue, Sep 01, 2009 at 02:54:54PM +0100, Catalin Marinas wrote:
> > venki kaps <venkiece2005@gmail.com> wrote:
> > > I have found the exact problem with respect to ARM jprobes.
> > >
> > > The problem with configure i.e, CONFIG_ARM_UNWIND = y; is enabled.
> > 
> > I haven't followed the kprobes implementation for ARM but does it make
> > any assumptions about the existence of a frame pointer on the stack?
> > Enabling stack unwinding automatically disables the framepointer.
> 
> If it uses CALLER_ADDRESSx() then it won't work with unwinding enabled.
> See 5613/1 (which is pending in the devel branch.)

In addition to that, when CONFIG_FRAME_POINTER is disabled, the lr
register isn't always saved on the stack by the called function. I'm not
sure whether kretprobe_trampoline is aware of this.

-- 
Catalin

Re: ARM + jprobes/kretprobes SEGV/hangs/OOPS in 2.6.29 kernel

[Nicolas Pitre]

On Tue, 1 Sep 2009, Catalin Marinas wrote:

> On Tue, 2009-09-01 at 15:25 +0100, Russell King wrote:
> > On Tue, Sep 01, 2009 at 02:54:54PM +0100, Catalin Marinas wrote:
> > > venki kaps <venkiece2005@gmail.com> wrote:
> > > > I have found the exact problem with respect to ARM jprobes.
> > > >
> > > > The problem with configure i.e, CONFIG_ARM_UNWIND = y; is enabled.
> > > 
> > > I haven't followed the kprobes implementation for ARM but does it make
> > > any assumptions about the existence of a frame pointer on the stack?
> > > Enabling stack unwinding automatically disables the framepointer.
> > 
> > If it uses CALLER_ADDRESSx() then it won't work with unwinding enabled.
> > See 5613/1 (which is pending in the devel branch.)
> 
> In addition to that, when CONFIG_FRAME_POINTER is disabled, the lr
> register isn't always saved on the stack by the called function. I'm not
> sure whether kretprobe_trampoline is aware of this.

The way a kretprobe works is to put a trap at the very first instruction 
of the targetted function, preserve the value of LR when the trap is 
hit, and substitute it with the address of kretprobe_trampoline.  Then 
the original first instruction is emulated to pass over the trap point 
and normal execution is resumed.  So whether or not LR is then saved on 
the stack doesn't matter to kretprobe_trampoline as it will restore the 
LR value saved during the initial trap.

Of course if you end up generating a backtrace within a kretprobed 
function then the result might look funny.


Nicolas

Re: ARM + jprobes/kretprobes SEGV/hangs/OOPS in 2.6.29 kernel

[venki kaps]

I have done some more analysis with respect to ARM kretprobes
and describing as below,

Before disabling unwinding support,
i have tested mainlined sample kretprobes_example.c with some changes.
the changes are,

Index: b/samples/kprobes/kretprobe_example.c
===================================================================
--- a/samples/kprobes/kretprobe_example.c
+++ b/samples/kprobes/kretprobe_example.c
@@ -59,6 +59,9 @@ static int ret_handler(struct kretprobe_
        s64 delta;
        ktime_t now;

+       printk("%s %d, Stack_dump :\n", __FILE__, __LINE__);
+       dump_stack();
+
        now = ktime_get();
        delta = ktime_to_ns(ktime_sub(now, data->entry_stamp));
        printk(KERN_INFO "%s returned %d and took %lld ns to execute\n",

$ insmod kretprobe_example.ko
$ ls
    <---- system hang

As i mentioned jprobes and kretprobes are having some issues with backtrace
with enabling unwinding support and as we know well about ARM, unwinding tables
are not stabilized which was pending issue in present kernels.

After disabling unwinding support,
              Kernel hacking  --->
	                 -*- frame pointer support
	            [] Enable stack unwinding support

Based on the above settings, I have verified the above test with dump_stack and
noticed test result as PASS.

I wanted to rely on FRAME_POINTER So have just added
'''select FRAME_POINTER''' to kprobe related configs as below as,

Index: b/samples/Kconfig
===================================================================
--- a/samples/Kconfig
+++ b/samples/Kconfig
@@ -31,6 +31,7 @@ config SAMPLE_KOBJECT
 config SAMPLE_KPROBES
 	tristate "Build kprobes examples -- loadable modules only"
 	depends on KPROBES && m
+	select FRAME_POINTER
 	help
 	  This build several kprobes example modules.

@@ -38,6 +39,7 @@ config SAMPLE_KRETPROBES
 	tristate "Build kretprobes example -- loadable modules only"
 	default m
 	depends on SAMPLE_KPROBES && KRETPROBES
+	select FRAME_POINTER

 config SAMPLE_PSRWLOCK
 	tristate "Build psrwlock example -- loadable modules only"
Index: b/arch/Kconfig
===================================================================
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -36,6 +36,7 @@ config KPROBES
 	bool "Kprobes"
 	depends on KALLSYMS && MODULES
 	depends on HAVE_KPROBES
+	select FRAME_POINTER
 	help
 	  Kprobes allows you to trap at almost any kernel address and
 	  execute a callback function.  register_kprobe() establishes
@@ -68,6 +69,7 @@ config HAVE_SYSCALL_WRAPPERS
 config KRETPROBES
 	def_bool y
 	depends on KPROBES && HAVE_KRETPROBES
+	select FRAME_POINTER

 config HAVE_IOREMAP_PROT
 	bool

I have some queries,
- shall i rely on FRAME_POINTER in kprobe related configs?
- shall i proceed with above changes?

Best Regards,
Venkappa



On Tue, Sep 1, 2009 at 11:26 PM, Nicolas Pitre <nico@cam.org> wrote:
> On Tue, 1 Sep 2009, Catalin Marinas wrote:
>
>> On Tue, 2009-09-01 at 15:25 +0100, Russell King wrote:
>> > On Tue, Sep 01, 2009 at 02:54:54PM +0100, Catalin Marinas wrote:
>> > > venki kaps <venkiece2005@gmail.com> wrote:
>> > > > I have found the exact problem with respect to ARM jprobes.
>> > > >
>> > > > The problem with configure i.e, CONFIG_ARM_UNWIND = y; is enabled.
>> > >
>> > > I haven't followed the kprobes implementation for ARM but does it make
>> > > any assumptions about the existence of a frame pointer on the stack?
>> > > Enabling stack unwinding automatically disables the framepointer.
>> >
>> > If it uses CALLER_ADDRESSx() then it won't work with unwinding enabled.
>> > See 5613/1 (which is pending in the devel branch.)
>>
>> In addition to that, when CONFIG_FRAME_POINTER is disabled, the lr
>> register isn't always saved on the stack by the called function. I'm not
>> sure whether kretprobe_trampoline is aware of this.
>
> The way a kretprobe works is to put a trap at the very first instruction
> of the targetted function, preserve the value of LR when the trap is
> hit, and substitute it with the address of kretprobe_trampoline.  Then
> the original first instruction is emulated to pass over the trap point
> and normal execution is resumed.  So whether or not LR is then saved on
> the stack doesn't matter to kretprobe_trampoline as it will restore the
> LR value saved during the initial trap.
>
> Of course if you end up generating a backtrace within a kretprobed
> function then the result might look funny.
>
>
> Nicolas
>