Problems using psr.dd

Problems using psr.dd

[Keith Owens]

I am unable to successfully use psr.dd on 2.4 and 2.6 kernels.  The
patches below plus the debug.c module demonstrate the problem.
do_trap29() sets regs->cr_ipsr.dd and returns.  The kernel drops
through ia64_leave_kernel, loads the modified ipsr and does rfi.
However the same debug fault occurs again, as if psr.dd were being
ignored.

What am I missing?

# depmod -ae;rmmod debug 2>/dev/null; modprobe debug
init_debug: start
dbr[0]=0xa0000000000b8f90 dbr[1]=0x41fffffffffffffc
setpsrdb: on=1
init_debug: test
trap 29 ifa=0xa0000000000b8f90 iip=0xa0000000000b8430 count=0
trap 29 ifa=0xa0000000000b8f90 iip=0xa0000000000b8430 count=1
trap 29 ifa=0xa0000000000b8f90 iip=0xa0000000000b8430 count=2
trap 29 ifa=0xa0000000000b8f90 iip=0xa0000000000b8430 count=3
db is looping, disabling it
setpsrdb: on=0
init_debug: end


--- linux/arch/ia64/kernel/Makefile_1.24	Fri Nov 14 16:57:08 2003
+++ linux/arch/ia64/kernel/Makefile	Fri Nov 14 16:19:49 2003
@@ -11,7 +11,7 @@
 
 O_TARGET := kernel.o
 
-export-objs := ia64_ksyms.o sal.o
+export-objs := ia64_ksyms.o sal.o traps.o
 
 obj-y := acpi.o entry.o gate.o efi.o efi_stub.o ia64_ksyms.o irq.o irq_ia64.o irq_lsapic.o ivt.o \
 	 machvec.o pal.o process.o perfmon.o ptrace.o sal.o salinfo.o semaphore.o setup.o	 \

--- linux/arch/ia64/kernel/entry.S_1.33	Fri Nov 14 16:57:08 2003
+++ linux/arch/ia64/kernel/entry.S	Fri Nov 14 16:55:03 2003
@@ -771,7 +771,127 @@
 	mov ar.rsc=rARRSC
 	mov ar.unat=rARUNAT
 	mov pr=rARPR,-1
-	rfi
+	;;
+	// lots of nops for Itanium2 errata 11
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	;;
+	rfi;;
+	// lots of nops for Itanium2 errata 11
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	nop 0
+	nop 0
+	nop 0;;
+	;;
+	;;
 END(ia64_leave_kernel)
 
 ENTRY(handle_syscall_error)

--- linux/arch/ia64/kernel/traps.c_1.27	Fri Nov 14 16:57:08 2003
+++ linux/arch/ia64/kernel/traps.c	Fri Nov 14 16:27:57 2003
@@ -452,6 +452,10 @@
 	return rv;
 }
 
+#include <linux/module.h>
+int (*trap29)(unsigned long ifa, struct pt_regs *regs);
+EXPORT_SYMBOL(trap29);
+
 void
 ia64_fault (unsigned long vector, unsigned long isr, unsigned long ifa,
 	    unsigned long iim, unsigned long itir, unsigned long arg5,
@@ -551,6 +555,8 @@
 		break;
 
 	      case 29: /* Debug */
+			if (trap29 && trap29(ifa, regs))
+				return;
 	      case 35: /* Taken Branch Trap */
 	      case 36: /* Single Step Trap */
 		switch (vector) {


== debug.c =
#include <linux/config.h>
#include <linux/slab.h>
#include <linux/kdb.h>
#include <linux/module.h>

MODULE_LICENSE("GPL");

int victim;
static int trap29count;
extern int (*trap29)(unsigned long ifa, struct pt_regs *regs);

static int do_trap29(unsigned long ifa, struct pt_regs *regs)
{
	if (user_mode(regs))
		return 0;
	kdb_printf("trap 29 ifa=0x%016lx iip=0x%016lx count=%d\n", ifa, regs->cr_iip, trap29count);
	if (trap29count >= 3) {
		kdb_printf("db is looping, disabling it\n");
		ia64_psr(regs)->db = 0;
	} else {
		++trap29count;
		ia64_psr(regs)->dd = 1;
	}
	return 1;
}

static void setdbr(unsigned long regnum, unsigned long address, unsigned long mask)
{
	kdb_printf("dbr[%ld]=0x%016lx dbr[%ld]=0x%016lx\n", regnum, address, regnum+1, mask);
	__asm__ __volatile__ ("mov dbr[%0]=%1;;mov dbr[%2]=%3;;"::"r"(regnum),"r"(address),"r"(regnum+1),"r"(mask));
	ia64_srlz_d();
}

static void setpsrdb(int on)
{
	unsigned long tmp;
	kdb_printf("%s: on=%d\n", __FUNCTION__, on);
	if (on)
		__asm__ __volatile__ ("mov %0=psr;;dep %0=-1,%0,24,1;;mov psr.l=%0;;srlz.d;;srlz.i;;":"=r"(tmp)::"memory");
	else
		__asm__ __volatile__ ("mov %0=psr;;dep %0=0,%0,24,1;;mov psr.l=%0;;srlz.d;;srlz.i;;":"=r"(tmp)::"memory");
}


static int __init init_debug(void)
{
	kdb_printf("%s: start\n", __FUNCTION__);
	trap29 = &do_trap29;
	setdbr(0, (unsigned long)&victim, 1UL<<62 | 1UL<<56 | (-4UL & 0xffffffffffffffUL));
	setpsrdb(1);
	kdb_printf("%s: test\n", __FUNCTION__);
	victim = 1;
	setpsrdb(0);
	kdb_printf("%s: end\n", __FUNCTION__);
	return 0;
}

static void __exit exit_debug(void)
{
	kdb_printf("%s: start\n", __FUNCTION__);
	trap29 = NULL;
	setdbr(0, 0, 0);
	kdb_printf("%s: end\n", __FUNCTION__);
}

module_init(init_debug)
module_exit(exit_debug)

Re: Problems using psr.dd

[David Mosberger]

>>>>> On Fri, 14 Nov 2003 17:09:15 +1100, Keith Owens <kaos@sgi.com> said:

  Keith> I am unable to successfully use psr.dd on 2.4 and 2.6
  Keith> kernels.  The patches below plus the debug.c module
  Keith> demonstrate the problem.  do_trap29() sets regs->cr_ipsr.dd
  Keith> and returns.  The kernel drops through ia64_leave_kernel,
  Keith> loads the modified ipsr and does rfi.  However the same debug
  Keith> fault occurs again, as if psr.dd were being ignored.

  Keith> What am I missing?

Did you find the answer already?

I'm wondering if the problem would still occurr if the
debug-fault-triggering memory access was in a function with an empty
register frame.

	--david

Re: Problems using psr.dd

[Keith Owens]

On Wed, 19 Nov 2003 18:39:40 -0800, 
David Mosberger <davidm@napali.hpl.hp.com> wrote:
>>>>>> On Fri, 14 Nov 2003 17:09:15 +1100, Keith Owens <kaos@sgi.com> said:
>
>  Keith> I am unable to successfully use psr.dd on 2.4 and 2.6
>  Keith> kernels.  The patches below plus the debug.c module
>  Keith> demonstrate the problem.  do_trap29() sets regs->cr_ipsr.dd
>  Keith> and returns.  The kernel drops through ia64_leave_kernel,
>  Keith> loads the modified ipsr and does rfi.  However the same debug
>  Keith> fault occurs again, as if psr.dd were being ignored.
>
>  Keith> What am I missing?
>
>Did you find the answer already?

No, your response is the only one so far.

>I'm wondering if the problem would still occurr if the
>debug-fault-triggering memory access was in a function with an empty
>register frame.

Like this?  It makes no difference, trap29 just keeps occuring, even
though psr.dd is being set..

static void set_victim(void)
{
        victim = 1;
 300:   0d 78 00 02 00 24       [MFI]       addl r15=0,r1
                        300: GPREL22    victim
 306:   00 00 00 02 00 c0                   nop.f 0x0
 30c:   11 00 00 90                         mov r14=1;;
 310:   1d 00 38 1e 90 11       [MFB]       st4 [r15]=r14
 316:   00 00 00 02 00 80                   nop.f 0x0
 31c:   08 00 84 00                         br.ret.sptk.many b0;;

}

Forgot to mention, this is on a vanilla McKinley box, not a SN2.

# cat /proc/cpuinfo 
processor  : 0
vendor     : GenuineIntel
arch       : IA-64
family     : Itanium 2
model      : 0
revision   : 5
archrev    : 0
features   : branchlong
cpu number : 0
cpu regs   : 4
cpu MHz    : 897.454998
itc MHz    : 897.454998
BogoMIPS   : 1342.17
...

Re: Problems using psr.dd

[Matt Chapman]

On Thu, Nov 20, 2003 at 03:32:53PM +1100, Keith Owens wrote:
> 
> Forgot to mention, this is on a vanilla McKinley box, not a SN2.

I tried your code on an Itanium 1 (using 2.6.0-test9-bk without kdb,
s/kdb_printf/printk/).  I get exactly the same behaviour.

However if I take out the printk in do_trap29 (reducing the register
stack depth?) it seems to work okay.

Matt

Re: Problems using psr.dd

[Keith Owens]

DavidM suggested that I redo the test debug module without kdb_printf()
and the problem went away!

It turns out that the problem is not kdb, it is an RSE problem which is
highly sensitive to the amount of RSE activity that occurs inside the
hardware breakpoint handler.  kdb_printf() always prints its messages
using polling mode, printk() delays output if the kernel is in
interrupt context.  The extra work that is done by kdb_printf() as
opposed to printk() is enough to trip the RSE problem.

If there is enough RSE activity during the hardware breakpoint handler
to flush the registers at the time of the breakpoint then returning
with psr.db = 1 and psr.dd = 1 to step over the hardware breakpoint
will not work.  The RSE activity that occurs as the flushed registers
are restored is counted as a valid instruction, psr.dd is cleared, the
original instruction is reexecuted and it trips again.

Beware of doing too much activity in the hardware breakpoint handler.
I will create a patch against traps.c to detect a return with psr.dd
set and issue loadrs to ensure that the RSE problem does not bite us.

The debug.c module below demonstrates the problem.  "modprobe debug.o"
usually works, but even without flushrs it sometimes trips the RSE
problem, depending on system activity.  "modprobe debug.o do_flushrs=1"
always hits the problem.

== debug.c ==

#include <linux/config.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/console.h>

int victim;
int do_flushrs;
static int trap29count;
extern int (*trap29)(unsigned long ifa, struct pt_regs *regs);

MODULE_LICENSE("GPL");
MODULE_PARM(do_flushrs, "i");

static int do_trap29(unsigned long ifa, struct pt_regs *regs)
{
	if (user_mode(regs))
		return 0;
	printk("trap 29 ifa=0x%016lx iip=0x%016lx count=%d victim=%d\n", ifa, regs->cr_iip, trap29count, victim);
	if (do_flushrs)
		asm volatile (";;flushrs;; " :::"memory");
	if (trap29count >= 3) {
		printk("db is looping, disabling it\n");
		ia64_psr(regs)->db = 0;
	} else {
		++trap29count;
		ia64_psr(regs)->dd = 1;
	}
	return 1;
}

static void setdbr(unsigned long regnum, unsigned long address, unsigned long mask)
{
	__asm__ __volatile__ ("mov dbr[%0]=%1;;mov dbr[%2]=%3;;"::"r"(regnum),"r"(address),"r"(regnum+1),"r"(mask));
	ia64_srlz_d();
}

static void setpsrdb(int on)
{
	unsigned long tmp;
	if (on)
		__asm__ __volatile__ ("mov %0=psr;;dep %0=-1,%0,24,1;;mov psr.l=%0;;srlz.d;;srlz.i;;":"=r"(tmp)::"memory");
	else
		__asm__ __volatile__ ("mov %0=psr;;dep %0=0,%0,24,1;;mov psr.l=%0;;srlz.d;;srlz.i;;":"=r"(tmp)::"memory");
}

static int __init init_debug(void)
{
	trap29 = &do_trap29;
	setdbr(0, (unsigned long)&victim, 1UL<<62 | 1UL<<56 | (-4UL & 0xffffffffffffffUL));
	setpsrdb(1);
	victim = 1;
	barrier();
	victim = 2;
	setpsrdb(0);
	return 0;
}

static void __exit exit_debug(void)
{
	trap29 = NULL;
	setdbr(0, 0, 0);
}

module_init(init_debug)
module_exit(exit_debug)

======

Patch against traps.c to invoke the test trap29() handler in module
debug.c.

--- 2.4.23-pre9-cset-1.1069.1.104-to-1.1111/arch/ia64/kernel/Makefile	Fri Oct 31 10:40:52 2003
+++ 2.4.23-pre9-cset-1.1069.1.104-to-1.1111-psr.dd/arch/ia64/kernel/Makefile	Thu Nov 20 18:49:15 2003
@@ -11,7 +11,7 @@ all: kernel.o head.o init_task.o
 
 O_TARGET := kernel.o
 
-export-objs := ia64_ksyms.o
+export-objs := ia64_ksyms.o traps.o
 
 obj-y := acpi.o entry.o gate.o efi.o efi_stub.o ia64_ksyms.o irq.o irq_ia64.o irq_lsapic.o ivt.o \
 	 machvec.o pal.o process.o perfmon.o ptrace.o sal.o salinfo.o semaphore.o setup.o	 \
--- 2.4.23-pre9-cset-1.1069.1.104-to-1.1111/arch/ia64/kernel/traps.c	Thu Nov 20 16:04:45 2003
+++ 2.4.23-pre9-cset-1.1069.1.104-to-1.1111-psr.dd/arch/ia64/kernel/traps.c	Thu Nov 20 18:49:15 2003
@@ -429,6 +429,10 @@ ia64_illegal_op_fault (unsigned long ec,
 	return rv;
 }
 
+#include <linux/module.h>
+int (*trap29)(unsigned long ifa, struct pt_regs *regs);
+EXPORT_SYMBOL(trap29);
+
 void
 ia64_fault (unsigned long vector, unsigned long isr, unsigned long ifa,
 	    unsigned long iim, unsigned long itir, unsigned long arg5,
@@ -528,6 +532,8 @@ ia64_fault (unsigned long vector, unsign
 		break;
 
 	      case 29: /* Debug */
+			if (trap29 && trap29(ifa, regs))
+				return;
 	      case 35: /* Taken Branch Trap */
 	      case 36: /* Single Step Trap */
 		switch (vector) {

Re: Problems using psr.dd

[David Mosberger]

>>>>> On Thu, 20 Nov 2003 18:52:33 +1100, Keith Owens <kaos@sgi.com> said:

  Keith> If there is enough RSE activity during the hardware
  Keith> breakpoint handler to flush the registers at the time of the
  Keith> breakpoint then returning with psr.db = 1 and psr.dd = 1 to
  Keith> step over the hardware breakpoint will not work.  The RSE
  Keith> activity that occurs as the flushed registers are restored is
  Keith> counted as a valid instruction, psr.dd is cleared, the
  Keith> original instruction is reexecuted and it trips again.

  Keith> Beware of doing too much activity in the hardware breakpoint
  Keith> handler.  I will create a patch against traps.c to detect a
  Keith> return with psr.dd set and issue loadrs to ensure that the
  Keith> RSE problem does not bite us.

I'm still not sure what's triggering the spurious fault:

 (1) The RSE activity generated for restoring the current frame, or
 (2) The RSE activity generated by the alloc instruction that eventually
     follows the instruction to which "rfi" returns to.

Unfortunately, the description of Erratum 11 isn't terribly clear to
me.  It says:

	... The rfi instruction is followed by additional instructions
	that generate register stack (RSE) activity (alloc, flushrs,
	br.ret).

I assume "followed" is referring to execution order, not program order
(i.e., inserting nop's around the "rfi" isn't going to do us any
good).  In your (original) case, the "rfi" is followed by a store
instruction, br.call, then alloc.  From reading the description, I
wouldn't have expected this to trigger the Erratum, but perhaps it
does.  Can someone clarify?

	--david

RE: Problems using psr.dd

[Seth, Rohit]

Erratum 11 applies to the cases where rfi is followed by RSE generating
instruction in static code stream (i.e. in the source).  Have not looked
at the Keith's changes but hopefully he has not put any instruction like
alloc after rfi in the source.

Thanks,
rohit

> -----Original Message-----
> From: linux-ia64-owner@vger.kernel.org [mailto:linux-ia64-
> owner@vger.kernel.org] On Behalf Of David Mosberger
> Sent: Thursday, November 20, 2003 10:48 AM
> To: Keith Owens
> Cc: linux-ia64@vger.kernel.org
> Subject: Re: Problems using psr.dd
> 
> >>>>> On Thu, 20 Nov 2003 18:52:33 +1100, Keith Owens <kaos@sgi.com>
said:
> 
>   Keith> If there is enough RSE activity during the hardware
>   Keith> breakpoint handler to flush the registers at the time of the
>   Keith> breakpoint then returning with psr.db = 1 and psr.dd = 1 to
>   Keith> step over the hardware breakpoint will not work.  The RSE
>   Keith> activity that occurs as the flushed registers are restored is
>   Keith> counted as a valid instruction, psr.dd is cleared, the
>   Keith> original instruction is reexecuted and it trips again.
> 
>   Keith> Beware of doing too much activity in the hardware breakpoint
>   Keith> handler.  I will create a patch against traps.c to detect a
>   Keith> return with psr.dd set and issue loadrs to ensure that the
>   Keith> RSE problem does not bite us.
> 
> I'm still not sure what's triggering the spurious fault:
> 
>  (1) The RSE activity generated for restoring the current frame, or
>  (2) The RSE activity generated by the alloc instruction that
eventually
>      follows the instruction to which "rfi" returns to.
> 
> Unfortunately, the description of Erratum 11 isn't terribly clear to
> me.  It says:
> 
> 	... The rfi instruction is followed by additional instructions
> 	that generate register stack (RSE) activity (alloc, flushrs,
> 	br.ret).
> 
> I assume "followed" is referring to execution order, not program order
> (i.e., inserting nop's around the "rfi" isn't going to do us any
> good).  In your (original) case, the "rfi" is followed by a store
> instruction, br.call, then alloc.  From reading the description, I
> wouldn't have expected this to trigger the Erratum, but perhaps it
> does.  Can someone clarify?
> 
> 	--david
> -
> To unsubscribe from this list: send the line "unsubscribe linux-ia64"
in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: Problems using psr.dd

[David Mosberger]

>>>>> On Thu, 20 Nov 2003 12:29:42 -0800, "Seth, Rohit" <rohit.seth@intel.com> said:

  Rohit> Erratum 11 applies to the cases where rfi is followed by RSE
  Rohit> generating instruction in static code stream (i.e. in the
  Rohit> source).

Ah, so Keith's interpretation was correct.

  Rohit> Have not looked at the Keith's changes but hopefully he has
  Rohit> not put any instruction like alloc after rfi in the source.

No, quite the opposite: he put NOPs around it.  So clearly this
remains an open issue.

	--david

Re: Problems using psr.dd

[Keith Owens]

On Thu, 20 Nov 2003 10:47:50 -0800, 
David Mosberger <davidm@napali.hpl.hp.com> wrote:
>>>>>> On Thu, 20 Nov 2003 18:52:33 +1100, Keith Owens <kaos@sgi.com> said:
>  Keith> Beware of doing too much activity in the hardware breakpoint
>  Keith> handler.  I will create a patch against traps.c to detect a
>  Keith> return with psr.dd set and issue loadrs to ensure that the
>  Keith> RSE problem does not bite us.
>
>I'm still not sure what's triggering the spurious fault:
>
> (1) The RSE activity generated for restoring the current frame, or
> (2) The RSE activity generated by the alloc instruction that eventually
>     follows the instruction to which "rfi" returns to.

It has to be (1).  psr.dd is getting cleared before the st8 is being
executed.  Just to be sure, I inserted 20 bundles of nop before and
after 'victim = 1;', the problem still occurs.

>Unfortunately, the description of Erratum 11 isn't terribly clear to
>me.  It says:
>
>	... The rfi instruction is followed by additional instructions
>	that generate register stack (RSE) activity (alloc, flushrs,
>	br.ret).
>
>I assume "followed" is referring to execution order, not program order
>(i.e., inserting nop's around the "rfi" isn't going to do us any
>good).  In your (original) case, the "rfi" is followed by a store
>instruction, br.call, then alloc.  From reading the description, I
>wouldn't have expected this to trigger the Erratum, but perhaps it
>does.  Can someone clarify?

Agreed that it is unclear.  Given the workaround for errata 11 (add 4
nop bundles around rfi) I interpret errata 11 as an instruction read
ahead problem, where RSE instructions in the bundles after rfi could be
read, decoded and trip RSE activity even though they were not executed.
In any case, I added 20 nop bundles around the trapping instruction and
it made no difference.

Arch volume 2, section 5.3 is quite explicit, "A non-faulting mandatory
RSE load will clear PSR.da and PSR.dd".  This does not appear to be a
problem when returning to user space with these bits set, probably
because the context switch does loadrs.  AFAICT it is only an issue
when using hardware debug registers on the kernel itself.  traps.c has
to detect that it is returning to kernel state with either of these
bits set and issue loadrs to ensure that the rfi does not require RSE
loads.  Patch in progress.

Re: Problems using psr.dd

[David Mosberger]

>>>>> On Fri, 21 Nov 2003 10:15:55 +1100, Keith Owens <kaos@sgi.com> said:

  Keith> Arch volume 2, section 5.3 is quite explicit, "A non-faulting
  Keith> mandatory RSE load will clear PSR.da and PSR.dd".

Ah, I see, that clears it up, then.  I had forgotten about this
behavior.  No mystery then.

Thanks,

	--david

RE: Problems using psr.dd

[Seth, Rohit]

> Arch volume 2, section 5.3 is quite explicit, "A non-faulting
mandatory
> RSE load will clear PSR.da and PSR.dd".  This does not appear to be a
> problem when returning to user space with these bits set, probably
> because the context switch does loadrs.  AFAICT it is only an issue
> when using hardware debug registers on the kernel itself.  traps.c has
> to detect that it is returning to kernel state with either of these
> bits set and issue loadrs to ensure that the rfi does not require RSE
> loads.  Patch in progress.
> 
I think you meant the kernel return path (and not context switch) while
referring to loadrs above.  It is true that loadrs operation is done
only while transitioning to user space from kernel and not when you are
going back to kernel mode.  So for the kernel->kernel case rfi can lead
to mandatory RSE activity and thus clearing the da and dd bits.
> -