[PATCH 1/3] powerpc/xmon: Dump ftrace buffers for the current CPU

[PATCH 1/3] powerpc/xmon: Dump ftrace buffers for the current CPU

[Breno Leitao]

Current xmon 'dt' command dumps the tracing buffer for all the CPUs,
which makes it possibly hard to read the logs due to the fact that most
of powerpc machines currently have many CPUs. Other than that, the CPU
lines are interleaved in the ftrace log.

This new option just dumps the ftrace buffer for the current CPU.

Signed-off-by: Breno Leitao <leitao@debian.org>
---
 arch/powerpc/xmon/xmon.c | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index 08e367e3e8c3..0cbd910193fa 100644
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -234,6 +234,7 @@ Commands:\n\
   "\
   dr	dump stream of raw bytes\n\
   dt	dump the tracing buffers (uses printk)\n\
+  dtc	dump the tracing buffers for current CPU (uses printk)\n\
 "
 #ifdef CONFIG_PPC_POWERNV
 "  dx#   dump xive on CPU #\n\
@@ -2342,6 +2343,19 @@ static void dump_one_paca(int cpu)
 	sync();
 }
 
+static void dump_tracing(void)
+{
+	int c;
+
+	c = inchar();
+	if (c == 'c')
+		ftrace_dump(DUMP_ORIG);
+	else
+		ftrace_dump(DUMP_ALL);
+
+	tracing_on();
+}
+
 static void dump_all_pacas(void)
 {
 	int cpu;
@@ -2507,6 +2521,11 @@ dump(void)
 	}
 #endif
 
+	if (c == 't') {
+		dump_tracing();
+		return;
+	}
+
 	if (c == '\n')
 		termch = c;
 
@@ -2525,9 +2544,6 @@ dump(void)
 		dump_log_buf();
 	} else if (c == 'o') {
 		dump_opal_msglog();
-	} else if (c == 't') {
-		ftrace_dump(DUMP_ALL);
-		tracing_on();
 	} else if (c == 'r') {
 		scanhex(&ndump);
 		if (ndump == 0)
-- 
2.13.2

[PATCH 2/3] powerpc/xmon: Disable and enable tracing command

[Breno Leitao]

If tracing is enabled and you get into xmon, the tracing buffer
continues to be updated, causing possible loss of data due to buffer
overflow and unnecessary tracing information coming from xmon functions.

This patch adds a new option that allows the tracing to be disabled and
re-enabled from inside xmon.

Signed-off-by: Breno Leitao <leitao@debian.org>
---
 arch/powerpc/xmon/xmon.c | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index 0cbd910193fa..19276d2f2f25 100644
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -89,6 +89,7 @@ static unsigned long nidump = 16;
 static unsigned long ncsum = 4096;
 static int termch;
 static char tmpstr[128];
+static char tracing_enabled = 1;
 
 static long bus_error_jmp[JMP_BUF_LEN];
 static int catch_memory_errors;
@@ -268,6 +269,7 @@ Commands:\n\
   Sr #	read SPR #\n\
   Sw #v write v to SPR #\n\
   t	print backtrace\n\
+  v	trace enable/disable\n\
   x	exit monitor and recover\n\
   X	exit monitor and don't recover\n"
 #if defined(CONFIG_PPC64) && !defined(CONFIG_PPC_BOOK3E)
@@ -983,6 +985,17 @@ cmds(struct pt_regs *excp)
 		case 'x':
 		case 'X':
 			return cmd;
+		case 'v':
+			if (tracing_is_on()) {
+				printk("Disabling tracing\n");
+				tracing_enabled = 0;
+				tracing_off();
+			} else {
+				printk("Enabling tracing\n");
+				tracing_enabled = 1;
+				tracing_on();
+			}
+			break;
 		case EOF:
 			printf(" <no input ...>\n");
 			mdelay(2000);
@@ -2353,7 +2366,8 @@ static void dump_tracing(void)
 	else
 		ftrace_dump(DUMP_ALL);
 
-	tracing_on();
+	if (tracing_enabled)
+		tracing_on();
 }
 
 static void dump_all_pacas(void)
-- 
2.13.2

Re: [PATCH 2/3] powerpc/xmon: Disable and enable tracing command

[Naveen N. Rao]

On 2017/07/31 02:22PM, Breno Leitao wrote:
> If tracing is enabled and you get into xmon, the tracing buffer
> continues to be updated, causing possible loss of data due to buffer
> overflow and unnecessary tracing information coming from xmon functions.
> 
> This patch adds a new option that allows the tracing to be disabled and
> re-enabled from inside xmon.

How is this new option useful? In the next patch, you disable tracing by 
default -- in what scenario do you expect to have to re-enable tracing 
from within xmon?

> 
> Signed-off-by: Breno Leitao <leitao@debian.org>
> ---
>  arch/powerpc/xmon/xmon.c | 16 +++++++++++++++-
>  1 file changed, 15 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
> index 0cbd910193fa..19276d2f2f25 100644
> --- a/arch/powerpc/xmon/xmon.c
> +++ b/arch/powerpc/xmon/xmon.c
> @@ -89,6 +89,7 @@ static unsigned long nidump = 16;
>  static unsigned long ncsum = 4096;
>  static int termch;
>  static char tmpstr[128];
> +static char tracing_enabled = 1;
> 
>  static long bus_error_jmp[JMP_BUF_LEN];
>  static int catch_memory_errors;
> @@ -268,6 +269,7 @@ Commands:\n\
>    Sr #	read SPR #\n\
>    Sw #v write v to SPR #\n\
>    t	print backtrace\n\
> +  v	trace enable/disable\n\
>    x	exit monitor and recover\n\
>    X	exit monitor and don't recover\n"
>  #if defined(CONFIG_PPC64) && !defined(CONFIG_PPC_BOOK3E)
> @@ -983,6 +985,17 @@ cmds(struct pt_regs *excp)
>  		case 'x':
>  		case 'X':
>  			return cmd;
> +		case 'v':
> +			if (tracing_is_on()) {
> +				printk("Disabling tracing\n");
> +				tracing_enabled = 0;
> +				tracing_off();

This only disables trace buffer updates - ftrace (and all its callbacks, 
et al) remains active, which isn't desirable. Can you see if this works 
for you:
https://patchwork.ozlabs.org/patch/769611/

- Naveen

Re: [PATCH 2/3] powerpc/xmon: Disable and enable tracing command

[Breno Leitao]

Hi Naveen,

On Tue, Aug 01, 2017 at 12:10:24PM +0530, Naveen N. Rao wrote:
> On 2017/07/31 02:22PM, Breno Leitao wrote:
> > If tracing is enabled and you get into xmon, the tracing buffer
> > continues to be updated, causing possible loss of data due to buffer
> > overflow and unnecessary tracing information coming from xmon functions.
> > 
> > This patch adds a new option that allows the tracing to be disabled and
> > re-enabled from inside xmon.
> 
> How is this new option useful? In the next patch, you disable tracing by 
> default -- in what scenario do you expect to have to re-enable tracing 
> from within xmon?

I see it being useful on two different scenarios:

1) You can reenable tracing if you want to call a function from xmon
(with 'p'), or even for code stepping (with 's').

2) You may also want to reenable tracing once you resume from xmon with
'zr'.

> > +		case 'v':
> > +			if (tracing_is_on()) {
> > +				printk("Disabling tracing\n");
> > +				tracing_enabled = 0;
> > +				tracing_off();
> 
> This only disables trace buffer updates - ftrace (and all its callbacks, 
> et al) remains active, which isn't desirable.

Why isn't it desirable? In fact, I thought it would be *the* desirable
function to call, since it does not do a lot of stuff, as disabling
tracing, in xmon mode, but, just disable the tracing buffer to be updated.

Since we are in xmon, we are in a very bad state, and something went
very wrong. Disabling the whole tracing might not be what we want to do
in this scenario, since it can hit the broken subsystem causing xmon to
fail.

For bad state scenario, I understand that it is desirable to be less
instrusive as possible, and tracing_off() does exactly it.

> Can you see if this works for you:
> https://patchwork.ozlabs.org/patch/769611/

Well, I understand that this patch solves a different issue, this does
not reduce the tracing caused by function tracer after you got into into
xmon.

As for example, with your patch applied, I can see a lot of xmon
functions polluting the tracing buffer as:

	1:mon> dt
	[  359.196593] Dumping ftrace buffer:
	[  359.196689] ---------------------------------
	[  359.196904]   1)               |          xmon_printf() {
	<110+ lines snipped>
	[  359.197727]   1) + 22.930 us   |          }
	[  359.199405]   1)               |          skipbl() {
	<50+ lines snipped>
	[  359.225069]   1) + 23.750 us   |          }


Since tracing continues to be enabled during xmon, these messages
continue to show up. That is exactly what I am trying to avoid with this
current patchset. Avoiding all xmon-related tracing is my main goal.

Thanks for your review,
Breno

Re: [PATCH 2/3] powerpc/xmon: Disable and enable tracing command

[Naveen N. Rao]

On 2017/08/01 11:21AM, Breno Leitao wrote:
> Hi Naveen,
> 
> On Tue, Aug 01, 2017 at 12:10:24PM +0530, Naveen N. Rao wrote:
> > On 2017/07/31 02:22PM, Breno Leitao wrote:
> > > If tracing is enabled and you get into xmon, the tracing buffer
> > > continues to be updated, causing possible loss of data due to buffer
> > > overflow and unnecessary tracing information coming from xmon functions.
> > > 
> > > This patch adds a new option that allows the tracing to be disabled and
> > > re-enabled from inside xmon.
> > 
> > How is this new option useful? In the next patch, you disable tracing by 
> > default -- in what scenario do you expect to have to re-enable tracing 
> > from within xmon?
> 
> I see it being useful on two different scenarios:
> 
> 1) You can reenable tracing if you want to call a function from xmon
> (with 'p'), or even for code stepping (with 's').

Hmm... those are just debugging aids, so I don't see why enabling the 
function tracer helps, unless you're debugging the tracer itself..

> 
> 2) You may also want to reenable tracing once you resume from xmon with
> 'zr'.

'zr' is for reboot, so not sure what you meant there...

If tracing was enabled before we went into xmon, I think that we should 
just restore it by default.

> 
> > > +		case 'v':
> > > +			if (tracing_is_on()) {
> > > +				printk("Disabling tracing\n");
> > > +				tracing_enabled = 0;
> > > +				tracing_off();
> > 
> > This only disables trace buffer updates - ftrace (and all its callbacks, 
> > et al) remains active, which isn't desirable.
> 
> Why isn't it desirable? In fact, I thought it would be *the* desirable
> function to call, since it does not do a lot of stuff, as disabling
> tracing, in xmon mode, but, just disable the tracing buffer to be updated.
> 
> Since we are in xmon, we are in a very bad state, and something went
> very wrong. Disabling the whole tracing might not be what we want to do
> in this scenario, since it can hit the broken subsystem causing xmon to
> fail.
> 
> For bad state scenario, I understand that it is desirable to be less
> instrusive as possible, and tracing_off() does exactly it.

My concern was that we will still go through all the callbacks and I was 
wondering if we could get rid of that. However, I don't see any easy way 
to do that with the current ftrace infrastructure.

> 
> > Can you see if this works for you:
> > https://patchwork.ozlabs.org/patch/769611/
> 
> Well, I understand that this patch solves a different issue, this does
> not reduce the tracing caused by function tracer after you got into into
> xmon.
> 
> As for example, with your patch applied, I can see a lot of xmon
> functions polluting the tracing buffer as:
> 
> 	1:mon> dt
> 	[  359.196593] Dumping ftrace buffer:
> 	[  359.196689] ---------------------------------
> 	[  359.196904]   1)               |          xmon_printf() {
> 	<110+ lines snipped>
> 	[  359.197727]   1) + 22.930 us   |          }
> 	[  359.199405]   1)               |          skipbl() {
> 	<50+ lines snipped>
> 	[  359.225069]   1) + 23.750 us   |          }
> 
> 
> Since tracing continues to be enabled during xmon, these messages
> continue to show up. That is exactly what I am trying to avoid with this
> current patchset. Avoiding all xmon-related tracing is my main goal.

Ok, makes sense.


Thanks,
Naveen
 

Re: [PATCH 2/3] powerpc/xmon: Disable and enable tracing command

[Breno Leitao]

On Wed, Aug 02, 2017 at 06:51:24PM +0530, Naveen N. Rao wrote:
> On 2017/08/01 11:21AM, Breno Leitao wrote:
> > Hi Naveen,
> > 
> > On Tue, Aug 01, 2017 at 12:10:24PM +0530, Naveen N. Rao wrote:
> > > On 2017/07/31 02:22PM, Breno Leitao wrote:
> > > > If tracing is enabled and you get into xmon, the tracing buffer
> > > > continues to be updated, causing possible loss of data due to buffer
> > > > overflow and unnecessary tracing information coming from xmon functions.
> > > > 
> > > > This patch adds a new option that allows the tracing to be disabled and
> > > > re-enabled from inside xmon.
> > > 
> > > How is this new option useful? In the next patch, you disable tracing by 
> > > default -- in what scenario do you expect to have to re-enable tracing 
> > > from within xmon?
> > 
> > I see it being useful on two different scenarios:
> > 
> > 1) You can reenable tracing if you want to call a function from xmon
> > (with 'p'), or even for code stepping (with 's').
> 
> Hmm... those are just debugging aids, so I don't see why enabling the 
> function tracer helps, unless you're debugging the tracer itself..
> 
> > 
> > 2) You may also want to reenable tracing once you resume from xmon with
> > 'zr'.
> 
> 'zr' is for reboot, so not sure what you meant there...

I meant 'x' in fact, which means 'exit monitor and recover'. This will
resume the kernel after getting into xmon.

> If tracing was enabled before we went into xmon, I think that we should 
> just restore it by default.

Makes sense. So, I will get ride of this 'v' feature and disable tracing
when entering the xmon, and reenabling it on recovering. That way, we
will avoid xmon functions showing up on the tracing buffer.

I will send a new patchset soon!

Thanks,
Breno

[PATCH 3/3] powerpc/xmon: Disable tracing on xmon by default

[Breno Leitao]

Currently tracing is enabled from inside xmon, which may cause some
noise into the tracing buffer, and makes it harder to find what, in
the tracing buffer, are kernel non-xmon functions and what is xmon
'noise' (as printk()s and terminal functions tracing).

This patch simple disables it by default, showing a better trace output
of the failing functions just before it gets into xmon.

Signed-off-by: Breno Leitao <leitao@debian.org>
---
 arch/powerpc/xmon/xmon.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index 19276d2f2f25..b614cc3a3a65 100644
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -89,7 +89,7 @@ static unsigned long nidump = 16;
 static unsigned long ncsum = 4096;
 static int termch;
 static char tmpstr[128];
-static char tracing_enabled = 1;
+static char tracing_enabled = 0;
 
 static long bus_error_jmp[JMP_BUF_LEN];
 static int catch_memory_errors;
@@ -463,6 +463,7 @@ static int xmon_core(struct pt_regs *regs, int fromipi)
 
 	local_irq_save(flags);
 	hard_irq_disable();
+	tracing_off();
 
 	bp = in_breakpoint_table(regs->nip, &offset);
 	if (bp != NULL) {
-- 
2.13.2

Re: [PATCH 1/3] powerpc/xmon: Dump ftrace buffers for the current CPU

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 2624 bytes --]

Hi Breno,

[auto build test ERROR on powerpc/next]
[also build test ERROR on v4.13-rc3 next-20170802]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url:    https://github.com/0day-ci/linux/commits/Breno-Leitao/powerpc-xmon-Dump-ftrace-buffers-for-the-current-CPU/20170801-054818
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-amigaone_defconfig (attached as .config)
compiler: powerpc-linux-gnu-gcc (Debian 6.1.1-9) 6.1.1 20160705
reproduce:
        wget https://raw.githubusercontent.com/01org/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        make.cross ARCH=powerpc 

All errors (new ones prefixed by >>):

   arch/powerpc/xmon/xmon.c: In function 'dump':
>> arch/powerpc/xmon/xmon.c:2525:3: error: implicit declaration of function 'dump_tracing' [-Werror=implicit-function-declaration]
      dump_tracing();
      ^~~~~~~~~~~~
   cc1: all warnings being treated as errors

vim +/dump_tracing +2525 arch/powerpc/xmon/xmon.c

  2523	
  2524		if (c == 't') {
> 2525			dump_tracing();
  2526			return;
  2527		}
  2528	
  2529		if (c == '\n')
  2530			termch = c;
  2531	
  2532		scanhex((void *)&adrs);
  2533		if (termch != '\n')
  2534			termch = 0;
  2535		if (c == 'i') {
  2536			scanhex(&nidump);
  2537			if (nidump == 0)
  2538				nidump = 16;
  2539			else if (nidump > MAX_DUMP)
  2540				nidump = MAX_DUMP;
  2541			adrs += ppc_inst_dump(adrs, nidump, 1);
  2542			last_cmd = "di\n";
  2543		} else if (c == 'l') {
  2544			dump_log_buf();
  2545		} else if (c == 'o') {
  2546			dump_opal_msglog();
  2547		} else if (c == 'r') {
  2548			scanhex(&ndump);
  2549			if (ndump == 0)
  2550				ndump = 64;
  2551			xmon_rawdump(adrs, ndump);
  2552			adrs += ndump;
  2553			last_cmd = "dr\n";
  2554		} else {
  2555			scanhex(&ndump);
  2556			if (ndump == 0)
  2557				ndump = 64;
  2558			else if (ndump > MAX_DUMP)
  2559				ndump = MAX_DUMP;
  2560	
  2561			switch (c) {
  2562			case '8':
  2563			case '4':
  2564			case '2':
  2565			case '1':
  2566				ndump = ALIGN(ndump, 16);
  2567				dump_by_size(adrs, ndump, c - '0');
  2568				last[1] = c;
  2569				last_cmd = last;
  2570				break;
  2571			default:
  2572				prdump(adrs, ndump);
  2573				last_cmd = "d\n";
  2574			}
  2575	
  2576			adrs += ndump;
  2577		}
  2578	}
  2579	

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 17216 bytes --]