just-in-time debugging?

just-in-time debugging?

[Tony Hoyle]

Is there a way (kernel or userspace... doesn't matter) that gdb/ddd
could be invoked when a program is about
to dump core, or perhaps on a certain signal (that the app could deliver
to itself when required).  The latter case
is what I need right now, as I have to debug an app that breaks
seemingly randomly & I need to halt when
certain assertions fail.  Core dumps aren't much use as you can't resume
them, otherwise I'd just force a segfault
or something.

I had a look at the do_coredump stuff and it looks like it could be
altered to call gdb in the same way that
modprobe gets called by kmod... however I don't sufficiently know the
code to work out whether it'd work properly
or not.  

A patch to glibc would perhaps be better, but I know that code even
less!

Something like responding to SIGTRAP would probably be ideal.

Tony

-- 

"Two weeks before due date, the programmers work 22 hour days cobbling an
 application from... (apparently) one programmer bashing his face into the
 keyboard." -- Dilbert

tmh@magenta-netlogic.com                http://www.nothing-on.tv 

RE: just-in-time debugging?

[Davide Libenzi]

[-- Attachment #1: Type: text/plain, Size: 1236 bytes --]


On 28-Apr-2001 Tony Hoyle wrote:
> Is there a way (kernel or userspace... doesn't matter) that gdb/ddd
> could be invoked when a program is about
> to dump core, or perhaps on a certain signal (that the app could deliver
> to itself when required).  The latter case
> is what I need right now, as I have to debug an app that breaks
> seemingly randomly & I need to halt when
> certain assertions fail.  Core dumps aren't much use as you can't resume
> them, otherwise I'd just force a segfault
> or something.
> 
> I had a look at the do_coredump stuff and it looks like it could be
> altered to call gdb in the same way that
> modprobe gets called by kmod... however I don't sufficiently know the
> code to work out whether it'd work properly
> or not.  

Sorry but why don't You run Your application with gdb ?
Once Your program crashes You'll get the prompt and You'll be able to
stack-trace and watching whatever You need.
The solution I use to be able to get inside the program even when the gdb is
not running is the one that You can find in the attached file.
Basically it install the handler that will create a script file that You can
use to automatically enter with gdb inside Your program while it's running.





- Davide


[-- Attachment #2: GdbHook.cpp --]
[-- Type: application/octet-stream, Size: 3918 bytes --]







#define GDBHOOK_DEBUGGER			"gdb"
#define GDBHOOK_MODE				"GDBHOOK_MODE"
#define GDBHOOK_BELL_COUNT			"GDBHOOK_BELL_COUNT"
#define GDBHOOK_STD_BELL_COUNT		4
#define GDBHOOK_STDPREFIX			"sig-hook"
#define GDBHOOK_WAIT				"GDBHOOK_WAIT"
#define GDBHOOK_STD_WAIT			4









static int
DbghWaitProc(char const * pszFileName);

static void
DbghSigHandler(int iSignal);

static int
DbghCleanupFiles(char const * pszPath);






static char	szProgram[256] = "",
			szPath[256] = "";








static int
DbghWaitProc(char const * pszFileName)
{

	char const *	pszBellCnt = getenv(GDBHOOK_BELL_COUNT);
	int	iBellCnt = (pszBellCnt != NULL) ? atoi(pszBellCnt): GDBHOOK_STD_BELL_COUNT;
   	char const *	pszWait = getenv(GDBHOOK_WAIT);
   	int	iWait = (pszWait != NULL) ? atoi(pszWait): GDBHOOK_STD_WAIT;
   	volatile int	iLoop = 0;

   	while (!iLoop && (access(pszFileName, F_OK) == 0))
   	{
   		if (iBellCnt > 0)
   			fprintf(stderr, "%c", 7), --iBellCnt;

		sleep(iWait);
	}

	return (0);

}



static void
DbghSigHandler(int iSignal)
{

	char const *	pszEnvVar = getenv(GDBHOOK_MODE);

	if ((pszEnvVar != NULL) && atoi(pszEnvVar))
	{
		char	szDbgFile[256] = "";

		sprintf(szDbgFile, "%s%s.%d.%u.hook", szPath, GDBHOOK_STDPREFIX, iSignal, getpid());

		FILE *	pFile = fopen(szDbgFile, "w");

		if (pFile != NULL)
		{
			time_t	tFault = time(NULL);

#ifdef SOLARIS

			fprintf(pFile,
   					"#!/bin/sh\n"
   					"#\n"
   					"# Signal = %s (%d)\n"
   					"# Time   = %s"
   					"# Pid    = %u\n"
   					"#\n"
   					"%s %s %u\n\n",
   					(iSignal < _sys_nsig) ? _sys_siglist[iSignal]: "????", iSignal, ctime(&tFault),
   					getpid(), GDBHOOK_DEBUGGER, szProgram, getpid());

#else // #ifdef SOLARIS

			fprintf(pFile,
   					"#!/bin/sh\n"
   					"#\n"
   					"# Signal = %s (%d)\n"
   					"# Time   = %s"
   					"# Pid    = %u\n"
   					"#\n"
   					"%s %s %u\n\n",
   					(iSignal < _NSIG) ? _sys_siglist[iSignal]: "????", iSignal, ctime(&tFault),
   					getpid(), GDBHOOK_DEBUGGER, szProgram, getpid());

#endif // #ifdef SOLARIS

   			fclose(pFile);


   			chmod(szDbgFile, 0755);


   			DbghWaitProc(szDbgFile);


   			signal(iSignal, SIG_DFL);
	   	}
	}
	else
		signal(iSignal, SIG_DFL);

}



static int
DbghCleanupFiles(char const * pszPath)
{

	glob_t 	globbuf;
	char	szPattern[256] = "";

	sprintf(szPattern, "%s*.hook", pszPath);

	if (glob(szPattern, 0, NULL, &globbuf) == 0)
	{

		for (int ii = 0; ii < globbuf.gl_pathc; ii++)
			unlink(globbuf.gl_pathv[ii]);

	}

	globfree(&globbuf);

	return (0);

}



int
DbghInit(char const * pszProgram)
{

	strcpy(szProgram, pszProgram);

	char const *	pszSlash = strrchr(pszProgram, '/');

	if (pszSlash != NULL)
	{
		int	iPathLength = (int) (pszSlash - pszProgram) + 1;

		strncpy(szPath, pszProgram, iPathLength);
	    szPath[iPathLength] = '\0';
	}
	else
		strcpy(szPath, "./");


	if (DbghCleanupFiles(szPath) < 0)
		return (-1);


	return (0);

}



int
DbghInstall(void)
{

	signal(SIGSEGV, DbghSigHandler);
	signal(SIGBUS, DbghSigHandler);
	signal(SIGFPE, DbghSigHandler);

#ifdef LINUX
	signal(SIGSTKFLT, DbghSigHandler);
#endif // #ifdef LINUX

	signal(SIGABRT, DbghSigHandler);

	return (0);

}



int
DbghWaitPoint(char const * pszName)
{

	char const *	pszEnvVar = getenv(GDBHOOK_MODE);

	if ((pszEnvVar != NULL) && atoi(pszEnvVar))
	{
		char	szDbgFile[256] = "";

		sprintf(szDbgFile, "%s%s.%u.hook", szPath, pszName, getpid());

		FILE *	pFile = fopen(szDbgFile, "w");

		if (pFile != NULL)
		{
			time_t	tFault = time(NULL);

			fprintf(pFile,
   					"#!/bin/sh\n"
   					"#\n"
   					"# Time   = %s"
   					"# Pid    = %u\n"
   					"#\n"
   					"%s %s %u\n\n",
   					ctime(&tFault), getpid(), GDBHOOK_DEBUGGER, szProgram, getpid());
   	
   			fclose(pFile);

   			
   			chmod(szDbgFile, 0755);

   			
   			DbghWaitProc(szDbgFile);
   			
	   	}
	}

	return (0);

}



int
DbghCleanup(void)
{


	return (0);

}

RE: just-in-time debugging?

[Tony Hoyle]

On 28 Apr 2001 13:44:48 -0700, Davide Libenzi wrote:
> Sorry but why don't You run Your application with gdb ?
> Once Your program crashes You'll get the prompt and You'll be able to
> stack-trace and watching whatever You need.
> The solution I use to be able to get inside the program even when the gdb is
> not running is the one that You can find in the attached file.
> Basically it install the handler that will create a script file that You can
> use to automatically enter with gdb inside Your program while it's running.



Because the program is invoked as part of a much larger system & I don't

know which process is going to crash when.  

Having gdb come up automatically would greatly decrease development
time.  I'm trying to track down multiple bugs (caused by me, but they
still need tracking down) which show up during stress testing.  The bug
will manifest itself in maybe the 1000th iteration...   If I could hack
gdb into coming up automatically when things went wrong it'd get rid of
the need to have thousands of printf's in the app (which is my primary
debugging tool at the moment).

At work I do this all the time... Windows pops up a dialog which
basically says 'the program has crashed, debug?' and drops you straight
into VC with everything intact.  It has assertion macros which wrap int3
instructions.  You then continue your app under normal debug conditions.

Tony

(Fighting with evolution because Mozilla broke imap again... sigh...)

-- 

"Two weeks before due date, the programmers work 22 hour days cobbling an
 application from... (apparently) one programmer bashing his face into the
 keyboard." -- Dilbert

tmh@magenta-netlogic.com                http://www.nothing-on.tv 

RE: just-in-time debugging?

[Davide Libenzi]

On 28-Apr-2001 Tony Hoyle wrote:
> On 28 Apr 2001 13:44:48 -0700, Davide Libenzi wrote:
>> Sorry but why don't You run Your application with gdb ?
>> Once Your program crashes You'll get the prompt and You'll be able to
>> stack-trace and watching whatever You need.
>> The solution I use to be able to get inside the program even when the gdb is
>> not running is the one that You can find in the attached file.
>> Basically it install the handler that will create a script file that You can
>> use to automatically enter with gdb inside Your program while it's running.
> 
> 
> 
> Because the program is invoked as part of a much larger system & I don't
> 
> know which process is going to crash when.  
> 
> Having gdb come up automatically would greatly decrease development
> time.  I'm trying to track down multiple bugs (caused by me, but they
> still need tracking down) which show up during stress testing.  The bug
> will manifest itself in maybe the 1000th iteration...   If I could hack
> gdb into coming up automatically when things went wrong it'd get rid of
> the need to have thousands of printf's in the app (which is my primary
> debugging tool at the moment).
> 
> At work I do this all the time... Windows pops up a dialog which
> basically says 'the program has crashed, debug?' and drops you straight
> into VC with everything intact.  It has assertion macros which wrap int3
> instructions.  You then continue your app under normal debug conditions.

Just check the code I sent, it works fine for Your needs.



- Davide

Re: just-in-time debugging?

[Sean Hunter]

My approach is something like the others.  I developed a small wrapper to catch
unaligned traps on alpha.  What it does is run a program in gdb with some
specified arguments (it also sets up so that the process gets a SIGBUS when it
does an unaligned access, but that's probably not relevant here).

Any case, its available by anonymous ftp at ftp://uncarved.com/unaligned.c 
in case you're interested...

Sean

On Sat, Apr 28, 2001 at 09:17:10PM +0100, Tony Hoyle wrote:
> Is there a way (kernel or userspace... doesn't matter) that gdb/ddd
> could be invoked when a program is about
> to dump core, or perhaps on a certain signal (that the app could deliver
> to itself when required).  The latter case
> is what I need right now, as I have to debug an app that breaks
> seemingly randomly & I need to halt when
> certain assertions fail.  Core dumps aren't much use as you can't resume
> them, otherwise I'd just force a segfault
> or something.
> 
> I had a look at the do_coredump stuff and it looks like it could be
> altered to call gdb in the same way that
> modprobe gets called by kmod... however I don't sufficiently know the
> code to work out whether it'd work properly
> or not.  
> 
> A patch to glibc would perhaps be better, but I know that code even
> less!
> 
> Something like responding to SIGTRAP would probably be ideal.
> 
> Tony
> 
> -- 
> 
> "Two weeks before due date, the programmers work 22 hour days cobbling an
>  application from... (apparently) one programmer bashing his face into the
>  keyboard." -- Dilbert
> 
> tmh@magenta-netlogic.com                http://www.nothing-on.tv 
> 
> -
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/
> 

Re: just-in-time debugging?

[Dan Kegel]

 Tony Hoyle (tmh@nothing-on.tv) wrote:
> Is there a way that gdb/ddd could be invoked when a program is about 
> to dump core...?

Yes, I use that to get a symbolic stack dump after a crash,
although I find that the gdb so invoked doesn't accept interactive
commands, and I have to use 'kill -9' afterwards.
Here's the code I use:

void dump_stack(int signum)
{
    (void) signum;
    char s[160];
    // The right command to execute depends on the program.  Adjust to taste.
    system("echo 'info threads\nbt\nthread 3\nbt\nthread 4\nbt\nthread 5\nbt\n' > gdbcmd");
 
    sprintf(s, "gdb -batch -x gdbcmd %s %d", argv0, (int) getpid());
    printf("Crashed!  Starting debugger to get stack dump.  You may need to kill -9 this process afterwards.\n");
    system(s);
    exit(1);
 
} 

main() {
   signal(SIGSEGV, dump_stack);
   signal(SIGBUS, dump_stack);   
...