[RFC] New ausearch output option & audit viewing in Spacewalk

[RFC] New ausearch output option & audit viewing in Spacewalk

[Joshua Roys]

[-- Attachment #1: Type: text/plain, Size: 599 bytes --]

Hello all.

As part of developing an audit viewing "plugin"[1] to Spacewalk[2], I 
wrote a small program to use libauparse to output (easily) 
machine-parsable audit logs.  I think this functionality would be nice 
to have in ausearch, and as such, wrote a patch for it.

As well as reviewing this patch, I would like your feedback concerning 
the Spacewalk audit plugin.  Any questions or constructive criticism is 
welcome.

Thanks,

Joshua Roys

[1]
https://www.redhat.com/archives/spacewalk-devel/2009-May/msg00121.html
http://www.stl.gtri.gatech.edu/jroys/
[2]
http://www.redhat.com/spacewalk/

[-- Attachment #2: audit-machineinterp.patch --]
[-- Type: text/plain, Size: 8522 bytes --]

diff --git a/src/ausearch-common.h b/src/ausearch-common.h
index 179e127..2abd039 100644
--- a/src/ausearch-common.h
+++ b/src/ausearch-common.h
@@ -44,7 +44,8 @@ extern int event_exit, event_exit_is_set;
 typedef enum { F_BOTH, F_FAILED, F_SUCCESS } failed_t;
 typedef enum { C_NEITHER, C_ADD, C_DEL } conf_act_t;
 typedef enum { S_UNSET=-1, S_FAILED, S_SUCCESS } success_t;
-typedef enum { RPT_RAW, RPT_DEFAULT, RPT_INTERP, RPT_PRETTY } report_t;
+typedef enum { RPT_RAW, RPT_DEFAULT, RPT_INTERP, RPT_PRETTY, RPT_MACHINEINTERP
+} report_t;
 
 extern failed_t event_failed;
 extern conf_act_t event_conf_act;
diff --git a/src/ausearch-options.c b/src/ausearch-options.c
index 1f64b49..9edd59f 100644
--- a/src/ausearch-options.c
+++ b/src/ausearch-options.c
@@ -74,7 +74,7 @@ S_HOSTNAME, S_INTERP, S_INFILE, S_MESSAGE_TYPE, S_PID, S_SYSCALL, S_OSUCCESS,
 S_TIME_END, S_TIME_START, S_TERMINAL, S_ALL_UID, S_EFF_UID, S_UID, S_LOGINID,
 S_VERSION, S_EXACT_MATCH, S_EXECUTABLE, S_CONTEXT, S_SUBJECT, S_OBJECT,
 S_PPID, S_KEY, S_RAW, S_NODE, S_IN_LOGS, S_JUST_ONE, S_SESSION, S_EXIT,
-S_LINEBUFFERED };
+S_LINEBUFFERED, S_MACHINEINTERP };
 
 static struct nv_pair optiontab[] = {
 	{ S_EVENT, "-a" },
@@ -97,6 +97,7 @@ static struct nv_pair optiontab[] = {
 	{ S_HOSTNAME, "--host" },
 	{ S_INTERP, "-i" },
 	{ S_INTERP, "--interpret" },
+	{ S_MACHINEINTERP, "-I" },
 	{ S_INFILE, "-if" },
 	{ S_INFILE, "--input" },
 	{ S_IN_LOGS, "--input-logs" },
@@ -173,6 +174,7 @@ static void usage(void)
 	"\t-h,--help\t\t\thelp\n"
 	"\t-hn,--host <Host Name>\t\tsearch based on remote host name\n"
 	"\t-i,--interpret\t\t\tInterpret results to be human readable\n"
+	"\t-I\t\t\t\tInterpret results to be machine readable\n"
 	"\t-if,--input <Input File name>\tuse this file instead of current logs\n"
 	"\t--input-logs\t\t\tUse the logs even if stdin is a pipe\n"
 	"\t--just-one\t\t\tEmit just one event\n"
@@ -474,6 +476,22 @@ int check_params(int count, char *vars[])
         	                retval = -1;
 			}
 			break;
+		case S_MACHINEINTERP:
+			if (report_format == RPT_DEFAULT)
+				report_format = RPT_MACHINEINTERP;
+			else {
+				fprintf(stderr, 
+					"Conflicting output format %s\n",
+					vars[c]);
+        	                retval = -1;
+			}
+			if (optarg) {
+				fprintf(stderr, 
+					"Argument is NOT required for %s\n",
+					vars[c]);
+        	                retval = -1;
+			}
+			break;
 		case S_INFILE:
 			if (!optarg) {
 				fprintf(stderr, 
diff --git a/src/ausearch-report.c b/src/ausearch-report.c
index 6bb5c1a..e88b2e2 100644
--- a/src/ausearch-report.c
+++ b/src/ausearch-report.c
@@ -59,6 +59,8 @@ extern char *unescape(char *buf);
 /* Local functions */
 static void output_raw(llist *l);
 static void output_default(llist *l);
+static void output_machineinterp(llist *l);
+static void output_machineinterp_node(const lnode *n);
 static void output_interpreted(llist *l);
 static void output_interpreted_node(const lnode *n);
 static void interpret(char *name, char *val, int comma, int rtype);
@@ -84,6 +86,9 @@ void output_record(llist *l)
 			break;
 		case RPT_PRETTY:
 			break;
+		case RPT_MACHINEINTERP:
+			output_machineinterp(l);
+			break;
 		default:
 			fprintf(stderr, "Report format error");
 			exit(1);
@@ -131,6 +136,233 @@ static void output_default(llist *l)
 }
 
 /*
+ * This function will take the linked list and format it for output such that
+ * the output is both interpreted and yet easily machine-parsable (perl, etc).
+ */
+static void output_machineinterp(llist *l)
+{
+	const lnode *n;
+
+	list_last(l);
+	n = list_get_cur(l);
+	if (!n) {
+		fprintf(stderr, "Error - no elements in record.");
+		return;
+	}
+	if (n->type >= AUDIT_DAEMON_START && n->type < AUDIT_SYSCALL) 
+		output_machineinterp_node(n);
+	else {
+		do {
+			output_machineinterp_node(n);
+			printf("----\n");
+		} while ((n=list_prev(l)));
+	}
+}
+
+/*
+ * This function will cycle through a message and lookup each type that 
+ * it finds. 
+ */
+static void output_machineinterp_node(const lnode *n)
+{
+	char *ptr, *str = n->message, *node = NULL;
+
+	/* Check and see if we start with a node */
+	if (str[0] == 'n') {
+		ptr=strchr(str, ' ');
+		if (ptr) {
+			*ptr = 0;
+			node = str;
+			str = ptr+1;
+		}
+	}
+
+	// First locate time stamp.
+	ptr = strchr(str, '(');
+	if (ptr == NULL) {
+		fprintf(stderr, "can't find time stamp\n");
+		return;
+	} else {
+		time_t t;
+		int milli,num = n->type;
+		unsigned long serial;
+		char *eptr;
+		const char *bptr;
+
+		*ptr++ = 0;
+		if (num == -1) {
+			// see if we are older and wiser now.
+			bptr = strchr(str, '[');
+			if (bptr && bptr < ptr) {
+				bptr++;
+				eptr = strchr(bptr, ']');
+				if (eptr) {
+					*eptr = 0;
+					errno = 0;
+					num = strtoul(bptr, NULL, 10);
+					*eptr = ']';
+					if (errno) 
+						num = -1;
+				}
+			}
+		}
+
+		// print everything up to it.
+		if (num >= 0) {
+			bptr = audit_msg_type_to_name(num);
+			if (bptr) {
+				if (node)
+					printf("node=%s\n", node);
+				printf("type=%s\n", bptr);
+				goto no_print;
+			}
+		} 
+		if (node)
+			printf("node=%s\n", node);
+		eptr = strchr(str, ' ');
+		if (eptr)
+			*eptr = 0;
+		printf("%s\n", str);
+no_print:
+
+		str = strchr(ptr, '.');
+		if (str == NULL)
+			return;
+		*str++ = 0;
+		errno = 0;
+		t = strtoul(ptr, NULL, 10);
+		if (errno)
+			return;
+		ptr = strchr(str, ':');
+		if (ptr == NULL)
+			return;
+		*ptr++ = 0;
+		milli = strtoul(str, NULL, 10);
+		if (errno)
+			return;
+		str = strchr(ptr, ')');
+		if(str == NULL)
+			return;
+		*str++ = 0;
+		serial = strtoul(ptr, NULL, 10);
+		if (errno)
+			return;
+		printf("serial=%lu\n", serial);
+		printf("seconds=%u\n", t);
+		printf("milli=%i\n", milli);
+		str = strchr(str, ':');
+		if (str) {
+			str++;
+			if (*str == ' ')
+				str++;
+		}
+	}
+
+	if (n->type == AUDIT_SYSCALL) 
+		a0 = n->a0;
+
+	// for each item.
+	while (str && *str && (ptr = strchr(str, '='))) {
+		char *name, *val;
+		int comma = 0;
+
+		// look back to last space - this is name
+		name = ptr;
+		while (*name != ' ' && name > str)
+			--name;
+		*ptr++ = 0;
+		if (*name == ' ')
+			name++;
+
+		// Some user messages have msg='uid=500   in this case
+		// skip the msg= piece since the real stuff is the uid=
+		if (strcmp(name, "msg") == 0) {
+			// msg='text... (more text)'
+			// kill the )' on the end
+			str = strrchr(ptr+1, ')');
+			if (str && str[1] == '\'' && str[2] == 0)
+				*str = 0;
+			str = ptr+1;
+			continue;
+		}
+
+		// handle avc: ... messages
+		if (strncmp(str, "avc: ", 5) == 0) {
+			char *seresult, *seperms;
+
+			// since we know there is 'avc: ', skip it
+			// and all following spaces
+			seresult = str + 5;
+			while (*seresult == ' ')
+				seresult++;
+
+			// find the { seperms } section
+			seperms = strchr(seresult, '{');
+			if (!seperms)
+				return;
+
+			// go backward from there killing all trailing spaces
+			str = seperms - 1;
+			while (*str == ' ')
+				*str-- = 0;
+			seperms++;
+
+			// find the end of the seperms
+			str = strchr(seperms, '}');
+			if (!str)
+				return;
+			*str-- = 0;
+
+			// kill spaces on both sides of seperms
+			while (*str == ' ')
+				str--;
+			while (*seperms == ' ')
+				seperms++;
+
+			printf("seresult=%s\n", seresult);
+			printf("seperms=%s\n", seperms);
+
+			str = name;
+		}
+
+		while (*str == '(' || *str == ' ')
+			str++;
+
+		// print everything up to the '='
+		printf("%s=", str);
+
+		// get string after = to the next space or end - this is value
+		if (*ptr == '\'' || *ptr == '"') {
+			str = strchr(ptr+1, *ptr);
+			if (str) {
+				str++;
+				if (*str)
+					*str++ = 0;
+			}
+		} else {
+			str = strchr(ptr, ',');
+			val = strchr(ptr, ' ');
+			if (str && val && (str < val)) {
+				*str++ = 0;
+				comma = 1;
+			} else if (str && (val == NULL)) {
+				*str++ = 0;
+				comma = 1;
+			} else if (val) {
+				str = val;
+				*str++ = 0;
+			}
+		}
+		// val points to begin & str 1 past end
+		val = ptr;
+
+		// print interpreted string
+		interpret(name, val, comma, n->type);
+		printf("\n");
+	}
+}
+
+/*
  * This function will take the linked list and format it for output. 
  * Interpretation is performed to aid understanding of records. The output
  * order is lifo for everything.
@@ -978,7 +1210,8 @@ static void interpret(char *name, char *val, int comma, int rtype)
 			print_tty_data(val);
 			break;
 		default:
-			printf("%s%c", val, comma ? ',' : ' ');
+			printf("%s%c", val, comma && report_format !=
+				RPT_MACHINEINTERP ? ',' : ' ');
 	}
 }
 

[-- Attachment #3: Type: text/plain, Size: 0 bytes --]

Re: [RFC] New ausearch output option & audit viewing in Spacewalk

[John Dennis]

Joshua Roys wrote:
> Hello all.
> 
> As part of developing an audit viewing "plugin"[1] to Spacewalk[2], I
> wrote a small program to use libauparse to output (easily)
> machine-parsable audit logs.  I think this functionality would be nice
> to have in ausearch, and as such, wrote a patch for it.
> 
> As well as reviewing this patch, I would like your feedback concerning
> the Spacewalk audit plugin.  Any questions or constructive criticism is
> welcome.

Would you please post an example of the output and/or the parsing rules.

Thanks

-- 
John Dennis <jdennis@redhat.com>

Looking to carve out IT costs?
www.redhat.com/carveoutcosts/

Re: [RFC] New ausearch output option & audit viewing in Spacewalk

[Joshua Roys]

On 06/08/2009 01:12 PM, John Dennis wrote:
> Joshua Roys wrote:
>> Hello all.
>>
>> As part of developing an audit viewing "plugin"[1] to Spacewalk[2], I
>> wrote a small program to use libauparse to output (easily)
>> machine-parsable audit logs.  I think this functionality would be nice
>> to have in ausearch, and as such, wrote a patch for it.
>>
>> As well as reviewing this patch, I would like your feedback concerning
>> the Spacewalk audit plugin.  Any questions or constructive criticism is
>> welcome.
>
> Would you please post an example of the output and/or the parsing rules.
>
> Thanks
>

Basically the following:

type=USER_ACCT
serial=222
seconds=1244481001
milli=141
user pid=24777
uid=root
auid=unset
ses=4294967295
subj=system_u:system_r:crond_t:s0-s0:c0.c1023
op=PAM:accounting
acct=root
exe=/usr/sbin/crond
hostname=?
addr=?
terminal=cron
res=success
----
type=CRED_ACQ
serial=223
seconds=1244481001
milli=141
user pid=24777
uid=root
auid=unset
ses=4294967295
subj=system_u:system_r:crond_t:s0-s0:c0.c1023
op=PAM:setcred
acct=root
exe=/usr/sbin/crond
hostname=?
addr=?
terminal=cron
res=success
----
type=LOGIN
serial=224
seconds=1244481001
milli=141
login pid=24777
uid=root
old auid=unset
new auid=root
old ses=4294967295
new ses=34
----

Simply key=value with all nodes separated by "----\n".  I should note 
that the patch has not been exhaustively tested, it's more of a RFC and 
also a request for those more knowledgeable of the audit logs to improve 
upon it.

Thanks,

Joshua Roys

Re: [RFC] New ausearch output option & audit viewing in Spacewalk

[Steve Grubb]

On Monday 08 June 2009 12:46:37 pm Joshua Roys wrote:
> As part of developing an audit viewing "plugin"[1] to Spacewalk[2], I
> wrote a small program to use libauparse to output (easily)
> machine-parsable audit logs.  I think this functionality would be nice
> to have in ausearch, and as such, wrote a patch for it.

Very interesting work. When you apply this patch and select its output format, 
what does the output look like?


> As well as reviewing this patch, I would like your feedback concerning
> the Spacewalk audit plugin.  Any questions or constructive criticism is
> welcome.

I think this is a very interesting project. But, I have to admit that I don't 
use ausearch as the normal presentation program when I'm researching some 
audit events. For example, a typical investigation may go something like 
this:

1) you run aureport to see what is going on. hmm...no avcs...but lots of 
files, therefore you are getting hits on rules. wonder which ones?
2) you run the key report to see what the nature of hits is like. The access 
key seems to be getting a lot of hits, wonder which files it might be?
3) you run ausearch selecting the access key and pipe that into the file 
summary report. You notice one file is getting lots of hits. Wonder who is 
doing it?
4) you run ausearch selecting the access key and the file name and pipe that 
into the user summary report.
5) you notice its one acct and you wonder what all failures that person has 
had this session so you re-run the last ausearch command with --just-one so 
you can find the ses=value. Then you run ausearch --session value --success no 
and send that to aureport to get an overview of the session.
...

So, I'd recommend adding aureport's main summary and the aureport key summary 
reports to the output so that you can see if there is any reason to do a 
deeper investigation.

Interesting work!

-Steve

Re: [RFC] New ausearch output option & audit viewing in Spacewalk

[Miloslav Trmac]

Hello,
----- "Joshua Roys" <joshua.roys@gtri.gatech.edu> wrote:
> As part of developing an audit viewing "plugin"[1] to Spacewalk[2], I 
> wrote a small program to use libauparse to output (easily) 
> machine-parsable audit logs.  I think this functionality would be nice 
> to have in ausearch, and as such, wrote a patch for it.
If all you want is machine-readable name=value pairs, auparse provides a C/Python interface.  It's not 100% the same as the ausearch output, though.
    Mirek

Re: [RFC] New ausearch output option & audit viewing in Spacewalk

[John Dennis]

Joshua Roys wrote:
> Hello all.
> 
> As part of developing an audit viewing "plugin"[1] to Spacewalk[2], I
> wrote a small program to use libauparse to output (easily)
> machine-parsable audit logs.  I think this functionality would be nice
> to have in ausearch, and as such, wrote a patch for it.
> 
> As well as reviewing this patch, I would like your feedback concerning
> the Spacewalk audit plugin.  Any questions or constructive criticism is
> welcome.

[Steve may want to correct and/or comment about my statements on the
parsing logic which is in ausearch.]

The idea is interesting but I think this is the wrong implementation
approach, there should only be one library which knows how to read audit
data, namely libauparse. The code you've added is duplicating some of
the logic in libauparse. If the audit format ever changes (or you have a
parsing bug) then this code will break. The fact ausearch has logic in
it to parse audit data is historical, at the time ausearch was written
libauparse did not exist yet. I believe Steve has said that ausearch
needs to be rewritten to layer on top of libauparse.

I'm glad to see the use of "interpret" on the value, this is often
valuable, but not always. It's critical for strings. But how about
things like uid's? You probably want both the uid number and the name it
maps to, perhaps it needs to output both the raw and interpreted values
separated by deliminters, or make it an option. I'd rather see a blank
line to delimit events rather than "----".

Also, it appears as though you're outputting records and not events (an
event is the union of all records with the same ID
(node,seconds,milli,serial). It think the output should be coallesced
into events.

-- 
John Dennis <jdennis@redhat.com>

Looking to carve out IT costs?
www.redhat.com/carveoutcosts/

Re: [RFC] New ausearch output option & audit viewing in Spacewalk

[Steve Grubb]

On Monday 08 June 2009 01:43:52 pm John Dennis wrote:
> [Steve may want to correct and/or comment about my statements on the
> parsing logic which is in ausearch.]
>
> The idea is interesting but I think this is the wrong implementation
> approach, there should only be one library which knows how to read audit
> data, namely libauparse. The code you've added is duplicating some of
> the logic in libauparse. 

Actually, if you look at the test cases in the aupase library, you will see 
that it basically does the same thing. The core code from the test cases is 
this:

  do {
          if (auparse_first_record(au) <= 0)
                  exit(1);
          do {
                  const au_event_t *e = auparse_get_timestamp(au);
                  if (e == NULL)
                          exit(1);
                  printf("    event time: %u.%u:%lu, host=%s\n",
                          (unsigned)e->sec, 
                          e->milli, e->serial, e->host ? e->host : "?");
                  auparse_first_field(au);
                  do {
                          printf("        %s=%s (%s)\n",
                                          auparse_get_field_name(au),
                                          auparse_get_field_str(au),
                                          auparse_interpret_field(au));
                  } while (auparse_next_field(au) > 0);
                  printf("\n");
          } while(auparse_next_record(au) > 0);
  } while (auparse_next_event(au) > 0);


One could easily make a single purpose program in probably less that 30 lines 
of code that reproduces the same output as patching ausearch. The auparse 
library still can't reconnect interlaced records, but you could init the app 
with AUSOURCE_DESCRIPTOR as the data source (for stdin) and pipe the ouput of 
ausearch --raw into the single purpose reformatter.


> If the audit format ever changes (or you have a 
> parsing bug) then this code will break. The fact ausearch has logic in
> it to parse audit data is historical, at the time ausearch was written
> libauparse did not exist yet. I believe Steve has said that ausearch
> needs to be rewritten to layer on top of libauparse.

This is very true. Some day it will be layered on top of auparse.

-Steve