[PATCH v3 ulogd 0/12] make progress ulogd_output_IPFIX

[PATCH v3 ulogd 0/12] make progress ulogd_output_IPFIX

[Ken-ichirou MATSUZAWA]

 Hello,

This patches confined primarly to build IPFIX message, does not send it yet.
Changes from v2:
  does not add new u64 usec okey at NFCT
  some minor fixes related to IPFIX at NFCT
  compare input keys using nfct_bitmask_test_bit

Thanks,

[PATCH v3 ulogd 01/12] ipfix: use nfct_bitmask

[Ken-ichirou MATSUZAWA]

from libnetfilter_conntrack instead of original

Signed-off-by Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp>
---
 configure.ac                |  12 +++++
 output/Makefile.am          |  10 +++++
 output/ulogd_output_IPFIX.c | 107 ++++++--------------------------------------
 3 files changed, 35 insertions(+), 94 deletions(-)

diff --git a/configure.ac b/configure.ac
index 522c345..bd46323 100644
--- a/configure.ac
+++ b/configure.ac
@@ -56,6 +56,17 @@ AC_ARG_ENABLE(nfct,
 AS_IF([test "x$enable_nfct" = "xyes"], [
     PKG_CHECK_MODULES([LIBNETFILTER_CONNTRACK], [libnetfilter_conntrack >= 1.0.2])
     AC_DEFINE([BUILD_NFCT], [1], [Building nfct module])
+
+    AC_MSG_CHECKING([does nfct_bitmask have clear and equal])
+    AC_CACHE_VAL(ac_cv_nfct_bitmask_clear_equal,
+    AC_TRY_COMPILE(
+        [ #include <libnetfilter_conntrack/libnetfilter_conntrack.h>],
+        [ struct nfct_bitmask *b = nrct_bitmask_new(8);
+          nfct_bitmask_clear(b); nfct_bitmask_equal(b, b); ],
+        ac_cv_nfct_bitmask_clear_equal=yes,
+        ac_cv_nfct_bitmask_clear_equal=no))
+    AC_MSG_RESULT($ac_cv_nfct_bitmask_clear_equal)
+    AM_CONDITIONAL([BUILD_IPFIX], [test "x$ac_cv_nfct_bitmask_clear_equal" = "xyes"])
 ])
 AM_CONDITIONAL([BUILD_NFCT], [test "x$enable_nfct" = "xyes"])
 AC_ARG_ENABLE(nfacct,
@@ -164,5 +175,6 @@ Ulogd configuration:
     SQLITE3 plugin:			${enable_sqlite3}
     DBI plugin:				${enable_dbi}
     JSON plugin:			${enable_jansson}
+    IPFIX plugin:			${ac_cv_nfct_bitmask_clear_equal}
 "
 echo "You can now run 'make' and 'make install'"
diff --git a/output/Makefile.am b/output/Makefile.am
index ff851ad..0cb4a20 100644
--- a/output/Makefile.am
+++ b/output/Makefile.am
@@ -13,6 +13,10 @@ if HAVE_JANSSON
 pkglib_LTLIBRARIES += ulogd_output_JSON.la
 endif
 
+if BUILD_IPFIX
+pkglib_LTLIBRARIES += ulogd_output_IPFIX.la
+endif
+
 ulogd_output_GPRINT_la_SOURCES = ulogd_output_GPRINT.c
 ulogd_output_GPRINT_la_LDFLAGS = -avoid-version -module
 
@@ -42,3 +46,9 @@ ulogd_output_JSON_la_SOURCES = ulogd_output_JSON.c
 ulogd_output_JSON_la_LIBADD  = ${libjansson_LIBS}
 ulogd_output_JSON_la_LDFLAGS = -avoid-version -module
 endif
+
+if BUILD_IPFIX
+ulogd_output_IPFIX_la_SOURCES = ulogd_output_IPFIX.c
+ulogd_output_IPFIX_la_LDFLAGS = -avoid-version -module
+ulogd_output_IPFIX_la_LIBADD = ${LIBNETFILTER_CONNTRACK_LIBS}
+endif
diff --git a/output/ulogd_output_IPFIX.c b/output/ulogd_output_IPFIX.c
index 761d272..01ac9a0 100644
--- a/output/ulogd_output_IPFIX.c
+++ b/output/ulogd_output_IPFIX.c
@@ -36,6 +36,8 @@
 #include <sys/socket.h>
 #include <netdb.h>
 
+#include <libnetfilter_conntrack/libnetfilter_conntrack.h>
+
 #include <ulogd/linuxlist.h>
 
 #ifdef IPPROTO_SCTP
@@ -68,89 +70,6 @@ struct sctp_sndrcvinfo {
 
 #define IPFIX_DEFAULT_TCPUDP_PORT	4739
 
-/* bitmask stuff */
-struct bitmask {
-	int size_bits;
-	char *buf;
-};
-
-#define SIZE_OCTETS(x)	((x/8)+1)
-
-void bitmask_clear(struct bitmask *bm)
-{
-	memset(bm->buf, 0, SIZE_OCTETS(bm->size_bits));
-}
-
-struct bitmask *bitmask_alloc(unsigned int num_bits)
-{
-	struct bitmask *bm;
-	unsigned int size_octets = SIZE_OCTETS(num_bits);
-
-	bm = malloc(sizeof(*bm) + size_octets);
-	if (!bm)
-		return NULL;
-
-	bm->size_bits = num_bits;
-	bm->buf = (void *)bm + sizeof(*bm);
-
-	bitmask_clear(bm);
-
-	return bm;
-}
-
-void bitmask_free(struct bitmask *bm)
-{
-	free(bm);
-}
-
-int bitmask_set_bit_to(struct bitmask *bm, unsigned int bits, int to)
-{
-	unsigned int byte = bits / 8;
-	unsigned int bit = bits % 8;
-	unsigned char *ptr;
-
-	if (byte > SIZE_OCTETS(bm->size_bits))
-		return -EINVAL;
-
-	if (to == 0)
-		bm->buf[byte] &= ~(1 << bit);
-	else
-		bm->buf[byte] |= (1 << bit);
-
-	return 0;
-}
-
-#define bitmask_clear_bit(bm, bit) \
-	bitmask_set_bit_to(bm, bit, 0)
-
-#define bitmask_set_bit(bm, bit) \
-	bitmask_set_bit_to(bm, bit, 1)
-
-int bitmasks_equal(const struct bitmask *bm1, const struct bitmask *bm2)
-{
-	if (bm1->size_bits != bm2->size_bits)
-		return -1;
-
-	if (!memcmp(bm1->buf, bm2->buf, SIZE_OCTETS(bm1->size_bits)))
-		return 1;
-	else
-		return 0;
-}
-
-struct bitmask *bitmask_dup(const struct bitmask *bm_orig)
-{
-	struct bitmask *bm_new;
-	int size = sizeof(*bm_new) + SIZE_OCTETS(bm_orig->size_bits);
-
-	bm_new = malloc(size);
-	if (!bm_new)
-		return NULL;
-
-	memcpy(bm_new, bm_orig, size);
-
-	return bm_new;
-}
-
 static struct config_keyset ipfix_kset = {
 	.num_ces = 3,
 	.ces = {
@@ -185,7 +104,7 @@ struct ipfix_template {
 
 struct ulogd_ipfix_template {
 	struct llist_head list;
-	struct bitmask *bitmask;
+	struct nfct_bitmask *bitmask;
 	unsigned int total_length;	/* length of the DATA */
 	char *tmpl_cur;		/* cursor into current template position */
 	struct ipfix_template tmpl;
@@ -201,7 +120,7 @@ struct ipfix_instance {
 	struct ipfix_template *tmpl;
 	unsigned int tmpl_len;
 
-	struct bitmask *valid_bitmask;	/* bitmask of valid keys */
+	struct nfct_bitmask *valid_bitmask;	/* bitmask of valid keys */
 
 	unsigned int total_length;	/* total size of all data elements */
 };
@@ -212,7 +131,7 @@ static u_int16_t next_template_id = ULOGD_IPFIX_TEMPL_BASE;
 /* Build the IPFIX template from the input keys */
 struct ulogd_ipfix_template *
 build_template_for_bitmask(struct ulogd_pluginstance *upi,
-			   struct bitmask *bm)
+			   struct nfct_bitmask *bm)
 {
 	struct ipfix_instance *ii = (struct ipfix_instance *) &upi->private;
 	struct ipfix_templ_rec_hdr *rhdr;
@@ -226,7 +145,7 @@ build_template_for_bitmask(struct ulogd_pluginstance *upi,
 		return NULL;
 	memset(tmpl, 0, size);
 
-	tmpl->bitmask = bitmask_dup(bm);
+	tmpl->bitmask = nfct_bitmask_clone(bm);
 	if (!tmpl->bitmask) {
 		free(tmpl);
 		return NULL;
@@ -288,14 +207,14 @@ build_template_for_bitmask(struct ulogd_pluginstance *upi,
 
 static struct ulogd_ipfix_template *
 find_template_for_bitmask(struct ulogd_pluginstance *upi,
-			  struct bitmask *bm)
+			  struct nfct_bitmask *bm)
 {
 	struct ipfix_instance *ii = (struct ipfix_instance *) &upi->private;
 	struct ulogd_ipfix_template *tmpl;
 	
 	/* FIXME: this can be done more efficient! */
 	llist_for_each_entry(tmpl, &ii->template_list, list) {
-		if (bitmasks_equal(bm, tmpl->bitmask))
+		if (nfct_bitmask_equal(bm, tmpl->bitmask))
 			return tmpl;
 	}
 	return NULL;
@@ -313,13 +232,13 @@ static int output_ipfix(struct ulogd_pluginstance *upi)
 	 * ulogd core could very easily flush it after every packet,
 	 * too. */
 
-	bitmask_clear(ii->valid_bitmask);
+	nfct_bitmask_clear(ii->valid_bitmask);
 
 	for (i = 0; i < upi->input.num_keys; i++) {
 		struct ulogd_key *key = upi->input.keys[i].u.source;
 
 		if (key->flags & ULOGD_RETF_VALID)
-			bitmask_set_bit(ii->valid_bitmask, i);
+			nfct_bitmask_set_bit(ii->valid_bitmask, i);
 	}
 	
 	/* lookup template ID for this bitmask */
@@ -430,7 +349,7 @@ static int start_ipfix(struct ulogd_pluginstance *pi)
 
 	ulogd_log(ULOGD_DEBUG, "starting ipfix\n");
 
-	ii->valid_bitmask = bitmask_alloc(pi->input.num_keys);
+	ii->valid_bitmask = nfct_bitmask_new(pi->input.num_keys);
 	if (!ii->valid_bitmask)
 		return -ENOMEM;
 
@@ -443,7 +362,7 @@ static int start_ipfix(struct ulogd_pluginstance *pi)
 	return 0;
 
 out_bm_free:
-	bitmask_free(ii->valid_bitmask);
+	nfct_bitmask_destroy(ii->valid_bitmask);
 	ii->valid_bitmask = NULL;
 
 	return ret;
@@ -455,7 +374,7 @@ static int stop_ipfix(struct ulogd_pluginstance *pi)
 
 	close(ii->fd);
 
-	bitmask_free(ii->valid_bitmask);
+	nfct_bitmask_destroy(ii->valid_bitmask);
 	ii->valid_bitmask = NULL;
 
 	return 0;
-- 
1.9.1

[PATCH v3 ulogd 02/12] ipfix: fix enterprise bit handling

[Ken-ichirou MATSUZAWA]

If this bit is zero, the Information Element identifier identifies an
Information Element in IANA-IPFIX, and the four-octet Enterprise Number field
MUST NOT be present.

Signed-off-by Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp>
---
 output/ulogd_output_IPFIX.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/output/ulogd_output_IPFIX.c b/output/ulogd_output_IPFIX.c
index 01ac9a0..333d532 100644
--- a/output/ulogd_output_IPFIX.c
+++ b/output/ulogd_output_IPFIX.c
@@ -182,15 +182,15 @@ build_template_for_bitmask(struct ulogd_pluginstance *upi,
 			struct ipfix_ietf_field *field = 
 				(struct ipfix_ietf_field *) tmpl->tmpl_cur;
 
-			field->type = htons(key->ipfix.field_id | 0x8000000);
+			field->type = htons(key->ipfix.field_id);
 			field->length = htons(length);
 			tmpl->tmpl_cur += sizeof(*field);
 		} else {
 			struct ipfix_vendor_field *field =
 				(struct ipfix_vendor_field *) tmpl->tmpl_cur;
 
+			field->type = htons(key->ipfix.field_id | 0x8000);
 			field->enterprise_num = htonl(key->ipfix.vendor);
-			field->type = htons(key->ipfix.field_id);
 			field->length = htons(length);
 			tmpl->tmpl_cur += sizeof(*field);
 		}
-- 
1.9.1

[PATCH v3 ulogd 03/12] ipfix: some cleanups

[Ken-ichirou MATSUZAWA]

remove unused variables.
use unsigned int to supress gcc warning.
change struct field name from total_length to data_length.
compare bitmask by nfct_bitmask_test_bit().
change tmpl_cur to void * to calc size.

Signed-off-by Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp>
---
 output/ulogd_output_IPFIX.c | 51 +++++++++++++++------------------------------
 1 file changed, 17 insertions(+), 34 deletions(-)

diff --git a/output/ulogd_output_IPFIX.c b/output/ulogd_output_IPFIX.c
index 333d532..a18b0d8 100644
--- a/output/ulogd_output_IPFIX.c
+++ b/output/ulogd_output_IPFIX.c
@@ -105,8 +105,8 @@ struct ipfix_template {
 struct ulogd_ipfix_template {
 	struct llist_head list;
 	struct nfct_bitmask *bitmask;
-	unsigned int total_length;	/* length of the DATA */
-	char *tmpl_cur;		/* cursor into current template position */
+	unsigned int data_length;	/* length of the DATA */
+	void *tmpl_cur;			/* cursor into current template position */
 	struct ipfix_template tmpl;
 };
 
@@ -116,13 +116,7 @@ struct ipfix_instance {
 	int sock_proto;	/* protocol (IPPROTO_*) */
 
 	struct llist_head template_list;
-
-	struct ipfix_template *tmpl;
-	unsigned int tmpl_len;
-
 	struct nfct_bitmask *valid_bitmask;	/* bitmask of valid keys */
-
-	unsigned int total_length;	/* total size of all data elements */
 };
 
 #define ULOGD_IPFIX_TEMPL_BASE 1024
@@ -133,8 +127,6 @@ struct ulogd_ipfix_template *
 build_template_for_bitmask(struct ulogd_pluginstance *upi,
 			   struct nfct_bitmask *bm)
 {
-	struct ipfix_instance *ii = (struct ipfix_instance *) &upi->private;
-	struct ipfix_templ_rec_hdr *rhdr;
 	struct ulogd_ipfix_template *tmpl;
 	unsigned int i, j;
 	int size = sizeof(struct ulogd_ipfix_template)
@@ -156,27 +148,14 @@ build_template_for_bitmask(struct ulogd_pluginstance *upi,
 
 	tmpl->tmpl_cur = tmpl->tmpl.buf;
 
-	tmpl->total_length = 0;
+	tmpl->data_length = 0;
 
 	for (i = 0, j = 0; i < upi->input.num_keys; i++) {
 		struct ulogd_key *key = &upi->input.keys[i];
 		int length = ulogd_key_size(key);
 
-		if (!(key->u.source->flags & ULOGD_RETF_VALID))
-			continue;
-
-		if (length < 0 || length > 0xfffe) {
-			ulogd_log(ULOGD_INFO, "ignoring key `%s' because "
-				  "it has an ipfix incompatible length\n",
-				  key->name);
+		if (!nfct_bitmask_test_bit(tmpl->bitmask, i))
 			continue;
-		}
-
-		if (key->ipfix.field_id == 0) {
-			ulogd_log(ULOGD_INFO, "ignoring key `%s' because "
-				  "it has no field_id\n", key->name);
-			continue;
-		}
 
 		if (key->ipfix.vendor == IPFIX_VENDOR_IETF) {
 			struct ipfix_ietf_field *field = 
@@ -194,7 +173,7 @@ build_template_for_bitmask(struct ulogd_pluginstance *upi,
 			field->length = htons(length);
 			tmpl->tmpl_cur += sizeof(*field);
 		}
-		tmpl->total_length += length;
+		tmpl->data_length += length;
 		j++;
 	}
 
@@ -224,8 +203,7 @@ static int output_ipfix(struct ulogd_pluginstance *upi)
 {
 	struct ipfix_instance *ii = (struct ipfix_instance *) &upi->private;
 	struct ulogd_ipfix_template *template;
-	unsigned int total_size;
-	int i;
+	unsigned int total_size, i;
 
 	/* FIXME: it would be more cache efficient if the IS_VALID
 	 * flags would be a separate bitmask outside of the array.
@@ -235,10 +213,17 @@ static int output_ipfix(struct ulogd_pluginstance *upi)
 	nfct_bitmask_clear(ii->valid_bitmask);
 
 	for (i = 0; i < upi->input.num_keys; i++) {
-		struct ulogd_key *key = upi->input.keys[i].u.source;
+		struct ulogd_key *key = &upi->input.keys[i];
+		int length = ulogd_key_size(key);
 
-		if (key->flags & ULOGD_RETF_VALID)
-			nfct_bitmask_set_bit(ii->valid_bitmask, i);
+		if (length < 0 || length > 0xfffe)
+			continue;
+		if (!(key->u.source->flags & ULOGD_RETF_VALID))
+			continue;
+		if (key->ipfix.field_id == 0)
+			continue;
+
+		nfct_bitmask_set_bit(ii->valid_bitmask, i);
 	}
 	
 	/* lookup template ID for this bitmask */
@@ -253,7 +238,7 @@ static int output_ipfix(struct ulogd_pluginstance *upi)
 		llist_add(&template->list, &ii->template_list);
 	}
 	
-	total_size = template->total_length;
+	total_size = template->data_length;
 
 	/* decide if it's time to retransmit our template and (optionally)
 	 * prepend it into the to-be-sent IPFIX message */
@@ -382,8 +367,6 @@ static int stop_ipfix(struct ulogd_pluginstance *pi)
 
 static void signal_handler_ipfix(struct ulogd_pluginstance *pi, int signal)
 {
-	struct ipfix_instance *li = (struct ipfix_instance *) &pi->private;
-
 	switch (signal) {
 	case SIGHUP:
 		ulogd_log(ULOGD_NOTICE, "ipfix: reopening connection\n");
-- 
1.9.1

[PATCH v3 ulogd 04/12] ipfix: add functions for ipfix dataset creation

[Ken-ichirou MATSUZAWA]

ulogd_key_putn() put key's value in network byteorder.
put_data_records() creates ipfix data records buffer.

Signed-off-by Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp>
---
 output/ulogd_output_IPFIX.c | 87 +++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 85 insertions(+), 2 deletions(-)

diff --git a/output/ulogd_output_IPFIX.c b/output/ulogd_output_IPFIX.c
index a18b0d8..49efb54 100644
--- a/output/ulogd_output_IPFIX.c
+++ b/output/ulogd_output_IPFIX.c
@@ -63,6 +63,19 @@ struct sctp_sndrcvinfo {
 };
 #endif
 
+#include <byteswap.h>
+#if __BYTE_ORDER == __BIG_ENDIAN
+#  ifndef __be64_to_cpu
+#  define __be64_to_cpu(x)	(x)
+#  endif
+# else
+# if __BYTE_ORDER == __LITTLE_ENDIAN
+#  ifndef __be64_to_cpu
+#  define __be64_to_cpu(x)	__bswap_64(x)
+#  endif
+# endif
+#endif
+
 #include <ulogd/ulogd.h>
 #include <ulogd/conffile.h>
 #include <ulogd/linuxlist.h>
@@ -182,8 +195,6 @@ build_template_for_bitmask(struct ulogd_pluginstance *upi,
 	return tmpl;
 }
 
-
-
 static struct ulogd_ipfix_template *
 find_template_for_bitmask(struct ulogd_pluginstance *upi,
 			  struct nfct_bitmask *bm)
@@ -199,6 +210,78 @@ find_template_for_bitmask(struct ulogd_pluginstance *upi,
 	return NULL;
 }
 
+static int ulogd_key_putn(struct ulogd_key *key, void *buf)
+{
+	int ret;
+
+	switch (key->type) {
+	case ULOGD_RET_INT8:
+	case ULOGD_RET_UINT8:
+	case ULOGD_RET_BOOL:
+		*(u_int8_t *)buf = ikey_get_u8(key);
+		ret = 1;
+		break;
+	case ULOGD_RET_INT16:
+	case ULOGD_RET_UINT16:
+		*(u_int16_t *)buf = htons(ikey_get_u16(key));
+		ret = 2;
+		break;
+	case ULOGD_RET_INT32:
+	case ULOGD_RET_UINT32:
+		*(u_int32_t *)buf = htonl(ikey_get_u32(key));
+		ret = 4;
+		break;
+	case ULOGD_RET_IPADDR:
+		*(u_int32_t *)buf = ikey_get_u32(key);
+		ret = 4;
+		break;
+	case ULOGD_RET_INT64:
+	case ULOGD_RET_UINT64:
+		*(u_int64_t *)buf = __be64_to_cpu(ikey_get_u64(key));
+		ret = 8;
+		break;
+	case ULOGD_RET_IP6ADDR:
+		memcpy(buf, ikey_get_u128(key), 16);
+		ret = 16;
+		break;
+	case ULOGD_RET_STRING:
+		ret = strlen(key->u.value.ptr);
+		memcpy(buf, key->u.value.ptr, ret);
+		break;
+	case ULOGD_RET_RAW:
+		ulogd_log(ULOGD_NOTICE, "put raw data in network byte order "
+			  "`%s' type 0x%x\n", key->name, key->type);
+		ret = key->len;
+		memcpy(buf, key->u.value.ptr, ret);
+		break;
+	default:
+		ulogd_log(ULOGD_ERROR, "unknown size - key "
+			  "`%s' type 0x%x\n", key->name, key->type);
+		ret = -1;
+		break;
+	}
+
+	return ret;
+}
+
+static int put_data_records(struct ulogd_pluginstance *upi,
+			    struct ulogd_ipfix_template *tmpl, void *buf)
+{
+	int ret;
+	unsigned int i, len = 0;
+
+	for (i = 0; i < upi->input.num_keys; i++) {
+		if (!nfct_bitmask_test_bit(tmpl->bitmask, i))
+			continue;
+		ret = ulogd_key_putn(&upi->input.keys[i], buf + len);
+		if (ret < 0)
+			return ret;
+		len += ret;
+	}
+
+	return len;
+}
+
 static int output_ipfix(struct ulogd_pluginstance *upi)
 {
 	struct ipfix_instance *ii = (struct ipfix_instance *) &upi->private;
-- 
1.9.1

[PATCH v3 ulogd 05/12] ipfix: add function for ipfix message creation

[Ken-ichirou MATSUZAWA]

This function creates ipfix message, template and data part but not scope.
Header sequence is kept by struct ipfix_instance, domain id is specified
by config file. The returned value has no export time so caller set this
and free the value after using it.

Signed-off-by Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp>
---
 include/ulogd/ipfix_protocol.h |  8 ++++-
 output/ulogd_output_IPFIX.c    | 71 +++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 77 insertions(+), 2 deletions(-)

diff --git a/include/ulogd/ipfix_protocol.h b/include/ulogd/ipfix_protocol.h
index 5d7e46a..266897e 100644
--- a/include/ulogd/ipfix_protocol.h
+++ b/include/ulogd/ipfix_protocol.h
@@ -15,7 +15,13 @@ struct ipfix_msg_hdr {
 	u_int16_t	length;
 	u_int32_t	export_time;
 	u_int32_t	seq;
-	u_int32_t	source_id;
+	u_int32_t	domain_id;
+};
+
+/* Section 3.3.2 */
+struct ipfix_set_hdr {
+	u_int16_t       set_id;
+	u_int16_t       length;
 };
 
 /* Section 3.4.1 */
diff --git a/output/ulogd_output_IPFIX.c b/output/ulogd_output_IPFIX.c
index 49efb54..352b5a7 100644
--- a/output/ulogd_output_IPFIX.c
+++ b/output/ulogd_output_IPFIX.c
@@ -28,6 +28,7 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <stdbool.h>
 #include <unistd.h>
 #include <string.h>
 #include <errno.h>
@@ -84,7 +85,7 @@ struct sctp_sndrcvinfo {
 #define IPFIX_DEFAULT_TCPUDP_PORT	4739
 
 static struct config_keyset ipfix_kset = {
-	.num_ces = 3,
+	.num_ces = 4,
 	.ces = {
 		{
 			.key 	 = "host",
@@ -103,12 +104,19 @@ static struct config_keyset ipfix_kset = {
 			.options = CONFIG_OPT_NONE,
 			.u	= { .string = "udp" },
 		},
+		{
+			.key	 = "domain_id",
+			.type	 = CONFIG_TYPE_INT,
+			.options = CONFIG_OPT_NONE,
+			.u.value = 0,
+		},
 	},
 };
 
 #define host_ce(x)	(x->ces[0])
 #define port_ce(x)	(x->ces[1])
 #define proto_ce(x)	(x->ces[2])
+#define domain_ce(x)	(x->ces[3])
 
 struct ipfix_template {
 	struct ipfix_templ_rec_hdr hdr;
@@ -130,6 +138,7 @@ struct ipfix_instance {
 
 	struct llist_head template_list;
 	struct nfct_bitmask *valid_bitmask;	/* bitmask of valid keys */
+	u_int32_t seq;
 };
 
 #define ULOGD_IPFIX_TEMPL_BASE 1024
@@ -282,6 +291,66 @@ static int put_data_records(struct ulogd_pluginstance *upi,
 	return len;
 }
 
+static struct ipfix_msg_hdr *build_ipfix_msg(struct ulogd_pluginstance *upi,
+					     struct ulogd_ipfix_template *template,
+					     bool need_template)
+{
+	struct ipfix_instance *ii = (struct ipfix_instance *) &upi->private;
+	u_int16_t tmpl_len;
+	struct ipfix_msg_hdr *msg_hdr;
+	struct ipfix_templ_rec_hdr *tmpl_hdr;
+	struct ipfix_set_hdr *data_hdr, *tmpl_set_hdr;
+	void *buf;
+	int msglen, ret;
+
+	msglen = sizeof(struct ipfix_msg_hdr) + sizeof(struct ipfix_set_hdr)
+		+ template->data_length;
+	if (need_template)
+		msglen = msglen + sizeof(struct ipfix_set_hdr)
+			+ (template->tmpl_cur - (void *)&template->tmpl);
+	buf = malloc(msglen);
+	if (buf == NULL)
+		return NULL;
+	memset(buf, 0, msglen);
+
+	/* ipfix msg header */
+	msg_hdr = buf;
+	msg_hdr->version = htons(10);
+	msg_hdr->length = htons(msglen);
+	msg_hdr->seq = htonl(ii->seq++);
+	msg_hdr->domain_id = htonl(domain_ce(upi->config_kset).u.value);
+	if (need_template) {
+		/* put set header and template records */
+		tmpl_set_hdr = buf + sizeof(*msg_hdr);
+		tmpl_set_hdr->set_id = htons(2);
+		tmpl_len = template->tmpl_cur - (void *)&template->tmpl;
+		tmpl_set_hdr->length = htons(sizeof(*tmpl_set_hdr) + tmpl_len);
+		tmpl_hdr = (void *)tmpl_set_hdr + sizeof(*tmpl_set_hdr);
+		memcpy((void *)tmpl_hdr, (void *)&template->tmpl, tmpl_len);
+		data_hdr = (void *)tmpl_hdr + tmpl_len;
+	} else {
+		data_hdr = buf + sizeof(*msg_hdr);
+	}
+
+	/* put set header and data records */
+	data_hdr->set_id = template->tmpl.hdr.templ_id; /* already ordered */
+	data_hdr->length = htons(sizeof(*data_hdr) + template->data_length);
+	ret = put_data_records(upi, template, (void *)data_hdr + sizeof(*data_hdr));
+	if (ret < 0) {
+		ulogd_log(ULOGD_ERROR, "could not build ipfix dataset");
+		goto free_buf;
+	} else if (ret > msglen) {
+		ulogd_log(ULOGD_ERROR, "overflowed on building ipfix dataset");
+		goto free_buf;
+	}
+
+	return msg_hdr;
+
+free_buf:
+	free(buf);
+	return NULL;
+}
+
 static int output_ipfix(struct ulogd_pluginstance *upi)
 {
 	struct ipfix_instance *ii = (struct ipfix_instance *) &upi->private;
-- 
1.9.1

[PATCH v3 ulogd 06/12] ipfix: decide whether prepending template by send times

[Ken-ichirou MATSUZAWA]

specified in config file and kept it until_template in struct ulogd_ipfix_template

Signed-off-by Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp>
---
 output/ulogd_output_IPFIX.c | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/output/ulogd_output_IPFIX.c b/output/ulogd_output_IPFIX.c
index 352b5a7..f9e23bb 100644
--- a/output/ulogd_output_IPFIX.c
+++ b/output/ulogd_output_IPFIX.c
@@ -85,7 +85,7 @@ struct sctp_sndrcvinfo {
 #define IPFIX_DEFAULT_TCPUDP_PORT	4739
 
 static struct config_keyset ipfix_kset = {
-	.num_ces = 4,
+	.num_ces = 5,
 	.ces = {
 		{
 			.key 	 = "host",
@@ -110,6 +110,12 @@ static struct config_keyset ipfix_kset = {
 			.options = CONFIG_OPT_NONE,
 			.u.value = 0,
 		},
+		{
+			.key	 = "send_template_per",
+			.type	 = CONFIG_TYPE_INT,
+			.options = CONFIG_OPT_NONE,
+			.u.value = 16,
+		},
 	},
 };
 
@@ -117,6 +123,7 @@ static struct config_keyset ipfix_kset = {
 #define port_ce(x)	(x->ces[1])
 #define proto_ce(x)	(x->ces[2])
 #define domain_ce(x)	(x->ces[3])
+#define template_per_ce(x)	(x->ces[4])
 
 struct ipfix_template {
 	struct ipfix_templ_rec_hdr hdr;
@@ -129,6 +136,7 @@ struct ulogd_ipfix_template {
 	unsigned int data_length;	/* length of the DATA */
 	void *tmpl_cur;			/* cursor into current template position */
 	struct ipfix_template tmpl;
+	int until_template;			/* decide if it's time to retransmit our template */
 };
 
 struct ipfix_instance {
@@ -171,6 +179,7 @@ build_template_for_bitmask(struct ulogd_pluginstance *upi,
 	tmpl->tmpl_cur = tmpl->tmpl.buf;
 
 	tmpl->data_length = 0;
+	tmpl->until_template = template_per_ce(upi->config_kset).u.value;
 
 	for (i = 0, j = 0; i < upi->input.num_keys; i++) {
 		struct ulogd_key *key = &upi->input.keys[i];
@@ -355,7 +364,8 @@ static int output_ipfix(struct ulogd_pluginstance *upi)
 {
 	struct ipfix_instance *ii = (struct ipfix_instance *) &upi->private;
 	struct ulogd_ipfix_template *template;
-	unsigned int total_size, i;
+	unsigned int i;
+	bool need_template = false;
 
 	/* FIXME: it would be more cache efficient if the IS_VALID
 	 * flags would be a separate bitmask outside of the array.
@@ -388,16 +398,12 @@ static int output_ipfix(struct ulogd_pluginstance *upi)
 			return ULOGD_IRET_ERR;
 		}
 		llist_add(&template->list, &ii->template_list);
+		need_template = true;
 	}
 	
-	total_size = template->data_length;
-
-	/* decide if it's time to retransmit our template and (optionally)
-	 * prepend it into the to-be-sent IPFIX message */
-	if (0 /* FIXME */) {
-		/* add size of template */
-		//total_size += (template->tmpl_cur - (void *)&template->tmpl);
-		total_size += sizeof(template->tmpl);
+	if (template->until_template-- == 0) {
+		need_template = true;
+		template->until_template = template_per_ce(upi->config_kset).u.value;
 	}
 
 	return ULOGD_IRET_OK;
-- 
1.9.1

[PATCH v3 ulogd 07/12] ipfix: print ipfix message

[Ken-ichirou MATSUZAWA]

by messy ipfix_fprintf_ functions.

Signed-off-by Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp>
---
 output/ulogd_output_IPFIX.c | 226 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 226 insertions(+)

diff --git a/output/ulogd_output_IPFIX.c b/output/ulogd_output_IPFIX.c
index f9e23bb..0ff2efe 100644
--- a/output/ulogd_output_IPFIX.c
+++ b/output/ulogd_output_IPFIX.c
@@ -31,6 +31,7 @@
 #include <stdbool.h>
 #include <unistd.h>
 #include <string.h>
+#include <time.h>
 #include <errno.h>
 
 #include <sys/types.h>
@@ -152,6 +153,8 @@ struct ipfix_instance {
 #define ULOGD_IPFIX_TEMPL_BASE 1024
 static u_int16_t next_template_id = ULOGD_IPFIX_TEMPL_BASE;
 
+static int ipfix_fprintf_header(FILE *fd, const struct ipfix_msg_hdr *hdr);
+
 /* Build the IPFIX template from the input keys */
 struct ulogd_ipfix_template *
 build_template_for_bitmask(struct ulogd_pluginstance *upi,
@@ -364,6 +367,7 @@ static int output_ipfix(struct ulogd_pluginstance *upi)
 {
 	struct ipfix_instance *ii = (struct ipfix_instance *) &upi->private;
 	struct ulogd_ipfix_template *template;
+	struct ipfix_msg_hdr *ipfix_msg;
 	unsigned int i;
 	bool need_template = false;
 
@@ -406,6 +410,15 @@ static int output_ipfix(struct ulogd_pluginstance *upi)
 		template->until_template = template_per_ce(upi->config_kset).u.value;
 	}
 
+	ipfix_msg = build_ipfix_msg(upi, template, need_template);
+	if (ipfix_msg == NULL)
+		return ULOGD_IRET_ERR;
+
+	ipfix_msg->export_time = htonl((u_int32_t)(time(NULL)));
+	ipfix_fprintf_header(stdout, ipfix_msg);
+	fprintf(stdout, "\n");
+
+	free(ipfix_msg);
 	return ULOGD_IRET_OK;
 }
 
@@ -604,3 +617,216 @@ void init(void)
 {
 	ulogd_register_plugin(&ipfix_plugin);
 }
+
+static int ipfix_fprintf_ietf_field(FILE *fd, const struct ipfix_ietf_field *field, int len);
+static int ipfix_fprintf_vendor_field(FILE *fd, const struct ipfix_vendor_field *field, int len);
+
+static int ipfix_fprintf_ietf_field(FILE *fd, const struct ipfix_ietf_field *field,
+				    int len)
+{
+	int ret;
+	void *ptr;
+
+	if (len < (int)sizeof(*field)) {
+		fprintf(fd, "ERROR ietf field: too short buflen for IETF field: %d\n", len);
+		return -1;
+	}
+
+	fprintf(fd, "+--------------------------------+--------------------------------+\n");
+	fprintf(fd, "|0 Information Emement id: %5d |            Field Length: %5d |\n",
+		ntohs(field->type), ntohs(field->length));
+
+	len -= sizeof(*field);
+	if (len == 0)
+		return sizeof(*field);
+
+	ptr = (void *)field + sizeof(*field);
+	if (*(u_int8_t *)ptr & 0x80)
+		ret = ipfix_fprintf_vendor_field(fd, ptr, len);
+	else
+		ret = ipfix_fprintf_ietf_field(fd, ptr, len);
+
+	if (ret == -1)
+		return -1;
+	return ret + sizeof(*field);
+}
+
+static int ipfix_fprintf_vendor_field(FILE *fd, const struct ipfix_vendor_field *field,
+				      int len)
+{
+	int ret;
+	void *ptr;
+
+	if (len < (int)sizeof(*field)) {
+		fprintf(fd, "ERROR vendor field: too short buflen for vendor field: %d\n", len);
+		return -1;
+	}
+
+	fprintf(fd, "+--------------------------------+--------------------------------+\n");
+	fprintf(fd, "|1 Information Emement id: %5d |            Field Length: %5d |\n",
+		ntohs(field->type) & 0x7fff, ntohs(field->length));
+	fprintf(fd, "+--------------------------------+--------------------------------+\n");
+	fprintf(fd, "|               Enterprise Number: %10d                     |\n",
+		ntohl(field->enterprise_num));
+
+	len -= sizeof(*field);
+	if (len == 0)
+		return sizeof(*field);
+
+	ptr = (void *)field + sizeof(*field);
+	if (*(u_int8_t *)ptr & 0x80) /* vendor */
+		ret = ipfix_fprintf_vendor_field(fd, ptr, len);
+	else /* ietf */
+		ret = ipfix_fprintf_ietf_field(fd, ptr, len);
+
+	if (ret == -1)
+		return -1;
+	return ret + sizeof(*field);
+}
+
+static int ipfix_fprintf_data_records(FILE *fd, const void *data, int len)
+{
+	int i;
+
+	fprintf(fd, "+-----------------------------------------------------------------+\n");
+	/* don't say messy...*/
+	for (i = 0; i < len; i += 4) {
+		switch (len - i - 4) {
+		case -3:
+			fprintf(fd, "|          0x%02x                                                   |\n",
+				*(u_int8_t *)(data + i));
+			break;
+		case -2:
+			fprintf(fd, "|          0x%02x          0x%02x                                     |\n",
+				*(u_int8_t *)(data + i), *(u_int8_t *)(data + i + 1));
+			break;
+		case -1:
+			fprintf(fd, "|          0x%02x          0x%02x          0x%02x                       |\n",
+				*(u_int8_t *)(data + i), *(u_int8_t *)(data + i + 1), *(u_int8_t *)(data + i + 2));
+			break;
+		default:
+			fprintf(fd, "|          0x%02x          0x%02x          0x%02x          0x%02x         |\n",
+				*(u_int8_t *)(data + i), *(u_int8_t *)(data + i + 1),
+				*(u_int8_t *)(data + i + 2), *(u_int8_t *)(data + i + 3));
+			break;
+		}
+	}
+	return len;
+}
+
+static int ipfix_fprintf_template_records(FILE *fd, const struct ipfix_templ_rec_hdr *hdr,
+					  int len)
+{
+	int ret;
+	void *field;
+
+	if (len < (int)sizeof(*hdr)) {
+		fprintf(fd, "ERROR template records: too short buflen for template record: %d\n", len);
+		return -1;
+	}
+
+	fprintf(fd, "+--------------------------------+--------------------------------+\n");
+	fprintf(fd, "|             Template ID: %5d |             Field Count: %5d |\n",
+		ntohs(hdr->templ_id), ntohs(hdr->field_count));
+
+	len -= sizeof(*hdr);
+	if (len == 0)
+		return sizeof(*hdr);
+
+	field = (void *)hdr + sizeof(*hdr);
+	if (*(u_int8_t *)field & 0x80)
+		ret = ipfix_fprintf_vendor_field(fd, field, len);
+	else
+		ret = ipfix_fprintf_ietf_field(fd, field, len);
+
+	if (ret == -1)
+		return -1;
+	return ret + sizeof(*hdr);
+}
+
+static int ipfix_fprintf_set_header(FILE *fd, const struct ipfix_set_hdr *hdr, int len)
+{
+	int ret, setlen, total_len;
+	void *ptr;
+
+	if (len < (int)sizeof(*hdr)) {
+		fprintf(fd, "ERROR set header: too short buflen for set header: %d\n", len);
+		return -1;
+	}
+	setlen = ntohs(hdr->length);
+	if (len < setlen) {
+		fprintf(fd, "ERROR set header: buflen: %d is smaller than set length field: %d\n", len, setlen);
+		/* return -1; */
+	}
+	if (setlen < (int)sizeof(*hdr)) {
+		fprintf(fd, "ERROR set header: too short set length field: %d\n", setlen);
+		return -1;
+	}
+
+	fprintf(fd, "+--------------------------------+--------------------------------+\n");
+	fprintf(fd, "|                  Set ID: %5d |                  Length: %5d |\n",
+		ntohs(hdr->set_id), setlen);
+
+	setlen -= sizeof(*hdr);
+	ptr = (void *)hdr + sizeof(*hdr);
+	total_len = sizeof(*hdr);
+
+	switch (ntohs(hdr->set_id)) {
+	case 2:
+		ret = ipfix_fprintf_template_records(fd, ptr, setlen);
+		break;
+	case 3:
+		/* XXX: ret = ipfix_fprintf_options_template_records(fd, ptr, setlen); */
+		fprintf(fd, "ERROR: options template is not implemented yet, sorry");
+		ret = setlen;
+		break;
+	default:
+		ret = ipfix_fprintf_data_records(fd, ptr, setlen);
+		break;
+	}
+
+	if (ret == -1 || ret != setlen)
+		return -1;
+
+	fprintf(fd, "+-----------------------------------------------------------------+\n");
+	return total_len + ret;
+}
+
+static int ipfix_fprintf_header(FILE *fd, const struct ipfix_msg_hdr *hdr)
+{
+	int ret, len;
+	char outstr[20];
+	void *ptr;
+	time_t t = (time_t)(ntohl(hdr->export_time));
+	struct tm *tmp = localtime(&t);
+
+	/* XXX: tmp == NULL and strftime == 0 */
+	strftime(outstr, sizeof(outstr), "%F %T", tmp);
+
+	fprintf(fd, "+--------------------------------+--------------------------------+\n");
+	fprintf(fd, "|          Version Number: %5d |                  Length: %5d |\n",
+		ntohs(hdr->version), ntohs(hdr->length));
+	fprintf(fd, "+--------------------------------+--------------------------------+\n");
+	fprintf(fd, "|                     Exoprt Time: %10d                     |\t%s\n",
+		ntohl(hdr->export_time), outstr);
+	fprintf(fd, "+-----------------------------------------------------------------+\n");
+	fprintf(fd, "|                 Sequence Number: %10d                     |\n",
+		ntohl(hdr->seq));
+	fprintf(fd, "+-----------------------------------------------------------------+\n");
+	fprintf(fd, "|           Observation Domain ID: %10d                     |\n",
+		ntohl(hdr->domain_id));
+	fprintf(fd, "+-----------------------------------------------------------------+\n");
+
+	len = ntohs(hdr->length) - sizeof(*hdr);
+	ptr = (void *)hdr + sizeof(*hdr);
+
+	while (len > 0) {
+		ret = ipfix_fprintf_set_header(fd, ptr, len);
+		if (ret == -1)
+			return -1;
+		len -= ret;
+		ptr += ret;
+	}
+
+	return ntohs(hdr->length) - len;
+}
-- 
1.9.1

[PATCH 08/12] ipfix: build headers with template

[Ken-ichirou MATSUZAWA]

Based on assumption that we send only two types packet, one has template
set and data set, another has data set only. Theses templates including
message header, set header and template records are put in struct
ulogd_ipfix_template as struct ipfix_msg_hdr.

Another assumption is that we can send only one dataset, cannot send
muliple datasets at onece. It's better to not include dataset buffer,
alloc it each time and use gather sendmsg to send multiple set with pre-
allocated headers.

Signed-off-by Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp>
---
 output/ulogd_output_IPFIX.c | 208 +++++++++++++++++++++++++++-----------------
 1 file changed, 129 insertions(+), 79 deletions(-)

diff --git a/output/ulogd_output_IPFIX.c b/output/ulogd_output_IPFIX.c
index 0ff2efe..f10a82b 100644
--- a/output/ulogd_output_IPFIX.c
+++ b/output/ulogd_output_IPFIX.c
@@ -126,18 +126,13 @@ static struct config_keyset ipfix_kset = {
 #define domain_ce(x)	(x->ces[3])
 #define template_per_ce(x)	(x->ces[4])
 
-struct ipfix_template {
-	struct ipfix_templ_rec_hdr hdr;
-	char buf[0];
-};
-
 struct ulogd_ipfix_template {
 	struct llist_head list;
 	struct nfct_bitmask *bitmask;
-	unsigned int data_length;	/* length of the DATA */
-	void *tmpl_cur;			/* cursor into current template position */
-	struct ipfix_template tmpl;
-	int until_template;			/* decide if it's time to retransmit our template */
+	int until_template;		/* decide if it's time to retransmit our template */
+	int tmpl_data_msg_len, data_only_msg_len;
+	struct ipfix_msg_hdr *tmpl_data_msg;	/* include records, set header of template, data */
+	struct ipfix_msg_hdr *data_only_msg;	/* include records, set header of data */
 };
 
 struct ipfix_instance {
@@ -155,36 +150,87 @@ static u_int16_t next_template_id = ULOGD_IPFIX_TEMPL_BASE;
 
 static int ipfix_fprintf_header(FILE *fd, const struct ipfix_msg_hdr *hdr);
 
-/* Build the IPFIX template from the input keys */
 struct ulogd_ipfix_template *
-build_template_for_bitmask(struct ulogd_pluginstance *upi,
+alloc_ulogd_ipfix_template(struct ulogd_pluginstance *upi,
 			   struct nfct_bitmask *bm)
 {
 	struct ulogd_ipfix_template *tmpl;
-	unsigned int i, j;
-	int size = sizeof(struct ulogd_ipfix_template)
-		   + (upi->input.num_keys * sizeof(struct ipfix_vendor_field));
+	unsigned int i;
+	int tmpl_len = 0, data_len = 0;
 
-	tmpl = malloc(size);
-	if (!tmpl)
+	for (i = 0; i < upi->input.num_keys; i++) {
+		struct ulogd_key *key = &upi->input.keys[i];
+		int length = ulogd_key_size(key);
+
+		if (!nfct_bitmask_test_bit(bm, i))
+			continue;
+
+		if (key->ipfix.vendor == IPFIX_VENDOR_IETF)
+			tmpl_len += sizeof(struct ipfix_ietf_field);
+		else
+			tmpl_len += sizeof(struct ipfix_vendor_field);
+
+		data_len += length;
+	}
+
+	tmpl = calloc(sizeof(struct ulogd_ipfix_template), 1);
+	if (tmpl == NULL)
 		return NULL;
-	memset(tmpl, 0, size);
 
 	tmpl->bitmask = nfct_bitmask_clone(bm);
-	if (!tmpl->bitmask) {
-		free(tmpl);
-		return NULL;
-	}
+	if (!tmpl->bitmask)
+		goto free_tmpl;
+
+	tmpl->data_only_msg_len = sizeof(struct ipfix_msg_hdr)
+		+ sizeof(struct ipfix_set_hdr) + data_len;
+	tmpl->tmpl_data_msg_len = tmpl->data_only_msg_len
+		+ sizeof(struct ipfix_templ_rec_hdr)
+		+ sizeof(struct ipfix_set_hdr) + tmpl_len;
 
-	/* initialize template header */
-	tmpl->tmpl.hdr.templ_id = htons(next_template_id++);
+	tmpl->tmpl_data_msg = malloc(tmpl->tmpl_data_msg_len);
+	if (tmpl->tmpl_data_msg == NULL)
+		goto free_bitmask;
+	memset(tmpl->tmpl_data_msg, 0, tmpl->tmpl_data_msg_len);
 
-	tmpl->tmpl_cur = tmpl->tmpl.buf;
+	tmpl->data_only_msg = malloc(tmpl->data_only_msg_len);
+	if (tmpl->data_only_msg == NULL)
+		goto free_tmpl_data_msg;
+	memset(tmpl->data_only_msg, 0, tmpl->data_only_msg_len);
+
+	return tmpl;
+
+free_tmpl_data_msg:
+	free(tmpl->tmpl_data_msg);
+free_bitmask:
+	free(tmpl->bitmask);
+free_tmpl:
+	free(tmpl);
+
+	return NULL;
+}
+
+/* Build the IPFIX template from the input keys */
+struct ulogd_ipfix_template *
+build_template_for_bitmask(struct ulogd_pluginstance *upi,
+			   struct nfct_bitmask *bm)
+{
+	struct ulogd_ipfix_template *tmpl;
+	struct ipfix_msg_hdr *msg_hdr;
+	struct ipfix_templ_rec_hdr *tmpl_hdr;
+	struct ipfix_set_hdr *set_hdr;
+	unsigned int i, field_count;
+	void *ptr;
+
+	tmpl = alloc_ulogd_ipfix_template(upi, bm);
+	if (tmpl == NULL)
+		return NULL;
 
-	tmpl->data_length = 0;
 	tmpl->until_template = template_per_ce(upi->config_kset).u.value;
 
-	for (i = 0, j = 0; i < upi->input.num_keys; i++) {
+	/* build template records */
+	ptr = (void *)tmpl->tmpl_data_msg + sizeof(struct ipfix_msg_hdr)
+		+ sizeof(struct ipfix_set_hdr) + sizeof(struct ipfix_templ_rec_hdr);
+	for (i = 0, field_count = 0; i < upi->input.num_keys; i++) {
 		struct ulogd_key *key = &upi->input.keys[i];
 		int length = ulogd_key_size(key);
 
@@ -192,26 +238,53 @@ build_template_for_bitmask(struct ulogd_pluginstance *upi,
 			continue;
 
 		if (key->ipfix.vendor == IPFIX_VENDOR_IETF) {
-			struct ipfix_ietf_field *field = 
-				(struct ipfix_ietf_field *) tmpl->tmpl_cur;
+			struct ipfix_ietf_field *field = (struct ipfix_ietf_field *)ptr;
 
 			field->type = htons(key->ipfix.field_id);
 			field->length = htons(length);
-			tmpl->tmpl_cur += sizeof(*field);
+			ptr += sizeof(*field);
 		} else {
-			struct ipfix_vendor_field *field =
-				(struct ipfix_vendor_field *) tmpl->tmpl_cur;
+			struct ipfix_vendor_field *field =(struct ipfix_vendor_field *)ptr;
 
 			field->type = htons(key->ipfix.field_id | 0x8000);
-			field->enterprise_num = htonl(key->ipfix.vendor);
 			field->length = htons(length);
-			tmpl->tmpl_cur += sizeof(*field);
+			field->enterprise_num = htonl(key->ipfix.vendor);
+			ptr += sizeof(*field);
 		}
-		tmpl->data_length += length;
-		j++;
+		field_count++;
 	}
 
-	tmpl->tmpl.hdr.field_count = htons(j);
+	/** initialize ipfix message header with template and data */
+	msg_hdr = tmpl->tmpl_data_msg;
+	msg_hdr->version = htons(10);
+	msg_hdr->length = htons(tmpl->tmpl_data_msg_len);
+	msg_hdr->domain_id = htonl(domain_ce(upi->config_kset).u.value);
+
+	/* initialize template set header */
+	set_hdr = (void *)msg_hdr + sizeof(*msg_hdr);
+	set_hdr->set_id = htons(2);
+	set_hdr->length = htons(tmpl->tmpl_data_msg_len - tmpl->data_only_msg_len);
+
+	/* initialize template record header */
+	tmpl_hdr = (void *)set_hdr + sizeof(*set_hdr);
+	tmpl_hdr->templ_id = htons(next_template_id++);
+	tmpl_hdr->field_count = htons(field_count);
+
+	/* initialize data set header */
+	set_hdr = ptr;
+	set_hdr->set_id = tmpl_hdr->templ_id;
+	set_hdr->length = htons(tmpl->data_only_msg_len - sizeof(struct ipfix_msg_hdr));
+
+	/** initialize ipfix message header with data only */
+	msg_hdr = tmpl->data_only_msg;
+	msg_hdr->version = htons(10);
+	msg_hdr->length = htons(tmpl->data_only_msg_len);
+	msg_hdr->domain_id = htonl(domain_ce(upi->config_kset).u.value);
+
+	/* initialize data set header */
+	set_hdr = (void *)msg_hdr + sizeof(*msg_hdr);
+	set_hdr->set_id = tmpl_hdr->templ_id;
+	set_hdr->length = htons(tmpl->data_only_msg_len - sizeof(struct ipfix_msg_hdr));
 
 	return tmpl;
 }
@@ -308,59 +381,36 @@ static struct ipfix_msg_hdr *build_ipfix_msg(struct ulogd_pluginstance *upi,
 					     bool need_template)
 {
 	struct ipfix_instance *ii = (struct ipfix_instance *) &upi->private;
-	u_int16_t tmpl_len;
 	struct ipfix_msg_hdr *msg_hdr;
-	struct ipfix_templ_rec_hdr *tmpl_hdr;
-	struct ipfix_set_hdr *data_hdr, *tmpl_set_hdr;
-	void *buf;
-	int msglen, ret;
-
-	msglen = sizeof(struct ipfix_msg_hdr) + sizeof(struct ipfix_set_hdr)
-		+ template->data_length;
-	if (need_template)
-		msglen = msglen + sizeof(struct ipfix_set_hdr)
-			+ (template->tmpl_cur - (void *)&template->tmpl);
-	buf = malloc(msglen);
-	if (buf == NULL)
-		return NULL;
-	memset(buf, 0, msglen);
+	void *data_records;
+	int ret, data_len;
 
-	/* ipfix msg header */
-	msg_hdr = buf;
-	msg_hdr->version = htons(10);
-	msg_hdr->length = htons(msglen);
-	msg_hdr->seq = htonl(ii->seq++);
-	msg_hdr->domain_id = htonl(domain_ce(upi->config_kset).u.value);
 	if (need_template) {
-		/* put set header and template records */
-		tmpl_set_hdr = buf + sizeof(*msg_hdr);
-		tmpl_set_hdr->set_id = htons(2);
-		tmpl_len = template->tmpl_cur - (void *)&template->tmpl;
-		tmpl_set_hdr->length = htons(sizeof(*tmpl_set_hdr) + tmpl_len);
-		tmpl_hdr = (void *)tmpl_set_hdr + sizeof(*tmpl_set_hdr);
-		memcpy((void *)tmpl_hdr, (void *)&template->tmpl, tmpl_len);
-		data_hdr = (void *)tmpl_hdr + tmpl_len;
+		int tmpl_len = template->tmpl_data_msg_len - template->data_only_msg_len;
+		msg_hdr = template->tmpl_data_msg;
+		data_records = (void *)msg_hdr + sizeof(struct ipfix_msg_hdr)
+			+ tmpl_len + sizeof(struct ipfix_set_hdr);
 	} else {
-		data_hdr = buf + sizeof(*msg_hdr);
+		msg_hdr = template->data_only_msg;
+		data_records = (void *)msg_hdr + sizeof(struct ipfix_msg_hdr)
+			+ sizeof(struct ipfix_set_hdr);
 	}
+	msg_hdr->seq = htonl(ii->seq++);
 
-	/* put set header and data records */
-	data_hdr->set_id = template->tmpl.hdr.templ_id; /* already ordered */
-	data_hdr->length = htons(sizeof(*data_hdr) + template->data_length);
-	ret = put_data_records(upi, template, (void *)data_hdr + sizeof(*data_hdr));
+	data_len = template->data_only_msg_len - sizeof(struct ipfix_msg_hdr)
+		- sizeof(struct ipfix_set_hdr);
+	memset(data_records, 0, data_len);
+
+	ret = put_data_records(upi, template, data_records);
 	if (ret < 0) {
 		ulogd_log(ULOGD_ERROR, "could not build ipfix dataset");
-		goto free_buf;
-	} else if (ret > msglen) {
+		return NULL;
+	} else if (ret > data_len) {
 		ulogd_log(ULOGD_ERROR, "overflowed on building ipfix dataset");
-		goto free_buf;
+		return NULL;
 	}
 
 	return msg_hdr;
-
-free_buf:
-	free(buf);
-	return NULL;
 }
 
 static int output_ipfix(struct ulogd_pluginstance *upi)
@@ -405,10 +455,11 @@ static int output_ipfix(struct ulogd_pluginstance *upi)
 		need_template = true;
 	}
 	
-	if (template->until_template-- == 0) {
+	if (template->until_template == 0) {
 		need_template = true;
 		template->until_template = template_per_ce(upi->config_kset).u.value;
 	}
+	template->until_template--;
 
 	ipfix_msg = build_ipfix_msg(upi, template, need_template);
 	if (ipfix_msg == NULL)
@@ -418,7 +469,6 @@ static int output_ipfix(struct ulogd_pluginstance *upi)
 	ipfix_fprintf_header(stdout, ipfix_msg);
 	fprintf(stdout, "\n");
 
-	free(ipfix_msg);
 	return ULOGD_IRET_OK;
 }
 
-- 
1.9.1

[PATCH v3 ulogd 09/12] nfct: fix ipfix field_id of flow.end.usec

[Ken-ichirou MATSUZAWA]

Signed-off-by Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp>
---
 input/flow/ulogd_inpflow_NFCT.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/input/flow/ulogd_inpflow_NFCT.c b/input/flow/ulogd_inpflow_NFCT.c
index 899b7e3..0b64a45 100644
--- a/input/flow/ulogd_inpflow_NFCT.c
+++ b/input/flow/ulogd_inpflow_NFCT.c
@@ -399,7 +399,7 @@ static struct ulogd_key nfct_okeys[] = {
 		.name	= "flow.end.usec",
 		.ipfix	= {
 			.vendor		= IPFIX_VENDOR_IETF,
-			.field_id	= IPFIX_flowEndSeconds,
+			.field_id	= IPFIX_flowEndMicroSeconds,
 		},
 	},
 	{
-- 
1.9.1

[PATCH v3 ulogd 10/12] nfct: fix icmp type and code output key size

[Ken-ichirou MATSUZAWA]

Signed-off-by Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp>
---
 input/flow/ulogd_inpflow_NFCT.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/input/flow/ulogd_inpflow_NFCT.c b/input/flow/ulogd_inpflow_NFCT.c
index 0b64a45..81f5bc3 100644
--- a/input/flow/ulogd_inpflow_NFCT.c
+++ b/input/flow/ulogd_inpflow_NFCT.c
@@ -542,10 +542,10 @@ static int propagate_ct(struct ulogd_pluginstance *main_upi,
 			     htons(nfct_get_attr_u16(ct, ATTR_ORIG_PORT_DST)));
 		break;
 	case IPPROTO_ICMP:
-		okey_set_u16(&ret[NFCT_ICMP_CODE],
-			     nfct_get_attr_u8(ct, ATTR_ICMP_CODE));
-		okey_set_u16(&ret[NFCT_ICMP_TYPE],
-			     nfct_get_attr_u8(ct, ATTR_ICMP_TYPE));
+		okey_set_u8(&ret[NFCT_ICMP_CODE],
+			    nfct_get_attr_u8(ct, ATTR_ICMP_CODE));
+		okey_set_u8(&ret[NFCT_ICMP_TYPE],
+			    nfct_get_attr_u8(ct, ATTR_ICMP_TYPE));
 		break;
 	}
 
-- 
1.9.1

[PATCH v3 ulogd 11/12] nfct/ipfix: introduce new vendor id

[Ken-ichirou MATSUZAWA]

IPFIX_VENDOR_REVERSE, defined in RFC 5103 6.1 Reverse Information Element
Private Enterprise Number. And use it at counter in nfct.

Signed-off-by: Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp>
---
 include/ulogd/ipfix_protocol.h  | 3 +++
 input/flow/ulogd_inpflow_NFCT.c | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/include/ulogd/ipfix_protocol.h b/include/ulogd/ipfix_protocol.h
index 266897e..86a0e84 100644
--- a/include/ulogd/ipfix_protocol.h
+++ b/include/ulogd/ipfix_protocol.h
@@ -9,6 +9,9 @@
 /* defined in http://www.iana.org/assignments/enterprise-numbers */
 #define IPFIX_VENDOR_NETFILTER	21373	/* FIXME: htonl? */
 
+/* defined in RFC 5103 IPFIX Biflow Export */
+#define IPFIX_VENDOR_REVERSE	29305
+
 /* Section 3.1 */
 struct ipfix_msg_hdr {
 	u_int16_t	version;
diff --git a/input/flow/ulogd_inpflow_NFCT.c b/input/flow/ulogd_inpflow_NFCT.c
index 81f5bc3..5912011 100644
--- a/input/flow/ulogd_inpflow_NFCT.c
+++ b/input/flow/ulogd_inpflow_NFCT.c
@@ -309,7 +309,7 @@ static struct ulogd_key nfct_okeys[] = {
 		.flags	= ULOGD_RETF_NONE,
 		.name	= "reply.raw.pktlen",
 		.ipfix	= {
-			.vendor 	= IPFIX_VENDOR_IETF,
+			.vendor 	= IPFIX_VENDOR_REVERSE,
 			.field_id 	= IPFIX_octetTotalCount,
 			/* FIXME: this could also be octetDeltaCount */
 		},
@@ -319,7 +319,7 @@ static struct ulogd_key nfct_okeys[] = {
 		.flags	= ULOGD_RETF_NONE,
 		.name	= "reply.raw.pktcount",
 		.ipfix	= {
-			.vendor 	= IPFIX_VENDOR_IETF,
+			.vendor 	= IPFIX_VENDOR_REVERSE,
 			.field_id 	= IPFIX_packetTotalCount,
 			/* FIXME: this could also be packetDeltaCount */
 		},
-- 
1.9.1

[PATCH v3 ulogd 12/12] ipfix: add debug symbol for yafscii

[Ken-ichirou MATSUZAWA]

http://tools.netsa.cert.org/yaf/yafscii.html
To see the file by yafscii
yafscii --in /tmp/ulogd.yaf and see /tmp/ulogd.yaf.txt

Signed-off-by: Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp>
---
 output/ulogd_output_IPFIX.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/output/ulogd_output_IPFIX.c b/output/ulogd_output_IPFIX.c
index f10a82b..e552a3c 100644
--- a/output/ulogd_output_IPFIX.c
+++ b/output/ulogd_output_IPFIX.c
@@ -38,6 +38,12 @@
 #include <sys/socket.h>
 #include <netdb.h>
 
+#define DEBUG_YAFSCII_FILE "/tmp/ulogd.yaf"
+#ifdef DEBUG_YAFSCII_FILE
+#include <sys/stat.h>
+#include <sys/fcntl.h>
+#endif
+
 #include <libnetfilter_conntrack/libnetfilter_conntrack.h>
 
 #include <ulogd/linuxlist.h>
@@ -143,6 +149,10 @@ struct ipfix_instance {
 	struct llist_head template_list;
 	struct nfct_bitmask *valid_bitmask;	/* bitmask of valid keys */
 	u_int32_t seq;
+
+#ifdef DEBUG_YAFSCII_FILE
+	int yaf_fd;
+#endif
 };
 
 #define ULOGD_IPFIX_TEMPL_BASE 1024
@@ -469,6 +479,9 @@ static int output_ipfix(struct ulogd_pluginstance *upi)
 	ipfix_fprintf_header(stdout, ipfix_msg);
 	fprintf(stdout, "\n");
 
+#ifdef DEBUG_YAFSCII_FILE
+	write(ii->yaf_fd, ipfix_msg, ntohs(ipfix_msg->length));
+#endif
 	return ULOGD_IRET_OK;
 }
 
@@ -565,6 +578,10 @@ static int start_ipfix(struct ulogd_pluginstance *pi)
 	if (ret < 0)
 		goto out_bm_free;
 
+#ifdef DEBUG_YAFSCII_FILE
+	ii->yaf_fd = open(DEBUG_YAFSCII_FILE, O_CREAT|O_WRONLY|O_TRUNC, S_IWUSR);
+#endif
+
 	return 0;
 
 out_bm_free:
@@ -578,6 +595,9 @@ static int stop_ipfix(struct ulogd_pluginstance *pi)
 {
 	struct ipfix_instance *ii = (struct ipfix_instance *) &pi->private;
 
+#ifdef DEBUG_YAFSCII_FILE
+	close(ii->yaf_fd);
+#endif
 	close(ii->fd);
 
 	nfct_bitmask_destroy(ii->valid_bitmask);
-- 
1.9.1