[PATCH v3 2/6] mbuf_offload: library to support attaching offloads to a mbuf

[Declan Doherty <declan.doherty@intel.com>]

This library add support for adding a chain of offload operations to a
mbuf. It contains the definition of the rte_mbuf_offload structure as
well as helper functions for attaching  offloads to mbufs and a mempool
management functions.

This initial implementation supports attaching multiple offload
operations to a single mbuf, but only a single offload operation of a
specific type can be attach to that mbuf.

Signed-off-by: Declan Doherty <declan.doherty@intel.com>
---
 config/common_bsdapp                               |   6 +
 config/common_linuxapp                             |   6 +
 lib/Makefile                                       |   1 +
 lib/librte_mbuf/rte_mbuf.h                         |   6 +
 lib/librte_mbuf_offload/Makefile                   |  52 ++++
 lib/librte_mbuf_offload/rte_mbuf_offload.c         | 100 +++++++
 lib/librte_mbuf_offload/rte_mbuf_offload.h         | 289 +++++++++++++++++++++
 .../rte_mbuf_offload_version.map                   |   7 +
 mk/rte.app.mk                                      |   1 +
 9 files changed, 468 insertions(+)
 create mode 100644 lib/librte_mbuf_offload/Makefile
 create mode 100644 lib/librte_mbuf_offload/rte_mbuf_offload.c
 create mode 100644 lib/librte_mbuf_offload/rte_mbuf_offload.h
 create mode 100644 lib/librte_mbuf_offload/rte_mbuf_offload_version.map

diff --git a/config/common_bsdapp b/config/common_bsdapp
index 8ce6af5..96d9d26 100644
--- a/config/common_bsdapp
+++ b/config/common_bsdapp
@@ -320,6 +320,12 @@ CONFIG_RTE_MBUF_REFCNT_ATOMIC=y
 CONFIG_RTE_PKTMBUF_HEADROOM=128
 
 #
+# Compile librte_mbuf_offload
+#
+CONFIG_RTE_LIBRTE_MBUF_OFFLOAD=y
+CONFIG_RTE_LIBRTE_MBUF_OFFLOAD_DEBUG=n
+
+#
 # Compile librte_timer
 #
 CONFIG_RTE_LIBRTE_TIMER=y
diff --git a/config/common_linuxapp b/config/common_linuxapp
index e7b9b25..c113c88 100644
--- a/config/common_linuxapp
+++ b/config/common_linuxapp
@@ -328,6 +328,12 @@ CONFIG_RTE_MBUF_REFCNT_ATOMIC=y
 CONFIG_RTE_PKTMBUF_HEADROOM=128
 
 #
+# Compile librte_mbuf_offload
+#
+CONFIG_RTE_LIBRTE_MBUF_OFFLOAD=y
+CONFIG_RTE_LIBRTE_MBUF_OFFLOAD_DEBUG=n
+
+#
 # Compile librte_timer
 #
 CONFIG_RTE_LIBRTE_TIMER=y
diff --git a/lib/Makefile b/lib/Makefile
index 4c5c1b4..ef172ea 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -36,6 +36,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_EAL) += librte_eal
 DIRS-$(CONFIG_RTE_LIBRTE_RING) += librte_ring
 DIRS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += librte_mempool
 DIRS-$(CONFIG_RTE_LIBRTE_MBUF) += librte_mbuf
+DIRS-$(CONFIG_RTE_LIBRTE_MBUF_OFFLOAD) += librte_mbuf_offload
 DIRS-$(CONFIG_RTE_LIBRTE_TIMER) += librte_timer
 DIRS-$(CONFIG_RTE_LIBRTE_CFGFILE) += librte_cfgfile
 DIRS-$(CONFIG_RTE_LIBRTE_CMDLINE) += librte_cmdline
diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h
index 689ef77..6b5c0c2 100644
--- a/lib/librte_mbuf/rte_mbuf.h
+++ b/lib/librte_mbuf/rte_mbuf.h
@@ -728,6 +728,9 @@ typedef uint8_t  MARKER8[0];  /**< generic marker with 1B alignment */
 typedef uint64_t MARKER64[0]; /**< marker that allows us to overwrite 8 bytes
                                * with a single assignment */
 
+/** Opaque rte_mbuf_offload  structure declarations */
+struct rte_mbuf_offload;
+
 /**
  * The generic rte_mbuf, containing a packet mbuf.
  */
@@ -841,6 +844,9 @@ struct rte_mbuf {
 
 	/** Timesync flags for use with IEEE1588. */
 	uint16_t timesync;
+
+	/* Chain of off-load operations to perform on mbuf */
+	struct rte_mbuf_offload *offload_ops;
 } __rte_cache_aligned;
 
 static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);
diff --git a/lib/librte_mbuf_offload/Makefile b/lib/librte_mbuf_offload/Makefile
new file mode 100644
index 0000000..acdb449
--- /dev/null
+++ b/lib/librte_mbuf_offload/Makefile
@@ -0,0 +1,52 @@
+#   BSD LICENSE
+#
+#   Copyright(c) 2015 Intel Corporation. All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Intel Corporation nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_mbuf_offload.a
+
+CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3
+
+EXPORT_MAP := rte_mbuf_offload_version.map
+
+LIBABIVER := 1
+
+# all source are stored in SRCS-y
+SRCS-$(CONFIG_RTE_LIBRTE_MBUF_OFFLOAD) := rte_mbuf_offload.c
+
+# install includes
+SYMLINK-$(CONFIG_RTE_LIBRTE_MBUF_OFFLOAD)-include := rte_mbuf_offload.h
+
+# this lib needs eal
+DEPDIRS-$(CONFIG_RTE_LIBRTE_MBUF_OFFLOAD) += lib/librte_mbuf
+DEPDIRS-$(CONFIG_RTE_LIBRTE_MBUF_OFFLOAD) += lib/librte_cryptodev
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/lib/librte_mbuf_offload/rte_mbuf_offload.c b/lib/librte_mbuf_offload/rte_mbuf_offload.c
new file mode 100644
index 0000000..5c0c9dd
--- /dev/null
+++ b/lib/librte_mbuf_offload/rte_mbuf_offload.c
@@ -0,0 +1,100 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <string.h>
+#include <rte_common.h>
+
+#include "rte_mbuf_offload.h"
+
+/** Initialize rte_mbuf_offload structure */
+static void
+rte_pktmbuf_offload_init(struct rte_mempool *mp,
+		__rte_unused void *opaque_arg,
+		void *_op_data,
+		__rte_unused unsigned i)
+{
+	struct rte_mbuf_offload *ol = _op_data;
+
+	memset(_op_data, 0, mp->elt_size);
+
+	ol->type = RTE_PKTMBUF_OL_NOT_SPECIFIED;
+	ol->mp = mp;
+}
+
+
+struct rte_mempool *
+rte_pktmbuf_offload_pool_create(const char *name, unsigned size,
+		unsigned cache_size, uint16_t priv_size, int socket_id)
+{
+	struct rte_pktmbuf_offload_pool_private *priv;
+	unsigned elt_size = sizeof(struct rte_mbuf_offload) + priv_size;
+
+
+	/* lookup mempool in case already allocated */
+	struct rte_mempool *mp = rte_mempool_lookup(name);
+
+	if (mp != NULL) {
+		priv = (struct rte_pktmbuf_offload_pool_private *)
+				rte_mempool_get_priv(mp);
+
+		if (priv->offload_priv_size <  priv_size ||
+				mp->elt_size != elt_size ||
+				mp->cache_size < cache_size ||
+				mp->size < size) {
+			mp = NULL;
+			return NULL;
+		}
+		return mp;
+	}
+
+	mp = rte_mempool_create(
+			name,
+			size,
+			elt_size,
+			cache_size,
+			sizeof(struct rte_pktmbuf_offload_pool_private),
+			NULL,
+			NULL,
+			rte_pktmbuf_offload_init,
+			NULL,
+			socket_id,
+			0);
+
+	if (mp == NULL)
+		return NULL;
+
+	priv = (struct rte_pktmbuf_offload_pool_private *)
+			rte_mempool_get_priv(mp);
+
+	priv->offload_priv_size = priv_size;
+	return mp;
+}
diff --git a/lib/librte_mbuf_offload/rte_mbuf_offload.h b/lib/librte_mbuf_offload/rte_mbuf_offload.h
new file mode 100644
index 0000000..0a59667
--- /dev/null
+++ b/lib/librte_mbuf_offload/rte_mbuf_offload.h
@@ -0,0 +1,289 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ *   Copyright 2014 6WIND S.A.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_MBUF_OFFLOAD_H_
+#define _RTE_MBUF_OFFLOAD_H_
+
+#include <rte_mbuf.h>
+#include <rte_crypto.h>
+
+
+/** packet mbuf offload operation types */
+enum rte_mbuf_ol_op_type {
+	RTE_PKTMBUF_OL_NOT_SPECIFIED = 0,
+	/**< Off-load not specified */
+	RTE_PKTMBUF_OL_CRYPTO
+	/**< Crypto offload operation */
+};
+
+/**
+ * Generic packet mbuf offload
+ * This is used to specify a offload operation to be performed on a rte_mbuf.
+ * Multiple offload operations can be chained to the same mbuf, but only a
+ * single offload operation of a particular type can be in the chain */
+struct rte_mbuf_offload {
+	struct rte_mbuf_offload *next;	/**< next offload in chain */
+	struct rte_mbuf *m;		/**< mbuf offload is attached to */
+	struct rte_mempool *mp;		/**< mempool offload allocated from */
+
+	enum rte_mbuf_ol_op_type type;	/**< offload type */
+	union {
+		struct rte_crypto_op crypto;	/**< Crypto operation */
+	} op;
+};
+
+/**< private data structure belonging to packet mbug offload mempool */
+struct rte_pktmbuf_offload_pool_private {
+	uint16_t offload_priv_size;
+	/**< Size of private area in each mbuf_offload. */
+};
+
+
+/**
+ * Creates a mempool of rte_mbuf_offload objects
+ *
+ * @param	name		mempool name
+ * @param	size		number of objects in mempool
+ * @param	cache_size	cache size of objects for each core
+ * @param	priv_size	size of private data to be allocated with each
+ *				rte_mbuf_offload object
+ * @param	socket_id	Socket on which to allocate mempool objects
+ *
+ * @return
+ * - On success returns a valid mempool of rte_mbuf_offload objects
+ * - On failure return NULL
+ */
+extern struct rte_mempool *
+rte_pktmbuf_offload_pool_create(const char *name, unsigned size,
+		unsigned cache_size, uint16_t priv_size, int socket_id);
+
+
+/**
+ * Returns private data size allocated with each rte_mbuf_offload object by
+ * the mempool
+ *
+ * @param	mpool	rte_mbuf_offload mempool
+ *
+ * @return	private data size
+ */
+static inline uint16_t
+__rte_pktmbuf_offload_priv_size(struct rte_mempool *mpool)
+{
+	struct rte_pktmbuf_offload_pool_private *priv =
+			rte_mempool_get_priv(mpool);
+
+	return priv->offload_priv_size;
+}
+
+/**
+ * Get specified off-load operation type from mbuf.
+ *
+ * @param	m		packet mbuf.
+ * @param	type		offload operation type requested.
+ *
+ * @return
+ * - On success retruns rte_mbuf_offload pointer
+ * - On failure returns NULL
+ *
+ */
+static inline struct rte_mbuf_offload *
+rte_pktmbuf_offload_get(struct rte_mbuf *m, enum rte_mbuf_ol_op_type type)
+{
+	struct rte_mbuf_offload *ol = m->offload_ops;
+
+	if (m->offload_ops != NULL && m->offload_ops->type == type)
+		return ol;
+
+	ol = m->offload_ops;
+	while (ol != NULL) {
+		if (ol->type == type)
+			return ol;
+
+		ol = ol->next;
+	}
+
+	return ol;
+}
+
+/**
+ * Attach a rte_mbuf_offload to a mbuf. We only support a single offload of any
+ * one type in our chain of offloads.
+ *
+ * @param	m	packet mbuf.
+ * @param	ol	rte_mbuf_offload strucutre to be attached
+ *
+ * @returns
+ * - On success returns the pointer to the offload we just added
+ * - On failure returns NULL
+ */
+static inline struct rte_mbuf_offload *
+rte_pktmbuf_offload_attach(struct rte_mbuf *m, struct rte_mbuf_offload *ol)
+{
+	struct rte_mbuf_offload **ol_last;
+
+	for (ol_last = &m->offload_ops;	ol_last[0] != NULL;
+			ol_last = &ol_last[0]->next)
+		if (ol_last[0]->type == ol->type)
+			return NULL;
+
+	ol_last[0] = ol;
+	ol_last[0]->m = m;
+	ol_last[0]->next = NULL;
+
+	return ol_last[0];
+}
+
+
+/** Rearms rte_mbuf_offload default parameters */
+static inline void
+__rte_pktmbuf_offload_reset(struct rte_mbuf_offload *ol,
+		enum rte_mbuf_ol_op_type type)
+{
+	ol->m = NULL;
+	ol->type = type;
+
+	switch (type) {
+	case RTE_PKTMBUF_OL_CRYPTO:
+		__rte_crypto_op_reset(&ol->op.crypto); break;
+	default:
+		break;
+	}
+}
+
+/** Allocate rte_mbuf_offload from mempool */
+static inline struct rte_mbuf_offload *
+__rte_pktmbuf_offload_raw_alloc(struct rte_mempool *mp)
+{
+	void *buf = NULL;
+
+	if (rte_mempool_get(mp, &buf) < 0)
+		return NULL;
+
+	return (struct rte_mbuf_offload *)buf;
+}
+
+/**
+ * Allocate a rte_mbuf_offload with a specified operation type from
+ * rte_mbuf_offload mempool
+ *
+ * @param	mpool		rte_mbuf_offload mempool
+ * @param	type		offload operation type
+ *
+ * @returns
+ * - On success returns a valid rte_mbuf_offload structure
+ * - On failure returns NULL
+ */
+static inline struct rte_mbuf_offload *
+rte_pktmbuf_offload_alloc(struct rte_mempool *mpool,
+		enum rte_mbuf_ol_op_type type)
+{
+	struct rte_mbuf_offload *ol = __rte_pktmbuf_offload_raw_alloc(mpool);
+
+	if (ol != NULL)
+		__rte_pktmbuf_offload_reset(ol, type);
+
+	return ol;
+}
+
+/**
+ * free rte_mbuf_offload structure
+ */
+static inline void
+rte_pktmbuf_offload_free(struct rte_mbuf_offload *ol)
+{
+	if (ol->mp != NULL)
+		rte_mempool_put(ol->mp, ol);
+}
+
+/**
+ * Checks if the private data of a rte_mbuf_offload has enough capacity for
+ * requested size
+ *
+ * @returns
+ * - if sufficient space available returns pointer to start of private data
+ * - if insufficient space returns NULL
+ */
+static inline void *
+__rte_pktmbuf_offload_check_priv_data_size(struct rte_mbuf_offload *ol,
+		uint16_t size)
+{
+	uint16_t priv_size;
+
+	if (likely(ol->mp != NULL)) {
+		priv_size = __rte_pktmbuf_offload_priv_size(ol->mp);
+
+		if (likely(priv_size >= size))
+			return (void *)(ol + 1);
+	}
+	return NULL;
+}
+
+/**
+ * Allocate space for crypto xforms in the private data space of the
+ * rte_mbuf_offload. This also defaults the crypto xform type and configures
+ * the chaining of the xform in the crypto operation
+ *
+ * @return
+ * - On success returns pointer to first crypto xform in crypto operations chain
+ * - On failure returns NULL */
+static inline struct rte_crypto_xform *
+rte_pktmbuf_offload_alloc_crypto_xforms(struct rte_mbuf_offload *ol,
+		unsigned nb_xforms)
+{
+	struct rte_crypto_xform *xform;
+	void *priv_data;
+	uint16_t size;
+
+	size = sizeof(struct rte_crypto_xform) * nb_xforms;
+	priv_data = __rte_pktmbuf_offload_check_priv_data_size(ol, size);
+
+	if (priv_data == NULL)
+		return NULL;
+
+	ol->op.crypto.xform = xform = (struct rte_crypto_xform *)priv_data;
+
+	do {
+		xform->type = RTE_CRYPTO_XFORM_NOT_SPECIFIED;
+		xform = xform->next = --nb_xforms > 0 ? xform + 1 : NULL;
+	} while (xform);
+
+	return ol->op.crypto.xform;
+}
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_MBUF_OFFLOAD_H_ */
diff --git a/lib/librte_mbuf_offload/rte_mbuf_offload_version.map b/lib/librte_mbuf_offload/rte_mbuf_offload_version.map
new file mode 100644
index 0000000..3d3b06a
--- /dev/null
+++ b/lib/librte_mbuf_offload/rte_mbuf_offload_version.map
@@ -0,0 +1,7 @@
+DPDK_2.2 {
+	global:
+
+	rte_pktmbuf_offload_pool_create;
+
+	local: *;
+};
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 80f68bb..9b4aed3 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -112,6 +112,7 @@ ifeq ($(CONFIG_RTE_BUILD_COMBINE_LIBS),n)
 
 _LDLIBS-$(CONFIG_RTE_LIBRTE_KVARGS)         += -lrte_kvargs
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MBUF)           += -lrte_mbuf
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MBUF_OFFLOAD)   += -lrte_mbuf_offload
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IP_FRAG)        += -lrte_ip_frag
 _LDLIBS-$(CONFIG_RTE_LIBRTE_ETHER)          += -lethdev
 _LDLIBS-$(CONFIG_RTE_LIBRTE_CRYPTODEV)      += -lrte_cryptodev
-- 
2.4.3