[PATCH] examples/vmdq: support i40e in vmdq example

[PATCH] examples/vmdq: support i40e in vmdq example

[Huawei Xie]

With i40e, the queue index of VMDQ pools doesn't always start from zero, and the queues aren't all occupied by VMDQ. These information are retrieved through rte_eth_dev_info_get, and used to initialise VMDQ.

Huawei Xie (1):
  support i40e in vmdq example

 examples/vmdq/main.c | 162 ++++++++++++++++++++++++++++++---------------------
 1 file changed, 97 insertions(+), 65 deletions(-)

-- 
1.8.1.4

[PATCH] examples/vmdq: support i40e in vmdq example

[Huawei Xie]

This patch supports i40e in vmdq example.
1. queue index is added by vmdq queue base in rte_eth_rx_burst.
2. pool index is added by vmdq pool base when mac address is added to pools.
3. add some error message print
Besides, due to some limitation in PMD,
1. mac addresses are needed to be pre-allocated to VMDQ pools.
2. ports are started before mac allocation.

Signed-off-by: Huawei Xie <huawei.xie-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Acked-by: Chen Jing D(Mark) <jing.d.chen-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Acked-by: Jijiang Liu <jijiang.liu-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Acked-by: Changchun Ouyang <changchun.ouyang.intel.com>
---
 examples/vmdq/main.c | 162 ++++++++++++++++++++++++++++++---------------------
 1 file changed, 97 insertions(+), 65 deletions(-)

diff --git a/examples/vmdq/main.c b/examples/vmdq/main.c
index 35df234..a7ffdef 100644
--- a/examples/vmdq/main.c
+++ b/examples/vmdq/main.c
@@ -194,6 +194,13 @@ const uint16_t vlan_tags[] = {
 	48, 49, 50, 51, 52, 53, 54, 55,
 	56, 57, 58, 59, 60, 61, 62, 63,
 };
+const uint16_t num_vlans = RTE_DIM(vlan_tags);
+static uint16_t num_pf_queues,  num_vmdq_queues;
+static uint16_t vmdq_pool_base, vmdq_queue_base;
+/* pool mac addr template, pool mac addr is like: 52 54 00 12 port# pool# */
+static struct ether_addr pool_addr_template = {
+	.addr_bytes = {0x52, 0x54, 0x00, 0x12, 0x00, 0x00}
+};
 
 /* ethernet addresses of ports */
 static struct ether_addr vmdq_ports_eth_addr[RTE_MAX_ETHPORTS];
@@ -213,22 +220,9 @@ get_eth_conf(struct rte_eth_conf *eth_conf, uint32_t num_pools)
 	unsigned i;
 
 	conf.nb_queue_pools = (enum rte_eth_nb_pools)num_pools;
+	conf.nb_pool_maps = num_pools;
 	conf.enable_default_pool = 0;
 	conf.default_pool = 0; /* set explicit value, even if not used */
-	switch (num_pools) {
-	/* For 10G NIC like 82599, 128 is valid for queue number */
-	case MAX_POOL_NUM_10G:
-		num_queues = MAX_QUEUE_NUM_10G;
-		conf.nb_pool_maps = MAX_POOL_MAP_NUM_10G;
-		break;
-	/* For 1G NIC like i350, 82580 and 82576, 8 is valid for queue number */
-	case MAX_POOL_NUM_1G:
-		num_queues = MAX_QUEUE_NUM_1G;
-		conf.nb_pool_maps = MAX_POOL_MAP_NUM_1G;
-		break;
-	default:
-		return -1;
-	}
 
 	for (i = 0; i < conf.nb_pool_maps; i++){
 		conf.pool_map[i].vlan_id = vlan_tags[ i ];
@@ -242,40 +236,6 @@ get_eth_conf(struct rte_eth_conf *eth_conf, uint32_t num_pools)
 }
 
 /*
- * Validate the pool number accrording to the max pool number gotten form dev_info
- * If the pool number is invalid, give the error message and return -1
- */
-static inline int
-validate_num_pools(uint32_t max_nb_pools)
-{
-	if (num_pools > max_nb_pools) {
-		printf("invalid number of pools\n");
-		return -1;
-	}
-
-	switch (max_nb_pools) {
-	/* For 10G NIC like 82599, 64 is valid for pool number */
-	case MAX_POOL_NUM_10G:
-		if (num_pools != MAX_POOL_NUM_10G) {
-			printf("invalid number of pools\n");
-			return -1;
-		}
-		break;
-	/* For 1G NIC like i350, 82580 and 82576, 8 is valid for pool number */
-	case MAX_POOL_NUM_1G:
-		if (num_pools != MAX_POOL_NUM_1G) {
-			printf("invalid number of pools\n");
-			return -1;
-		}
-		break;
-	default:
-		return -1;
-	}
-
-	return 0;
-}
-
-/*
  * Initialises a given port using global settings and with the rx buffers
  * coming from the mbuf_pool passed as parameter
  */
@@ -284,26 +244,55 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool)
 {
 	struct rte_eth_dev_info dev_info;
 	struct rte_eth_conf port_conf;
-	uint16_t rxRings, txRings = (uint16_t)rte_lcore_count();
+	uint16_t rxRings, txRings;
 	const uint16_t rxRingSize = RTE_TEST_RX_DESC_DEFAULT, txRingSize = RTE_TEST_TX_DESC_DEFAULT;
 	int retval;
 	uint16_t q;
+	uint16_t queues_per_pool;
 	uint32_t max_nb_pools;
 
 	/* The max pool number from dev_info will be used to validate the pool number specified in cmd line */
 	rte_eth_dev_info_get (port, &dev_info);
 	max_nb_pools = (uint32_t)dev_info.max_vmdq_pools;
-	retval = validate_num_pools(max_nb_pools);
-	if (retval < 0)
-		return retval;
+	if (num_pools != max_nb_pools) {
+		printf("num_pools %d != max_nb_pools %d! Currently we only"
+			"support configuring all vmdq pools\n",
+			num_pools, max_nb_pools);
+		return -1;
+	}
 
 	retval = get_eth_conf(&port_conf, num_pools);
 	if (retval < 0)
 		return retval;
 
+	/*
+	 * NIC queues are divided into pf queues and vmdq queues.
+	 */
+	/* There is assumption here all ports have the same configuration */
+	num_pf_queues = dev_info.max_rx_queues - dev_info.vmdq_queue_num;
+	queues_per_pool = dev_info.vmdq_queue_num / dev_info.max_vmdq_pools;
+	num_vmdq_queues = num_pools * queues_per_pool;
+	num_queues = num_pf_queues + num_vmdq_queues;
+	vmdq_queue_base = dev_info.vmdq_queue_base;
+	vmdq_pool_base  = dev_info.vmdq_pool_base;
+
+	printf("pf queue num: %u, configured vmdq pool num: %u,"
+		" each vmdq pool has %u queues\n",
+		num_pf_queues, num_pools, queues_per_pool);
+	printf("vmdq queue base: %d pool base %d\n",
+		vmdq_queue_base, vmdq_pool_base);
 	if (port >= rte_eth_dev_count()) return -1;
 
-	rxRings = (uint16_t)num_queues,
+	/*
+	 * Though in this example, we only receive packets from the first queue
+	 * of each pool and send packets through first rte_lcore_count() tx
+	 * queues of vmdq queues, all queues including pf queues are setup.
+	 * This is because VMDQ queues doesn't always start from zero, and the
+	 * PMD layer doesn't support selectively initialising part of rx/tx
+	 * queues well.
+	 */
+	rxRings = (uint16_t)dev_info.max_rx_queues;
+	txRings = (uint16_t)dev_info.max_tx_queues;
 	retval = rte_eth_dev_configure(port, rxRings, txRings, &port_conf);
 	if (retval != 0)
 		return retval;
@@ -312,20 +301,26 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool)
 		retval = rte_eth_rx_queue_setup(port, q, rxRingSize,
 						rte_eth_dev_socket_id(port), &rx_conf_default,
 						mbuf_pool);
-		if (retval < 0)
+		if (retval < 0) {
+			printf("initialise rx queue %d failed\n", q);
 			return retval;
+		}
 	}
 
 	for (q = 0; q < txRings; q ++) {
 		retval = rte_eth_tx_queue_setup(port, q, txRingSize,
 						rte_eth_dev_socket_id(port), &tx_conf_default);
-		if (retval < 0)
+		if (retval < 0) {
+			printf("initialise tx queue %d failed\n", q);
 			return retval;
+		}
 	}
 
 	retval  = rte_eth_dev_start(port);
-	if (retval < 0)
+	if (retval < 0) {
+		printf("port %d start failed\n", port);
 		return retval;
+	}
 
 	rte_eth_macaddr_get(port, &vmdq_ports_eth_addr[port]);
 	printf("Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8
@@ -338,6 +333,25 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool)
 			vmdq_ports_eth_addr[port].addr_bytes[4],
 			vmdq_ports_eth_addr[port].addr_bytes[5]);
 
+	/* Set mac for each pool */
+	for (q = 0; q < num_pools; q++) {
+		struct ether_addr mac;
+		mac = pool_addr_template;
+		mac.addr_bytes[4] = port;
+		mac.addr_bytes[5] = q;
+		printf("Port %u vmdq pool %u set mac %02x:%02x:%02x:%02x:%02x:%02x\n",
+			port, q,
+			mac.addr_bytes[0], mac.addr_bytes[1],
+			mac.addr_bytes[2], mac.addr_bytes[3],
+			mac.addr_bytes[4], mac.addr_bytes[5]);
+		retval = rte_eth_dev_mac_addr_add(port, &mac,
+				q + vmdq_pool_base);
+		if (retval) {
+			printf("mac addr add failed at pool %d\n", q);
+			return retval;
+		}
+	}
+
 	return 0;
 }
 
@@ -353,6 +367,11 @@ vmdq_parse_num_pools(const char *q_arg)
 	if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
 		return -1;
 
+	if (num_pools > num_vlans) {
+		printf("num_pools %d > num_vlans %d\n", num_pools, num_vlans);
+		return -1;
+	}
+
 	num_pools = n;
 
 	return 0;
@@ -481,7 +500,7 @@ lcore_main(__attribute__((__unused__)) void* dummy)
 	uint16_t core_id = 0;
 	uint16_t startQueue, endQueue;
 	uint16_t q, i, p;
-	const uint16_t remainder = (uint16_t)(num_queues % num_cores);
+	const uint16_t remainder = (uint16_t)(num_vmdq_queues % num_cores);
 
 	for (i = 0; i < num_cores; i ++)
 		if (lcore_ids[i] == lcore_id) {
@@ -491,17 +510,27 @@ lcore_main(__attribute__((__unused__)) void* dummy)
 
 	if (remainder != 0) {
 		if (core_id < remainder) {
-			startQueue = (uint16_t)(core_id * (num_queues/num_cores + 1));
-			endQueue = (uint16_t)(startQueue + (num_queues/num_cores) + 1);
+			startQueue = (uint16_t)(core_id *
+					(num_vmdq_queues / num_cores + 1));
+			endQueue = (uint16_t)(startQueue +
+					(num_vmdq_queues / num_cores) + 1);
 		} else {
-			startQueue = (uint16_t)(core_id * (num_queues/num_cores) + remainder);
-			endQueue = (uint16_t)(startQueue + (num_queues/num_cores));
+			startQueue = (uint16_t)(core_id *
+					(num_vmdq_queues / num_cores) +
+					remainder);
+			endQueue = (uint16_t)(startQueue +
+					(num_vmdq_queues / num_cores));
 		}
 	} else {
-		startQueue = (uint16_t)(core_id * (num_queues/num_cores));
-		endQueue = (uint16_t)(startQueue + (num_queues/num_cores));
+		startQueue = (uint16_t)(core_id *
+				(num_vmdq_queues / num_cores));
+		endQueue = (uint16_t)(startQueue +
+				(num_vmdq_queues / num_cores));
 	}
 
+	/* vmdq queue idx doesn't always start from zero.*/
+	startQueue += vmdq_queue_base;
+	endQueue   += vmdq_queue_base;
 	printf("core %u(lcore %u) reading queues %i-%i\n", (unsigned)core_id,
 		(unsigned)lcore_id, startQueue, endQueue - 1);
 
@@ -533,8 +562,11 @@ lcore_main(__attribute__((__unused__)) void* dummy)
 				for (i = 0; i < rxCount; i++)
 					update_mac_address(buf[i], dport);
 
-				const uint16_t txCount = rte_eth_tx_burst(dport,
-					core_id, buf, rxCount);
+				const uint16_t txCount = rte_eth_tx_burst(
+					dport,
+					vmdq_queue_base + core_id,
+					buf,
+					rxCount);
 
 				if (txCount != rxCount) {
 					for (i = txCount; i < rxCount; i++)
-- 
1.8.1.4

Re: [PATCH] examples/vmdq: support i40e in vmdq example

[Xie, Huawei]

This patch depends on "[dpdk-dev] [PATCH 0/6] i40e VMDQ support"

> -----Original Message-----
> From: Xie, Huawei
> Sent: Wednesday, September 24, 2014 6:54 PM
> To: dev-VfR2kkLFssw@public.gmane.org
> Cc: Xie, Huawei
> Subject: [PATCH] examples/vmdq: support i40e in vmdq example
> 
> This patch supports i40e in vmdq example.
> 1. queue index is added by vmdq queue base in rte_eth_rx_burst.
> 2. pool index is added by vmdq pool base when mac address is added to pools.
> 3. add some error message print
> Besides, due to some limitation in PMD,
> 1. mac addresses are needed to be pre-allocated to VMDQ pools.
> 2. ports are started before mac allocation.
> 
> Signed-off-by: Huawei Xie <huawei.xie-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> Acked-by: Chen Jing D(Mark) <jing.d.chen-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> Acked-by: Jijiang Liu <jijiang.liu-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> Acked-by: Changchun Ouyang <changchun.ouyang.intel.com>
> ---
>  examples/vmdq/main.c | 162 ++++++++++++++++++++++++++++++---------------
> ------
>  1 file changed, 97 insertions(+), 65 deletions(-)
> 
> diff --git a/examples/vmdq/main.c b/examples/vmdq/main.c
> index 35df234..a7ffdef 100644
> --- a/examples/vmdq/main.c
> +++ b/examples/vmdq/main.c
> @@ -194,6 +194,13 @@ const uint16_t vlan_tags[] = {
>  	48, 49, 50, 51, 52, 53, 54, 55,
>  	56, 57, 58, 59, 60, 61, 62, 63,
>  };
> +const uint16_t num_vlans = RTE_DIM(vlan_tags);
> +static uint16_t num_pf_queues,  num_vmdq_queues;
> +static uint16_t vmdq_pool_base, vmdq_queue_base;
> +/* pool mac addr template, pool mac addr is like: 52 54 00 12 port# pool# */
> +static struct ether_addr pool_addr_template = {
> +	.addr_bytes = {0x52, 0x54, 0x00, 0x12, 0x00, 0x00}
> +};
> 
>  /* ethernet addresses of ports */
>  static struct ether_addr vmdq_ports_eth_addr[RTE_MAX_ETHPORTS];
> @@ -213,22 +220,9 @@ get_eth_conf(struct rte_eth_conf *eth_conf, uint32_t
> num_pools)
>  	unsigned i;
> 
>  	conf.nb_queue_pools = (enum rte_eth_nb_pools)num_pools;
> +	conf.nb_pool_maps = num_pools;
>  	conf.enable_default_pool = 0;
>  	conf.default_pool = 0; /* set explicit value, even if not used */
> -	switch (num_pools) {
> -	/* For 10G NIC like 82599, 128 is valid for queue number */
> -	case MAX_POOL_NUM_10G:
> -		num_queues = MAX_QUEUE_NUM_10G;
> -		conf.nb_pool_maps = MAX_POOL_MAP_NUM_10G;
> -		break;
> -	/* For 1G NIC like i350, 82580 and 82576, 8 is valid for queue number */
> -	case MAX_POOL_NUM_1G:
> -		num_queues = MAX_QUEUE_NUM_1G;
> -		conf.nb_pool_maps = MAX_POOL_MAP_NUM_1G;
> -		break;
> -	default:
> -		return -1;
> -	}
> 
>  	for (i = 0; i < conf.nb_pool_maps; i++){
>  		conf.pool_map[i].vlan_id = vlan_tags[ i ];
> @@ -242,40 +236,6 @@ get_eth_conf(struct rte_eth_conf *eth_conf, uint32_t
> num_pools)
>  }
> 
>  /*
> - * Validate the pool number accrording to the max pool number gotten form
> dev_info
> - * If the pool number is invalid, give the error message and return -1
> - */
> -static inline int
> -validate_num_pools(uint32_t max_nb_pools)
> -{
> -	if (num_pools > max_nb_pools) {
> -		printf("invalid number of pools\n");
> -		return -1;
> -	}
> -
> -	switch (max_nb_pools) {
> -	/* For 10G NIC like 82599, 64 is valid for pool number */
> -	case MAX_POOL_NUM_10G:
> -		if (num_pools != MAX_POOL_NUM_10G) {
> -			printf("invalid number of pools\n");
> -			return -1;
> -		}
> -		break;
> -	/* For 1G NIC like i350, 82580 and 82576, 8 is valid for pool number */
> -	case MAX_POOL_NUM_1G:
> -		if (num_pools != MAX_POOL_NUM_1G) {
> -			printf("invalid number of pools\n");
> -			return -1;
> -		}
> -		break;
> -	default:
> -		return -1;
> -	}
> -
> -	return 0;
> -}
> -
> -/*
>   * Initialises a given port using global settings and with the rx buffers
>   * coming from the mbuf_pool passed as parameter
>   */
> @@ -284,26 +244,55 @@ port_init(uint8_t port, struct rte_mempool
> *mbuf_pool)
>  {
>  	struct rte_eth_dev_info dev_info;
>  	struct rte_eth_conf port_conf;
> -	uint16_t rxRings, txRings = (uint16_t)rte_lcore_count();
> +	uint16_t rxRings, txRings;
>  	const uint16_t rxRingSize = RTE_TEST_RX_DESC_DEFAULT, txRingSize =
> RTE_TEST_TX_DESC_DEFAULT;
>  	int retval;
>  	uint16_t q;
> +	uint16_t queues_per_pool;
>  	uint32_t max_nb_pools;
> 
>  	/* The max pool number from dev_info will be used to validate the pool
> number specified in cmd line */
>  	rte_eth_dev_info_get (port, &dev_info);
>  	max_nb_pools = (uint32_t)dev_info.max_vmdq_pools;
> -	retval = validate_num_pools(max_nb_pools);
> -	if (retval < 0)
> -		return retval;
> +	if (num_pools != max_nb_pools) {
> +		printf("num_pools %d != max_nb_pools %d! Currently we only"
> +			"support configuring all vmdq pools\n",
> +			num_pools, max_nb_pools);
> +		return -1;
> +	}
> 
>  	retval = get_eth_conf(&port_conf, num_pools);
>  	if (retval < 0)
>  		return retval;
> 
> +	/*
> +	 * NIC queues are divided into pf queues and vmdq queues.
> +	 */
> +	/* There is assumption here all ports have the same configuration */
> +	num_pf_queues = dev_info.max_rx_queues -
> dev_info.vmdq_queue_num;
> +	queues_per_pool = dev_info.vmdq_queue_num /
> dev_info.max_vmdq_pools;
> +	num_vmdq_queues = num_pools * queues_per_pool;
> +	num_queues = num_pf_queues + num_vmdq_queues;
> +	vmdq_queue_base = dev_info.vmdq_queue_base;
> +	vmdq_pool_base  = dev_info.vmdq_pool_base;
> +
> +	printf("pf queue num: %u, configured vmdq pool num: %u,"
> +		" each vmdq pool has %u queues\n",
> +		num_pf_queues, num_pools, queues_per_pool);
> +	printf("vmdq queue base: %d pool base %d\n",
> +		vmdq_queue_base, vmdq_pool_base);
>  	if (port >= rte_eth_dev_count()) return -1;
> 
> -	rxRings = (uint16_t)num_queues,
> +	/*
> +	 * Though in this example, we only receive packets from the first queue
> +	 * of each pool and send packets through first rte_lcore_count() tx
> +	 * queues of vmdq queues, all queues including pf queues are setup.
> +	 * This is because VMDQ queues doesn't always start from zero, and the
> +	 * PMD layer doesn't support selectively initialising part of rx/tx
> +	 * queues well.
> +	 */
> +	rxRings = (uint16_t)dev_info.max_rx_queues;
> +	txRings = (uint16_t)dev_info.max_tx_queues;
>  	retval = rte_eth_dev_configure(port, rxRings, txRings, &port_conf);
>  	if (retval != 0)
>  		return retval;
> @@ -312,20 +301,26 @@ port_init(uint8_t port, struct rte_mempool
> *mbuf_pool)
>  		retval = rte_eth_rx_queue_setup(port, q, rxRingSize,
>  						rte_eth_dev_socket_id(port),
> &rx_conf_default,
>  						mbuf_pool);
> -		if (retval < 0)
> +		if (retval < 0) {
> +			printf("initialise rx queue %d failed\n", q);
>  			return retval;
> +		}
>  	}
> 
>  	for (q = 0; q < txRings; q ++) {
>  		retval = rte_eth_tx_queue_setup(port, q, txRingSize,
>  						rte_eth_dev_socket_id(port),
> &tx_conf_default);
> -		if (retval < 0)
> +		if (retval < 0) {
> +			printf("initialise tx queue %d failed\n", q);
>  			return retval;
> +		}
>  	}
> 
>  	retval  = rte_eth_dev_start(port);
> -	if (retval < 0)
> +	if (retval < 0) {
> +		printf("port %d start failed\n", port);
>  		return retval;
> +	}
> 
>  	rte_eth_macaddr_get(port, &vmdq_ports_eth_addr[port]);
>  	printf("Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8
> @@ -338,6 +333,25 @@ port_init(uint8_t port, struct rte_mempool
> *mbuf_pool)
>  			vmdq_ports_eth_addr[port].addr_bytes[4],
>  			vmdq_ports_eth_addr[port].addr_bytes[5]);
> 
> +	/* Set mac for each pool */
> +	for (q = 0; q < num_pools; q++) {
> +		struct ether_addr mac;
> +		mac = pool_addr_template;
> +		mac.addr_bytes[4] = port;
> +		mac.addr_bytes[5] = q;
> +		printf("Port %u vmdq pool %u set
> mac %02x:%02x:%02x:%02x:%02x:%02x\n",
> +			port, q,
> +			mac.addr_bytes[0], mac.addr_bytes[1],
> +			mac.addr_bytes[2], mac.addr_bytes[3],
> +			mac.addr_bytes[4], mac.addr_bytes[5]);
> +		retval = rte_eth_dev_mac_addr_add(port, &mac,
> +				q + vmdq_pool_base);
> +		if (retval) {
> +			printf("mac addr add failed at pool %d\n", q);
> +			return retval;
> +		}
> +	}
> +
>  	return 0;
>  }
> 
> @@ -353,6 +367,11 @@ vmdq_parse_num_pools(const char *q_arg)
>  	if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
>  		return -1;
> 
> +	if (num_pools > num_vlans) {
> +		printf("num_pools %d > num_vlans %d\n", num_pools,
> num_vlans);
> +		return -1;
> +	}
> +
>  	num_pools = n;
> 
>  	return 0;
> @@ -481,7 +500,7 @@ lcore_main(__attribute__((__unused__)) void* dummy)
>  	uint16_t core_id = 0;
>  	uint16_t startQueue, endQueue;
>  	uint16_t q, i, p;
> -	const uint16_t remainder = (uint16_t)(num_queues % num_cores);
> +	const uint16_t remainder = (uint16_t)(num_vmdq_queues % num_cores);
> 
>  	for (i = 0; i < num_cores; i ++)
>  		if (lcore_ids[i] == lcore_id) {
> @@ -491,17 +510,27 @@ lcore_main(__attribute__((__unused__)) void*
> dummy)
> 
>  	if (remainder != 0) {
>  		if (core_id < remainder) {
> -			startQueue = (uint16_t)(core_id *
> (num_queues/num_cores + 1));
> -			endQueue = (uint16_t)(startQueue +
> (num_queues/num_cores) + 1);
> +			startQueue = (uint16_t)(core_id *
> +					(num_vmdq_queues / num_cores + 1));
> +			endQueue = (uint16_t)(startQueue +
> +					(num_vmdq_queues / num_cores) + 1);
>  		} else {
> -			startQueue = (uint16_t)(core_id *
> (num_queues/num_cores) + remainder);
> -			endQueue = (uint16_t)(startQueue +
> (num_queues/num_cores));
> +			startQueue = (uint16_t)(core_id *
> +					(num_vmdq_queues / num_cores) +
> +					remainder);
> +			endQueue = (uint16_t)(startQueue +
> +					(num_vmdq_queues / num_cores));
>  		}
>  	} else {
> -		startQueue = (uint16_t)(core_id * (num_queues/num_cores));
> -		endQueue = (uint16_t)(startQueue + (num_queues/num_cores));
> +		startQueue = (uint16_t)(core_id *
> +				(num_vmdq_queues / num_cores));
> +		endQueue = (uint16_t)(startQueue +
> +				(num_vmdq_queues / num_cores));
>  	}
> 
> +	/* vmdq queue idx doesn't always start from zero.*/
> +	startQueue += vmdq_queue_base;
> +	endQueue   += vmdq_queue_base;
>  	printf("core %u(lcore %u) reading queues %i-%i\n", (unsigned)core_id,
>  		(unsigned)lcore_id, startQueue, endQueue - 1);
> 
> @@ -533,8 +562,11 @@ lcore_main(__attribute__((__unused__)) void* dummy)
>  				for (i = 0; i < rxCount; i++)
>  					update_mac_address(buf[i], dport);
> 
> -				const uint16_t txCount =
> rte_eth_tx_burst(dport,
> -					core_id, buf, rxCount);
> +				const uint16_t txCount = rte_eth_tx_burst(
> +					dport,
> +					vmdq_queue_base + core_id,
> +					buf,
> +					rxCount);
> 
>  				if (txCount != rxCount) {
>  					for (i = txCount; i < rxCount; i++)
> --
> 1.8.1.4

Re: [PATCH] examples/vmdq: support i40e in vmdq example

[Xie, Huawei]

> -----Original Message-----
> From: dev [mailto:dev-bounces-VfR2kkLFssw@public.gmane.org] On Behalf Of Xie, Huawei
> Sent: Wednesday, September 24, 2014 6:58 PM
> To: dev-VfR2kkLFssw@public.gmane.org
> Subject: Re: [dpdk-dev] [PATCH] examples/vmdq: support i40e in vmdq example
> 
> This patch depends on "[dpdk-dev] [PATCH 0/6] i40e VMDQ support"
> 
> > -----Original Message-----
> > From: Xie, Huawei
> > Sent: Wednesday, September 24, 2014 6:54 PM
> > To: dev-VfR2kkLFssw@public.gmane.org
> > Cc: Xie, Huawei
> > Subject: [PATCH] examples/vmdq: support i40e in vmdq example
> >
> > This patch supports i40e in vmdq example.
> > 1. queue index is added by vmdq queue base in rte_eth_rx_burst.
> > 2. pool index is added by vmdq pool base when mac address is added to pools.
> > 3. add some error message print
> > Besides, due to some limitation in PMD,
> > 1. mac addresses are needed to be pre-allocated to VMDQ pools.
> > 2. ports are started before mac allocation.
> >
> > Signed-off-by: Huawei Xie <huawei.xie-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> > Acked-by: Chen Jing D(Mark) <jing.d.chen-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> > Acked-by: Jijiang Liu <jijiang.liu-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> > Acked-by: Changchun Ouyang <changchun.ouyang.intel.com>
Sorry, there is typo error, changchun.ouyang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
> > ---
> >  examples/vmdq/main.c | 162 ++++++++++++++++++++++++++++++-------------
> --
> > ------
> >  1 file changed, 97 insertions(+), 65 deletions(-)
> >
> > diff --git a/examples/vmdq/main.c b/examples/vmdq/main.c
> > index 35df234..a7ffdef 100644
> > --- a/examples/vmdq/main.c
> > +++ b/examples/vmdq/main.c
> > @@ -194,6 +194,13 @@ const uint16_t vlan_tags[] = {
> >  	48, 49, 50, 51, 52, 53, 54, 55,
> >  	56, 57, 58, 59, 60, 61, 62, 63,
> >  };
> > +const uint16_t num_vlans = RTE_DIM(vlan_tags);
> > +static uint16_t num_pf_queues,  num_vmdq_queues;
> > +static uint16_t vmdq_pool_base, vmdq_queue_base;
> > +/* pool mac addr template, pool mac addr is like: 52 54 00 12 port# pool# */
> > +static struct ether_addr pool_addr_template = {
> > +	.addr_bytes = {0x52, 0x54, 0x00, 0x12, 0x00, 0x00}
> > +};
> >
> >  /* ethernet addresses of ports */
> >  static struct ether_addr vmdq_ports_eth_addr[RTE_MAX_ETHPORTS];
> > @@ -213,22 +220,9 @@ get_eth_conf(struct rte_eth_conf *eth_conf,
> uint32_t
> > num_pools)
> >  	unsigned i;
> >
> >  	conf.nb_queue_pools = (enum rte_eth_nb_pools)num_pools;
> > +	conf.nb_pool_maps = num_pools;
> >  	conf.enable_default_pool = 0;
> >  	conf.default_pool = 0; /* set explicit value, even if not used */
> > -	switch (num_pools) {
> > -	/* For 10G NIC like 82599, 128 is valid for queue number */
> > -	case MAX_POOL_NUM_10G:
> > -		num_queues = MAX_QUEUE_NUM_10G;
> > -		conf.nb_pool_maps = MAX_POOL_MAP_NUM_10G;
> > -		break;
> > -	/* For 1G NIC like i350, 82580 and 82576, 8 is valid for queue number */
> > -	case MAX_POOL_NUM_1G:
> > -		num_queues = MAX_QUEUE_NUM_1G;
> > -		conf.nb_pool_maps = MAX_POOL_MAP_NUM_1G;
> > -		break;
> > -	default:
> > -		return -1;
> > -	}
> >
> >  	for (i = 0; i < conf.nb_pool_maps; i++){
> >  		conf.pool_map[i].vlan_id = vlan_tags[ i ];
> > @@ -242,40 +236,6 @@ get_eth_conf(struct rte_eth_conf *eth_conf,
> uint32_t
> > num_pools)
> >  }
> >
> >  /*
> > - * Validate the pool number accrording to the max pool number gotten form
> > dev_info
> > - * If the pool number is invalid, give the error message and return -1
> > - */
> > -static inline int
> > -validate_num_pools(uint32_t max_nb_pools)
> > -{
> > -	if (num_pools > max_nb_pools) {
> > -		printf("invalid number of pools\n");
> > -		return -1;
> > -	}
> > -
> > -	switch (max_nb_pools) {
> > -	/* For 10G NIC like 82599, 64 is valid for pool number */
> > -	case MAX_POOL_NUM_10G:
> > -		if (num_pools != MAX_POOL_NUM_10G) {
> > -			printf("invalid number of pools\n");
> > -			return -1;
> > -		}
> > -		break;
> > -	/* For 1G NIC like i350, 82580 and 82576, 8 is valid for pool number */
> > -	case MAX_POOL_NUM_1G:
> > -		if (num_pools != MAX_POOL_NUM_1G) {
> > -			printf("invalid number of pools\n");
> > -			return -1;
> > -		}
> > -		break;
> > -	default:
> > -		return -1;
> > -	}
> > -
> > -	return 0;
> > -}
> > -
> > -/*
> >   * Initialises a given port using global settings and with the rx buffers
> >   * coming from the mbuf_pool passed as parameter
> >   */
> > @@ -284,26 +244,55 @@ port_init(uint8_t port, struct rte_mempool
> > *mbuf_pool)
> >  {
> >  	struct rte_eth_dev_info dev_info;
> >  	struct rte_eth_conf port_conf;
> > -	uint16_t rxRings, txRings = (uint16_t)rte_lcore_count();
> > +	uint16_t rxRings, txRings;
> >  	const uint16_t rxRingSize = RTE_TEST_RX_DESC_DEFAULT, txRingSize =
> > RTE_TEST_TX_DESC_DEFAULT;
> >  	int retval;
> >  	uint16_t q;
> > +	uint16_t queues_per_pool;
> >  	uint32_t max_nb_pools;
> >
> >  	/* The max pool number from dev_info will be used to validate the pool
> > number specified in cmd line */
> >  	rte_eth_dev_info_get (port, &dev_info);
> >  	max_nb_pools = (uint32_t)dev_info.max_vmdq_pools;
> > -	retval = validate_num_pools(max_nb_pools);
> > -	if (retval < 0)
> > -		return retval;
> > +	if (num_pools != max_nb_pools) {
> > +		printf("num_pools %d != max_nb_pools %d! Currently we only"
> > +			"support configuring all vmdq pools\n",
> > +			num_pools, max_nb_pools);
> > +		return -1;
> > +	}
> >
> >  	retval = get_eth_conf(&port_conf, num_pools);
> >  	if (retval < 0)
> >  		return retval;
> >
> > +	/*
> > +	 * NIC queues are divided into pf queues and vmdq queues.
> > +	 */
> > +	/* There is assumption here all ports have the same configuration */
> > +	num_pf_queues = dev_info.max_rx_queues -
> > dev_info.vmdq_queue_num;
> > +	queues_per_pool = dev_info.vmdq_queue_num /
> > dev_info.max_vmdq_pools;
> > +	num_vmdq_queues = num_pools * queues_per_pool;
> > +	num_queues = num_pf_queues + num_vmdq_queues;
> > +	vmdq_queue_base = dev_info.vmdq_queue_base;
> > +	vmdq_pool_base  = dev_info.vmdq_pool_base;
> > +
> > +	printf("pf queue num: %u, configured vmdq pool num: %u,"
> > +		" each vmdq pool has %u queues\n",
> > +		num_pf_queues, num_pools, queues_per_pool);
> > +	printf("vmdq queue base: %d pool base %d\n",
> > +		vmdq_queue_base, vmdq_pool_base);
> >  	if (port >= rte_eth_dev_count()) return -1;
> >
> > -	rxRings = (uint16_t)num_queues,
> > +	/*
> > +	 * Though in this example, we only receive packets from the first queue
> > +	 * of each pool and send packets through first rte_lcore_count() tx
> > +	 * queues of vmdq queues, all queues including pf queues are setup.
> > +	 * This is because VMDQ queues doesn't always start from zero, and the
> > +	 * PMD layer doesn't support selectively initialising part of rx/tx
> > +	 * queues well.
> > +	 */
> > +	rxRings = (uint16_t)dev_info.max_rx_queues;
> > +	txRings = (uint16_t)dev_info.max_tx_queues;
> >  	retval = rte_eth_dev_configure(port, rxRings, txRings, &port_conf);
> >  	if (retval != 0)
> >  		return retval;
> > @@ -312,20 +301,26 @@ port_init(uint8_t port, struct rte_mempool
> > *mbuf_pool)
> >  		retval = rte_eth_rx_queue_setup(port, q, rxRingSize,
> >  						rte_eth_dev_socket_id(port),
> > &rx_conf_default,
> >  						mbuf_pool);
> > -		if (retval < 0)
> > +		if (retval < 0) {
> > +			printf("initialise rx queue %d failed\n", q);
> >  			return retval;
> > +		}
> >  	}
> >
> >  	for (q = 0; q < txRings; q ++) {
> >  		retval = rte_eth_tx_queue_setup(port, q, txRingSize,
> >  						rte_eth_dev_socket_id(port),
> > &tx_conf_default);
> > -		if (retval < 0)
> > +		if (retval < 0) {
> > +			printf("initialise tx queue %d failed\n", q);
> >  			return retval;
> > +		}
> >  	}
> >
> >  	retval  = rte_eth_dev_start(port);
> > -	if (retval < 0)
> > +	if (retval < 0) {
> > +		printf("port %d start failed\n", port);
> >  		return retval;
> > +	}
> >
> >  	rte_eth_macaddr_get(port, &vmdq_ports_eth_addr[port]);
> >  	printf("Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8
> > @@ -338,6 +333,25 @@ port_init(uint8_t port, struct rte_mempool
> > *mbuf_pool)
> >  			vmdq_ports_eth_addr[port].addr_bytes[4],
> >  			vmdq_ports_eth_addr[port].addr_bytes[5]);
> >
> > +	/* Set mac for each pool */
> > +	for (q = 0; q < num_pools; q++) {
> > +		struct ether_addr mac;
> > +		mac = pool_addr_template;
> > +		mac.addr_bytes[4] = port;
> > +		mac.addr_bytes[5] = q;
> > +		printf("Port %u vmdq pool %u set
> > mac %02x:%02x:%02x:%02x:%02x:%02x\n",
> > +			port, q,
> > +			mac.addr_bytes[0], mac.addr_bytes[1],
> > +			mac.addr_bytes[2], mac.addr_bytes[3],
> > +			mac.addr_bytes[4], mac.addr_bytes[5]);
> > +		retval = rte_eth_dev_mac_addr_add(port, &mac,
> > +				q + vmdq_pool_base);
> > +		if (retval) {
> > +			printf("mac addr add failed at pool %d\n", q);
> > +			return retval;
> > +		}
> > +	}
> > +
> >  	return 0;
> >  }
> >
> > @@ -353,6 +367,11 @@ vmdq_parse_num_pools(const char *q_arg)
> >  	if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
> >  		return -1;
> >
> > +	if (num_pools > num_vlans) {
> > +		printf("num_pools %d > num_vlans %d\n", num_pools,
> > num_vlans);
> > +		return -1;
> > +	}
> > +
> >  	num_pools = n;
> >
> >  	return 0;
> > @@ -481,7 +500,7 @@ lcore_main(__attribute__((__unused__)) void* dummy)
> >  	uint16_t core_id = 0;
> >  	uint16_t startQueue, endQueue;
> >  	uint16_t q, i, p;
> > -	const uint16_t remainder = (uint16_t)(num_queues % num_cores);
> > +	const uint16_t remainder = (uint16_t)(num_vmdq_queues % num_cores);
> >
> >  	for (i = 0; i < num_cores; i ++)
> >  		if (lcore_ids[i] == lcore_id) {
> > @@ -491,17 +510,27 @@ lcore_main(__attribute__((__unused__)) void*
> > dummy)
> >
> >  	if (remainder != 0) {
> >  		if (core_id < remainder) {
> > -			startQueue = (uint16_t)(core_id *
> > (num_queues/num_cores + 1));
> > -			endQueue = (uint16_t)(startQueue +
> > (num_queues/num_cores) + 1);
> > +			startQueue = (uint16_t)(core_id *
> > +					(num_vmdq_queues / num_cores + 1));
> > +			endQueue = (uint16_t)(startQueue +
> > +					(num_vmdq_queues / num_cores) + 1);
> >  		} else {
> > -			startQueue = (uint16_t)(core_id *
> > (num_queues/num_cores) + remainder);
> > -			endQueue = (uint16_t)(startQueue +
> > (num_queues/num_cores));
> > +			startQueue = (uint16_t)(core_id *
> > +					(num_vmdq_queues / num_cores) +
> > +					remainder);
> > +			endQueue = (uint16_t)(startQueue +
> > +					(num_vmdq_queues / num_cores));
> >  		}
> >  	} else {
> > -		startQueue = (uint16_t)(core_id * (num_queues/num_cores));
> > -		endQueue = (uint16_t)(startQueue + (num_queues/num_cores));
> > +		startQueue = (uint16_t)(core_id *
> > +				(num_vmdq_queues / num_cores));
> > +		endQueue = (uint16_t)(startQueue +
> > +				(num_vmdq_queues / num_cores));
> >  	}
> >
> > +	/* vmdq queue idx doesn't always start from zero.*/
> > +	startQueue += vmdq_queue_base;
> > +	endQueue   += vmdq_queue_base;
> >  	printf("core %u(lcore %u) reading queues %i-%i\n", (unsigned)core_id,
> >  		(unsigned)lcore_id, startQueue, endQueue - 1);
> >
> > @@ -533,8 +562,11 @@ lcore_main(__attribute__((__unused__)) void*
> dummy)
> >  				for (i = 0; i < rxCount; i++)
> >  					update_mac_address(buf[i], dport);
> >
> > -				const uint16_t txCount =
> > rte_eth_tx_burst(dport,
> > -					core_id, buf, rxCount);
> > +				const uint16_t txCount = rte_eth_tx_burst(
> > +					dport,
> > +					vmdq_queue_base + core_id,
> > +					buf,
> > +					rxCount);
> >
> >  				if (txCount != rxCount) {
> >  					for (i = txCount; i < rxCount; i++)
> > --
> > 1.8.1.4

Re: [PATCH] examples/vmdq: support i40e in vmdq example

[Cao, Min]

Tested-by: Min Cao <min.cao-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>

This patch has been verified on fortville and it is ready to be integrated to dpdk.org.

-----Original Message-----
From: dev [mailto:dev-bounces-VfR2kkLFssw@public.gmane.org] On Behalf Of Huawei Xie
Sent: Wednesday, September 24, 2014 6:54 PM
To: dev-VfR2kkLFssw@public.gmane.org
Subject: [dpdk-dev] [PATCH] examples/vmdq: support i40e in vmdq example

With i40e, the queue index of VMDQ pools doesn't always start from zero, and the queues aren't all occupied by VMDQ. These information are retrieved through rte_eth_dev_info_get, and used to initialise VMDQ.

Huawei Xie (1):
  support i40e in vmdq example

 examples/vmdq/main.c | 162 ++++++++++++++++++++++++++++++---------------------
 1 file changed, 97 insertions(+), 65 deletions(-)

-- 
1.8.1.4