DPDK-dev Archive on lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH v4 2/2] app/test_pmd: fix DCB configuration
From: Bernard Iremonger @ 2016-10-17 17:01 UTC (permalink / raw)
  To: dev, rahul.r.shah, wenzhuo.lu; +Cc: Bernard Iremonger
In-Reply-To: <1476720468-16774-1-git-send-email-bernard.iremonger@intel.com>

when dcb mode is DCB_VT_ENABLED set nb_rxq and nb_txq to 1.

Fixes: 2a977b891f99 ("app/testpmd: fix DCB configuration")
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 app/test-pmd/testpmd.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index 6185be6..ee567c3 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -2002,8 +2002,8 @@ init_port_dcb_config(portid_t pid,
 	 * and has the same number of rxq and txq in dcb mode
 	 */
 	if (dcb_mode == DCB_VT_ENABLED) {
-		nb_rxq = rte_port->dev_info.max_rx_queues;
-		nb_txq = rte_port->dev_info.max_tx_queues;
+		nb_rxq = 1;
+		nb_txq = 1;
 	} else {
 		/*if vt is disabled, use all pf queues */
 		if (rte_port->dev_info.vmdq_pool_base == 0) {
-- 
2.10.1

^ permalink raw reply related

* [PATCH v4 1/2] net/ixgbe: support multiqueue mode VMDq DCB with SRIOV
From: Bernard Iremonger @ 2016-10-17 17:01 UTC (permalink / raw)
  To: dev, rahul.r.shah, wenzhuo.lu; +Cc: Bernard Iremonger
In-Reply-To: <1476720468-16774-1-git-send-email-bernard.iremonger@intel.com>

Modify ixgbe_check_mq_mode function,
when SRIOV is enabled, enable mq_mode
ETH_MQ_RX_VMDQ_DCB and ETH_MQ_TX_VMDQ_DCB.

Modify ixgbe_dcb_tx_hw_config function,
replace the struct ixgbe_hw parameter with a
struct rte_eth_dev parameter and handle SRIOV enabled.

Modify ixgbe_dev_mq_rx_configure function,
when SRIOV is enabled, enable mq_mode ETH_MQ_RX_VMDQ_DCB.

Modify ixgbe_configure_dcb function,
drop check on dev->data->nb_rx_queues.

Signed-off-by: Rahul R Shah <rahul.r.shah@intel.com>
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 drivers/net/ixgbe/ixgbe_ethdev.c | 11 ++++++-----
 drivers/net/ixgbe/ixgbe_rxtx.c   | 36 ++++++++++++++++++++++--------------
 2 files changed, 28 insertions(+), 19 deletions(-)

diff --git a/drivers/net/ixgbe/ixgbe_ethdev.c b/drivers/net/ixgbe/ixgbe_ethdev.c
index 4ca5747..4d5ce83 100644
--- a/drivers/net/ixgbe/ixgbe_ethdev.c
+++ b/drivers/net/ixgbe/ixgbe_ethdev.c
@@ -1977,6 +1977,9 @@ ixgbe_check_mq_mode(struct rte_eth_dev *dev)
 		/* check multi-queue mode */
 		switch (dev_conf->rxmode.mq_mode) {
 		case ETH_MQ_RX_VMDQ_DCB:
+			PMD_INIT_LOG(INFO, "ETH_MQ_RX_VMDQ_DCB mode supported in SRIOV");
+			dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
+			break;
 		case ETH_MQ_RX_VMDQ_DCB_RSS:
 			/* DCB/RSS VMDQ in SRIOV mode, not implement yet */
 			PMD_INIT_LOG(ERR, "SRIOV active,"
@@ -2012,11 +2015,9 @@ ixgbe_check_mq_mode(struct rte_eth_dev *dev)
 
 		switch (dev_conf->txmode.mq_mode) {
 		case ETH_MQ_TX_VMDQ_DCB:
-			/* DCB VMDQ in SRIOV mode, not implement yet */
-			PMD_INIT_LOG(ERR, "SRIOV is active,"
-					" unsupported VMDQ mq_mode tx %d.",
-					dev_conf->txmode.mq_mode);
-			return -EINVAL;
+			PMD_INIT_LOG(INFO, "ETH_MQ_TX_VMDQ_DCB mode supported in SRIOV");
+			dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
+			break;
 		default: /* ETH_MQ_TX_VMDQ_ONLY or ETH_MQ_TX_NONE */
 			dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_ONLY;
 			break;
diff --git a/drivers/net/ixgbe/ixgbe_rxtx.c b/drivers/net/ixgbe/ixgbe_rxtx.c
index 2ce8234..4c13ed2 100644
--- a/drivers/net/ixgbe/ixgbe_rxtx.c
+++ b/drivers/net/ixgbe/ixgbe_rxtx.c
@@ -1,7 +1,7 @@
 /*-
  *   BSD LICENSE
  *
- *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
  *   Copyright 2014 6WIND S.A.
  *   All rights reserved.
  *
@@ -3313,15 +3313,16 @@ ixgbe_vmdq_dcb_configure(struct rte_eth_dev *dev)
 
 /**
  * ixgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters
- * @hw: pointer to hardware structure
+ * @dev: pointer to eth_dev structure
  * @dcb_config: pointer to ixgbe_dcb_config structure
  */
 static void
-ixgbe_dcb_tx_hw_config(struct ixgbe_hw *hw,
+ixgbe_dcb_tx_hw_config(struct rte_eth_dev *dev,
 		       struct ixgbe_dcb_config *dcb_config)
 {
 	uint32_t reg;
 	uint32_t q;
+	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
 	PMD_INIT_FUNC_TRACE();
 	if (hw->mac.type != ixgbe_mac_82598EB) {
@@ -3339,11 +3340,17 @@ ixgbe_dcb_tx_hw_config(struct ixgbe_hw *hw,
 		if (dcb_config->vt_mode)
 			reg |= IXGBE_MTQC_VT_ENA;
 		IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
-
-		/* Disable drop for all queues */
-		for (q = 0; q < 128; q++)
-			IXGBE_WRITE_REG(hw, IXGBE_QDE,
-				(IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT)));
+		if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
+			/* Disable drop for all queues in VMDQ mode*/
+			for (q = 0; q < 128; q++)
+				IXGBE_WRITE_REG(hw, IXGBE_QDE,
+						(IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT) | IXGBE_QDE_ENABLE));
+		} else {
+			/* Enable drop for all queues in SRIOV mode */
+			for (q = 0; q < 128; q++)
+				IXGBE_WRITE_REG(hw, IXGBE_QDE,
+						(IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT)));
+		}
 
 		/* Enable the Tx desc arbiter */
 		reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
@@ -3378,7 +3385,7 @@ ixgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev,
 			vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
 
 	/*Configure general DCB TX parameters*/
-	ixgbe_dcb_tx_hw_config(hw, dcb_config);
+	ixgbe_dcb_tx_hw_config(dev, dcb_config);
 }
 
 static void
@@ -3661,7 +3668,7 @@ ixgbe_dcb_hw_configure(struct rte_eth_dev *dev,
 		/*get DCB TX configuration parameters from rte_eth_conf*/
 		ixgbe_dcb_tx_config(dev, dcb_config);
 		/*Configure general DCB TX parameters*/
-		ixgbe_dcb_tx_hw_config(hw, dcb_config);
+		ixgbe_dcb_tx_hw_config(dev, dcb_config);
 		break;
 	default:
 		PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration");
@@ -3810,7 +3817,7 @@ void ixgbe_configure_dcb(struct rte_eth_dev *dev)
 	    (dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB_RSS))
 		return;
 
-	if (dev->data->nb_rx_queues != ETH_DCB_NUM_QUEUES)
+	if (dev->data->nb_rx_queues > ETH_DCB_NUM_QUEUES)
 		return;
 
 	/** Configure DCB hardware **/
@@ -4082,12 +4089,13 @@ ixgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
 		case ETH_MQ_RX_VMDQ_RSS:
 			ixgbe_config_vf_rss(dev);
 			break;
-
-		/* FIXME if support DCB/RSS together with VMDq & SRIOV */
 		case ETH_MQ_RX_VMDQ_DCB:
+			ixgbe_vmdq_dcb_configure(dev);
+			break;
+		/* FIXME if support DCB/RSS together with VMDq & SRIOV */
 		case ETH_MQ_RX_VMDQ_DCB_RSS:
 			PMD_INIT_LOG(ERR,
-				"Could not support DCB with VMDq & SRIOV");
+				"Could not support DCB/RSS with VMDq & SRIOV");
 			return -1;
 		default:
 			ixgbe_config_vf_default(dev);
-- 
2.10.1

^ permalink raw reply related

* [PATCH v4 0/2] net/ixgbe: VMDq DCB with SRIOV
From: Bernard Iremonger @ 2016-10-17 17:01 UTC (permalink / raw)
  To: dev, rahul.r.shah, wenzhuo.lu; +Cc: Bernard Iremonger
In-Reply-To: <1476720468-16774-1-git-send-email-bernard.iremonger@intel.com>

Changes in v4:
changes to ixgbe patch following comments.

Changes in v3:
rebase to latest master.
update commit message for ixgbe patch
add testpmd patch.

Changes in v2:
rebase to  latest master.

Bernard Iremonger (2):
  net/ixgbe: support multiqueue mode VMDq DCB with SRIOV
  app/test_pmd: fix DCB configuration

 app/test-pmd/testpmd.c           |  4 ++--
 drivers/net/ixgbe/ixgbe_ethdev.c | 11 ++++++-----
 drivers/net/ixgbe/ixgbe_rxtx.c   | 36 ++++++++++++++++++++++--------------
 3 files changed, 30 insertions(+), 21 deletions(-)

-- 
2.10.1

^ permalink raw reply

* Re: [PATCH v3 02/32] qede/base: formatting changes
From: Ferruh Yigit @ 2016-10-17 16:28 UTC (permalink / raw)
  To: Rasesh Mody, thomas.monjalon, bruce.richardson; +Cc: dev, Dept-EngDPDKDev
In-Reply-To: <1476562089-21823-3-git-send-email-rasesh.mody@qlogic.com>

Hi Rasesh,

On 10/15/2016 9:07 PM, Rasesh Mody wrote:
> Fixes white spaces and tabs.
> 
> Signed-off-by: Rasesh Mody <rasesh.mody@qlogic.com>
> ---

When testing patch by patch compilation, applying this patch gives
following build error, the build error disappears after patch 12/32.

It should successfully compile after each patch, can you please fix this?


Build error:

.../drivers/net/qede/base/ecore_sp_commands.c: In function
‘ecore_sp_pf_update_tunn_cfg’:
.../drivers/net/qede/base/ecore_sp_commands.c:473:2: error: this ‘if’
clause does not guard... [-Werror=misleading-indentation]
  if (p_tunn->update_geneve_udp_port)
  ^~
.../drivers/net/qede/base/ecore_sp_commands.c:477:3: note: ...this
statement, but the latter is misleadingly indented as if it is guarded
by the ‘if’
   ecore_set_hw_tunn_mode(p_hwfn, p_hwfn->p_main_ptt,
   ^~~~~~~~~~~~~~~~~~~~~~
cc1: all warnings being treated as errors
make[10]: *** [base/ecore_sp_commands.o] Error 1
make[10]: *** Waiting for unfinished jobs....
.../drivers/net/qede/base/ecore_spq.c: In function ‘ecore_spq_add_entry’:
.../drivers/net/qede/base/ecore_spq.c:628:3: error: this ‘if’ clause
does not guard... [-Werror=misleading-indentation]
   if (p_ent->comp_cb.cookie == &p_ent->comp_done)
   ^~
.../drivers/net/qede/base/ecore_spq.c:631:4: note: ...this statement,
but the latter is misleadingly indented as if it is guarded by the ‘if’
    *p_en2 = *p_ent;
    ^
cc1: all warnings being treated as errors


Thanks,
ferruh

^ permalink raw reply

* Re: [PATCH v2] net/ixgbe: support multiqueue mode VMDq DCB with SRIOV
From: Iremonger, Bernard @ 2016-10-17 16:27 UTC (permalink / raw)
  To: Lu, Wenzhuo, dev@dpdk.org, Shah, Rahul R
In-Reply-To: <6A0DE07E22DDAD4C9103DF62FEBC09093933BE4F@shsmsx102.ccr.corp.intel.com>

Hi Wenzhuo,

<snip>

> > Subject: [PATCH v2] net/ixgbe: support multiqueue mode VMDq DCB with
> > SRIOV
> >
> > modify ixgbe_dcb_tx_hw_config function.
> > modify ixgbe_dev_mq_rx_configure function.
> > modify ixgbe_configure_dcb function.
> Would you like to add more details about why we need to modify these
> functions and what has been done? Thanks.

I have updated the commit message for this patch with more information.
I have also added a testpmd patch and sent a v3 patchset.


> > Changes in v2:
> > Rebased to DPDK v16.11-rc1
> >
> > Signed-off-by: Rahul R Shah <rahul.r.shah@intel.com>
> > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > ---
> >  drivers/net/ixgbe/ixgbe_ethdev.c |  9 ++++-----
> >  drivers/net/ixgbe/ixgbe_rxtx.c   | 37 +++++++++++++++++++++------------
> ----
> >  2 files changed, 25 insertions(+), 21 deletions(-)
> >
> > diff --git a/drivers/net/ixgbe/ixgbe_ethdev.c
> > b/drivers/net/ixgbe/ixgbe_ethdev.c
> > index 4ca5747..114698d 100644
> > --- a/drivers/net/ixgbe/ixgbe_ethdev.c
> > +++ b/drivers/net/ixgbe/ixgbe_ethdev.c
> > @@ -1977,6 +1977,8 @@ ixgbe_check_mq_mode(struct rte_eth_dev
> *dev)
> >  		/* check multi-queue mode */
> >  		switch (dev_conf->rxmode.mq_mode) {
> >  		case ETH_MQ_RX_VMDQ_DCB:
> Should we store the mq mode here?

Yes, it probably should be stored here.

 
> > +			PMD_INIT_LOG(INFO, "ETH_MQ_RX_VMDQ_DCB
> > mode supported in SRIOV");
> > +			break;
> >  		case ETH_MQ_RX_VMDQ_DCB_RSS:
> >  			/* DCB/RSS VMDQ in SRIOV mode, not implement
> yet */
> >  			PMD_INIT_LOG(ERR, "SRIOV active,"
> > @@ -2012,11 +2014,8 @@ ixgbe_check_mq_mode(struct rte_eth_dev
> *dev)
> >
> >  		switch (dev_conf->txmode.mq_mode) {
> >  		case ETH_MQ_TX_VMDQ_DCB:
> > -			/* DCB VMDQ in SRIOV mode, not implement yet */
> > -			PMD_INIT_LOG(ERR, "SRIOV is active,"
> > -					" unsupported VMDQ mq_mode
> > tx %d.",
> > -					dev_conf->txmode.mq_mode);
> > -			return -EINVAL;
> > +			PMD_INIT_LOG(INFO, "ETH_MQ_TX_VMDQ_DCB
> mode
> > supported in SRIOV");
> > +			break;
> >  		default: /* ETH_MQ_TX_VMDQ_ONLY or
> ETH_MQ_TX_NONE */
> >  			dev->data->dev_conf.txmode.mq_mode =
> ETH_MQ_TX_VMDQ_ONLY;
> >  			break;
> > diff --git a/drivers/net/ixgbe/ixgbe_rxtx.c
> > b/drivers/net/ixgbe/ixgbe_rxtx.c index
> > 2ce8234..bb13889 100644
> > --- a/drivers/net/ixgbe/ixgbe_rxtx.c
> > +++ b/drivers/net/ixgbe/ixgbe_rxtx.c
> > @@ -1,7 +1,7 @@
> >  /*-
> >   *   BSD LICENSE
> >   *
> > - *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
> > + *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
> >   *   Copyright 2014 6WIND S.A.
> >   *   All rights reserved.
> >   *
> > @@ -3313,15 +3313,16 @@ ixgbe_vmdq_dcb_configure(struct
> rte_eth_dev
> > *dev)
> >
> >  /**
> >   * ixgbe_dcb_config_tx_hw_config - Configure general DCB TX
> > parameters
> > - * @hw: pointer to hardware structure
> > + * @dev: pointer to eth_dev structure
> >   * @dcb_config: pointer to ixgbe_dcb_config structure
> >   */
> >  static void
> > -ixgbe_dcb_tx_hw_config(struct ixgbe_hw *hw,
> > +ixgbe_dcb_tx_hw_config(struct rte_eth_dev *dev,
> >  		       struct ixgbe_dcb_config *dcb_config)  {
> >  	uint32_t reg;
> >  	uint32_t q;
> > +	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data-
> > >dev_private);
> >
> >  	PMD_INIT_FUNC_TRACE();
> >  	if (hw->mac.type != ixgbe_mac_82598EB) { @@ -3339,11 +3340,17
> @@
> > ixgbe_dcb_tx_hw_config(struct ixgbe_hw *hw,
> >  		if (dcb_config->vt_mode)
> >  			reg |= IXGBE_MTQC_VT_ENA;
> >  		IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
> > -
> > -		/* Disable drop for all queues */
> > -		for (q = 0; q < 128; q++)
> > -			IXGBE_WRITE_REG(hw, IXGBE_QDE,
> > -				(IXGBE_QDE_WRITE | (q <<
> > IXGBE_QDE_IDX_SHIFT)));
> > +		if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
> > +			/* Disable drop for all queues in VMDQ mode*/
> > +			for (q = 0; q < 128; q++)
> > +				IXGBE_WRITE_REG(hw, IXGBE_QDE,
> > +						(IXGBE_QDE_WRITE | (q <<
> > IXGBE_QDE_IDX_SHIFT) | IXGBE_QDE_ENABLE));
> > +		} else {
> > +			/* Enable drop for all queues in SRIOV mode */
> > +			for (q = 0; q < 128; q++)
> > +				IXGBE_WRITE_REG(hw, IXGBE_QDE,
> > +						(IXGBE_QDE_WRITE | (q <<
> > IXGBE_QDE_IDX_SHIFT)));
> > +		}
> I think it has nothing to do with mq mode. Do I miss something?

Behavior is different when SRIOV is enabled.

> 
> >
> >  		/* Enable the Tx desc arbiter */
> >  		reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS); @@ -3378,7
> > +3385,7 @@ ixgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev,
> >  			vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS
> ?
> > 0xFFFF : 0xFFFFFFFF);
> >
> >  	/*Configure general DCB TX parameters*/
> > -	ixgbe_dcb_tx_hw_config(hw, dcb_config);
> > +	ixgbe_dcb_tx_hw_config(dev, dcb_config);
> >  }
> >
> >  static void
> > @@ -3661,7 +3668,7 @@ ixgbe_dcb_hw_configure(struct rte_eth_dev
> *dev,
> >  		/*get DCB TX configuration parameters from rte_eth_conf*/
> >  		ixgbe_dcb_tx_config(dev, dcb_config);
> >  		/*Configure general DCB TX parameters*/
> > -		ixgbe_dcb_tx_hw_config(hw, dcb_config);
> > +		ixgbe_dcb_tx_hw_config(dev, dcb_config);
> >  		break;
> >  	default:
> >  		PMD_INIT_LOG(ERR, "Incorrect DCB TX mode
> configuration"); @@
> > -3810,9 +3817,6 @@ void ixgbe_configure_dcb(struct rte_eth_dev *dev)
> >  	    (dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB_RSS))
> >  		return;
> >
> > -	if (dev->data->nb_rx_queues != ETH_DCB_NUM_QUEUES)
> > -		return;
> I remember it's a limitation of implementation. The reason is the resource
> allocation. Why could we remove it now?

ETH_DCB_NUM_QUEUES is 128,  nb_rx_queues may not be 128.

> 
> > -
> >  	/** Configure DCB hardware **/
> >  	ixgbe_dcb_hw_configure(dev, dcb_cfg);  } @@ -4082,12 +4086,13
> @@
> > ixgbe_dev_mq_rx_configure(struct rte_eth_dev
> > *dev)
> >  		case ETH_MQ_RX_VMDQ_RSS:
> >  			ixgbe_config_vf_rss(dev);
> >  			break;
> > -
> > -		/* FIXME if support DCB/RSS together with VMDq & SRIOV */
> >  		case ETH_MQ_RX_VMDQ_DCB:
> > +			ixgbe_vmdq_dcb_configure(dev);
> > +			break;
> > +		/* FIXME if support DCB/RSS together with VMDq & SRIOV */
> >  		case ETH_MQ_RX_VMDQ_DCB_RSS:
> >  			PMD_INIT_LOG(ERR,
> > -				"Could not support DCB with VMDq &
> SRIOV");
> > +				"Could not support DCB/RSS with VMDq &
> > SRIOV");
> >  			return -1;
> >  		default:
> >  			ixgbe_config_vf_default(dev);
> > --
> > 2.10.1

Regards,

Bernard.

^ permalink raw reply

* Re: [PATCH v5 1/6] ethdev: add Tx preparation
From: Kulasek, TomaszX @ 2016-10-17 16:25 UTC (permalink / raw)
  To: Thomas Monjalon; +Cc: dev@dpdk.org, Ananyev, Konstantin, olivier.matz@6wind.com
In-Reply-To: <3567043.JXAYMJHPG7@xps13>

Hi Thomas,

> -----Original Message-----
> From: Thomas Monjalon [mailto:thomas.monjalon@6wind.com]
> Sent: Friday, October 14, 2016 16:20
> To: Kulasek, TomaszX <tomaszx.kulasek@intel.com>
> Cc: dev@dpdk.org; Ananyev, Konstantin <konstantin.ananyev@intel.com>;
> olivier.matz@6wind.com
> Subject: Re: [dpdk-dev] [PATCH v5 1/6] ethdev: add Tx preparation
> 
> 2016-10-14 14:02, Kulasek, TomaszX:
> > From: Thomas Monjalon [mailto:thomas.monjalon@6wind.com]
> > > 2016-10-13 19:36, Tomasz Kulasek:
> > > > +/**
> > > > + * Fix pseudo header checksum for TSO and non-TSO tcp/udp packets
> > > before
> > > > + * hardware tx checksum.
> > > > + * For non-TSO tcp/udp packets full pseudo-header checksum is
> > > > + counted
> > > and set.
> > > > + * For TSO the IP payload length is not included.
> > > > + */
> > > > +static inline int
> > > > +rte_phdr_cksum_fix(struct rte_mbuf *m)
> > >
> > > You probably don't need this function since the recent improvements
> > > from Olivier.
> >
> > Do you mean this improvement: "net: add function to calculate a checksum
> in a mbuf"
> > http://dpdk.org/dev/patchwork/patch/16542/
> >
> > I see only full raw checksum computation on mbuf in Olivier patches,
> while this function counts only pseudo-header checksum to be used with tx
> offload.
> 
> OK. Please check what exists already in librte_net (especially rte_ip.h)
> and try to re-use code if possible. Thanks

I have already sent v6 with requested changes in Friday. There's no equivalent of rte_phdr_cksum_fix in librte_net. This function already uses rte_ipv4_phdr_cksum and rte_ipv6_phdr_cksum and there's nothing similar on the higher level to simplify it even more.

Tomasz

^ permalink raw reply

* [PATCH v3 2/2] app/test_pmd: fix DCB configuration
From: Bernard Iremonger @ 2016-10-17 16:07 UTC (permalink / raw)
  To: dev, rahul.r.shah, wenzhuo.lu; +Cc: Bernard Iremonger
In-Reply-To: <1476451768-21278-1-git-send-email-bernard.iremonger@intel.com>

when dcb mode is DCB_VT_ENABLED set nb_rxq and nb_txq to 1.

Fixes: 2a977b891f99 ("app/testpmd: fix DCB configuration")
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 app/test-pmd/testpmd.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index 6185be6..ee567c3 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -2002,8 +2002,8 @@ init_port_dcb_config(portid_t pid,
 	 * and has the same number of rxq and txq in dcb mode
 	 */
 	if (dcb_mode == DCB_VT_ENABLED) {
-		nb_rxq = rte_port->dev_info.max_rx_queues;
-		nb_txq = rte_port->dev_info.max_tx_queues;
+		nb_rxq = 1;
+		nb_txq = 1;
 	} else {
 		/*if vt is disabled, use all pf queues */
 		if (rte_port->dev_info.vmdq_pool_base == 0) {
-- 
2.10.1

^ permalink raw reply related

* [PATCH v3 1/2] net/ixgbe: support multiqueue mode VMDq DCB with SRIOV
From: Bernard Iremonger @ 2016-10-17 16:07 UTC (permalink / raw)
  To: dev, rahul.r.shah, wenzhuo.lu; +Cc: Bernard Iremonger
In-Reply-To: <1476451768-21278-1-git-send-email-bernard.iremonger@intel.com>

Modify ixgbe_check_mq_mode function,
when SRIOV is enabled, enable mq_mode
ETH_MQ_RX_VMDQ_DCB and ETH_MQ_TX_VMDQ_DCB.

Modify ixgbe_dcb_tx_hw_config function,
replace the struct ixgbe_hw parameter with a
struct rte_eth_dev parameter and handle SRIOV enabled.

Modify ixgbe_dev_mq_rx_configure function,
when SRIOV is enabled, enable mq_mode ETH_MQ_RX_VMDQ_DCB.

Modify ixgbe_configure_dcb function,
drop check on dev->data->nb_rx_queues.

Signed-off-by: Rahul R Shah <rahul.r.shah@intel.com>
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 drivers/net/ixgbe/ixgbe_ethdev.c |  9 ++++-----
 drivers/net/ixgbe/ixgbe_rxtx.c   | 37 +++++++++++++++++++++----------------
 2 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/drivers/net/ixgbe/ixgbe_ethdev.c b/drivers/net/ixgbe/ixgbe_ethdev.c
index 4ca5747..114698d 100644
--- a/drivers/net/ixgbe/ixgbe_ethdev.c
+++ b/drivers/net/ixgbe/ixgbe_ethdev.c
@@ -1977,6 +1977,8 @@ ixgbe_check_mq_mode(struct rte_eth_dev *dev)
 		/* check multi-queue mode */
 		switch (dev_conf->rxmode.mq_mode) {
 		case ETH_MQ_RX_VMDQ_DCB:
+			PMD_INIT_LOG(INFO, "ETH_MQ_RX_VMDQ_DCB mode supported in SRIOV");
+			break;
 		case ETH_MQ_RX_VMDQ_DCB_RSS:
 			/* DCB/RSS VMDQ in SRIOV mode, not implement yet */
 			PMD_INIT_LOG(ERR, "SRIOV active,"
@@ -2012,11 +2014,8 @@ ixgbe_check_mq_mode(struct rte_eth_dev *dev)
 
 		switch (dev_conf->txmode.mq_mode) {
 		case ETH_MQ_TX_VMDQ_DCB:
-			/* DCB VMDQ in SRIOV mode, not implement yet */
-			PMD_INIT_LOG(ERR, "SRIOV is active,"
-					" unsupported VMDQ mq_mode tx %d.",
-					dev_conf->txmode.mq_mode);
-			return -EINVAL;
+			PMD_INIT_LOG(INFO, "ETH_MQ_TX_VMDQ_DCB mode supported in SRIOV");
+			break;
 		default: /* ETH_MQ_TX_VMDQ_ONLY or ETH_MQ_TX_NONE */
 			dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_ONLY;
 			break;
diff --git a/drivers/net/ixgbe/ixgbe_rxtx.c b/drivers/net/ixgbe/ixgbe_rxtx.c
index 2ce8234..bb13889 100644
--- a/drivers/net/ixgbe/ixgbe_rxtx.c
+++ b/drivers/net/ixgbe/ixgbe_rxtx.c
@@ -1,7 +1,7 @@
 /*-
  *   BSD LICENSE
  *
- *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
  *   Copyright 2014 6WIND S.A.
  *   All rights reserved.
  *
@@ -3313,15 +3313,16 @@ ixgbe_vmdq_dcb_configure(struct rte_eth_dev *dev)
 
 /**
  * ixgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters
- * @hw: pointer to hardware structure
+ * @dev: pointer to eth_dev structure
  * @dcb_config: pointer to ixgbe_dcb_config structure
  */
 static void
-ixgbe_dcb_tx_hw_config(struct ixgbe_hw *hw,
+ixgbe_dcb_tx_hw_config(struct rte_eth_dev *dev,
 		       struct ixgbe_dcb_config *dcb_config)
 {
 	uint32_t reg;
 	uint32_t q;
+	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
 	PMD_INIT_FUNC_TRACE();
 	if (hw->mac.type != ixgbe_mac_82598EB) {
@@ -3339,11 +3340,17 @@ ixgbe_dcb_tx_hw_config(struct ixgbe_hw *hw,
 		if (dcb_config->vt_mode)
 			reg |= IXGBE_MTQC_VT_ENA;
 		IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
-
-		/* Disable drop for all queues */
-		for (q = 0; q < 128; q++)
-			IXGBE_WRITE_REG(hw, IXGBE_QDE,
-				(IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT)));
+		if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
+			/* Disable drop for all queues in VMDQ mode*/
+			for (q = 0; q < 128; q++)
+				IXGBE_WRITE_REG(hw, IXGBE_QDE,
+						(IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT) | IXGBE_QDE_ENABLE));
+		} else {
+			/* Enable drop for all queues in SRIOV mode */
+			for (q = 0; q < 128; q++)
+				IXGBE_WRITE_REG(hw, IXGBE_QDE,
+						(IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT)));
+		}
 
 		/* Enable the Tx desc arbiter */
 		reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
@@ -3378,7 +3385,7 @@ ixgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev,
 			vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
 
 	/*Configure general DCB TX parameters*/
-	ixgbe_dcb_tx_hw_config(hw, dcb_config);
+	ixgbe_dcb_tx_hw_config(dev, dcb_config);
 }
 
 static void
@@ -3661,7 +3668,7 @@ ixgbe_dcb_hw_configure(struct rte_eth_dev *dev,
 		/*get DCB TX configuration parameters from rte_eth_conf*/
 		ixgbe_dcb_tx_config(dev, dcb_config);
 		/*Configure general DCB TX parameters*/
-		ixgbe_dcb_tx_hw_config(hw, dcb_config);
+		ixgbe_dcb_tx_hw_config(dev, dcb_config);
 		break;
 	default:
 		PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration");
@@ -3810,9 +3817,6 @@ void ixgbe_configure_dcb(struct rte_eth_dev *dev)
 	    (dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB_RSS))
 		return;
 
-	if (dev->data->nb_rx_queues != ETH_DCB_NUM_QUEUES)
-		return;
-
 	/** Configure DCB hardware **/
 	ixgbe_dcb_hw_configure(dev, dcb_cfg);
 }
@@ -4082,12 +4086,13 @@ ixgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
 		case ETH_MQ_RX_VMDQ_RSS:
 			ixgbe_config_vf_rss(dev);
 			break;
-
-		/* FIXME if support DCB/RSS together with VMDq & SRIOV */
 		case ETH_MQ_RX_VMDQ_DCB:
+			ixgbe_vmdq_dcb_configure(dev);
+			break;
+		/* FIXME if support DCB/RSS together with VMDq & SRIOV */
 		case ETH_MQ_RX_VMDQ_DCB_RSS:
 			PMD_INIT_LOG(ERR,
-				"Could not support DCB with VMDq & SRIOV");
+				"Could not support DCB/RSS with VMDq & SRIOV");
 			return -1;
 		default:
 			ixgbe_config_vf_default(dev);
-- 
2.10.1

^ permalink raw reply related

* [PATCH v3 0/2] net/ixgbe: VMDq DCB with SRIOV
From: Bernard Iremonger @ 2016-10-17 16:07 UTC (permalink / raw)
  To: dev, rahul.r.shah, wenzhuo.lu; +Cc: Bernard Iremonger
In-Reply-To: <1476451768-21278-1-git-send-email-bernard.iremonger@intel.com>

Changes in v3:
rebase to latest master.
update commit message for ixgbe patch
add testpmd patch.

Changes in v2:
rebase to  latest master.

Bernard Iremonger (2):
  net/ixgbe: support multiqueue mode VMDq DCB with SRIOV
  app/test_pmd: fix DCB configuration

 app/test-pmd/testpmd.c           |  4 ++--
 drivers/net/ixgbe/ixgbe_ethdev.c |  9 ++++-----
 drivers/net/ixgbe/ixgbe_rxtx.c   | 37 +++++++++++++++++++++----------------
 3 files changed, 27 insertions(+), 23 deletions(-)

-- 
2.10.1

^ permalink raw reply

* Re: [PATCH 0/3] net: fix out of order rx read issue
From: Chen, Jing D @ 2016-10-17 15:55 UTC (permalink / raw)
  To: Zhang, Qi Z, Wu, Jingjing, Zhang, Helin; +Cc: dev@dpdk.org, Zhang, Qi Z
In-Reply-To: <1476692614-35793-1-git-send-email-qi.z.zhang@intel.com>

Hi,

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Qi Zhang
> Sent: Monday, October 17, 2016 1:24 AM
> To: Wu, Jingjing <jingjing.wu@intel.com>; Zhang, Helin
> <helin.zhang@intel.com>
> Cc: dev@dpdk.org; Zhang, Qi Z <qi.z.zhang@intel.com>
> Subject: [dpdk-dev] [PATCH 0/3] net: fix out of order rx read issue
> 
> Volatile point has been cast to non-volatile point when call _mm_loadu_si128,
> so add compile barrier to prevent compiler reorder.
> 
> Qi Zhang (3):
>   net/i40e: fix out of order rx read issue
>   net/ixgbe: fix out of order rx read issue
>   net/fm10k: fix out of order rx read issue
> 
>  drivers/net/fm10k/fm10k_rxtx_vec.c     | 3 +++
>  drivers/net/i40e/i40e_rxtx_vec.c       | 3 +++
>  drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 3 +++
>  3 files changed, 9 insertions(+)

I have an overall comment on the committed message. You'd better to describe
why we have out of order issue here and the requirement on the execution order
of the 4 loads. 

^ permalink raw reply

* [PATCH] examples/ip_pipeline: fix freeBSD build error
From: Jasvinder Singh @ 2016-10-17 15:48 UTC (permalink / raw)
  To: dev; +Cc: cristian.dumitrescu

Error log:
 CC init.o
 examples/ip_pipeline/init.c:38:22: fatal error: linux/if.h: No such file or directory
 #include <linux/if.h>
                      ^
Fixes: 3f2c9f3bb6c6 ("examples/ip_pipeline: add TAP port")

Signed-off-by: Jasvinder Singh <jasvinder.singh@intel.com>
---
 examples/ip_pipeline/init.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/examples/ip_pipeline/init.c b/examples/ip_pipeline/init.c
index 4fed474..3b36b53 100644
--- a/examples/ip_pipeline/init.c
+++ b/examples/ip_pipeline/init.c
@@ -35,8 +35,10 @@
 #include <stdio.h>
 #include <string.h>
 #include <netinet/in.h>
+#ifdef RTE_EXEC_ENV_LINUXAPP
 #include <linux/if.h>
 #include <linux/if_tun.h>
+#endif
 #include <fcntl.h>
 #include <sys/ioctl.h>
 #include <unistd.h>
@@ -1160,6 +1162,15 @@ app_init_tm(struct app_params *app)
 	}
 }
 
+#ifndef RTE_EXEC_ENV_LINUXAPP
+static void
+app_init_tap(struct app_params *app) {
+	if (app->n_pktq_tap == 0)
+		return;
+
+	rte_panic("TAP device not supported.\n");
+}
+#else
 static void
 app_init_tap(struct app_params *app)
 {
@@ -1187,6 +1198,7 @@ app_init_tap(struct app_params *app)
 		app->tap[i] = fd;
 	}
 }
+#endif
 
 #ifdef RTE_LIBRTE_KNI
 static int
@@ -1404,6 +1416,7 @@ void app_pipeline_params_get(struct app_params *app,
 			out->burst_size = app->tm_params[in->id].burst_read;
 			break;
 		}
+#ifdef RTE_EXEC_ENV_LINUXAPP
 		case APP_PKTQ_IN_TAP:
 		{
 			struct app_pktq_tap_params *tap_params =
@@ -1420,6 +1433,7 @@ void app_pipeline_params_get(struct app_params *app,
 			out->burst_size = app->tap_params[in->id].burst_read;
 			break;
 		}
+#endif
 #ifdef RTE_LIBRTE_KNI
 		case APP_PKTQ_IN_KNI:
 		{
@@ -1564,6 +1578,7 @@ void app_pipeline_params_get(struct app_params *app,
 				app->tm_params[in->id].burst_write;
 			break;
 		}
+#ifdef RTE_EXEC_ENV_LINUXAPP
 		case APP_PKTQ_OUT_TAP:
 		{
 			struct rte_port_fd_writer_params *params =
@@ -1575,6 +1590,7 @@ void app_pipeline_params_get(struct app_params *app,
 				app->tap_params[in->id].burst_write;
 			break;
 		}
+#endif
 #ifdef RTE_LIBRTE_KNI
 		case APP_PKTQ_OUT_KNI:
 		{
-- 
2.5.5

^ permalink raw reply related

* [PATCH 3/3] net/fm10k: fix out of order rx read issue
From: Qi Zhang @ 2016-10-17  8:23 UTC (permalink / raw)
  To: jingjing.wu, helin.zhang; +Cc: dev, Qi Zhang
In-Reply-To: <1476692614-35793-1-git-send-email-qi.z.zhang@intel.com>

Volatile point has been cast to non-volatile
point when call _mm_loadu_si128, so add compile
barrier to prevent compiler reorder.

Fixes: 7092be8437bd ("fm10k: add vector RX")

Signed-off-by: Qi Zhang <qi.z.zhang@intel.com>
---
 drivers/net/fm10k/fm10k_rxtx_vec.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c b/drivers/net/fm10k/fm10k_rxtx_vec.c
index b724486..27f3e43 100644
--- a/drivers/net/fm10k/fm10k_rxtx_vec.c
+++ b/drivers/net/fm10k/fm10k_rxtx_vec.c
@@ -478,6 +478,7 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
 		/* Read desc statuses backwards to avoid race condition */
 		/* A.1 load 4 pkts desc */
 		descs0[3] = _mm_loadu_si128((__m128i *)(rxdp + 3));
+		rte_compiler_barrier();
 
 		/* B.2 copy 2 mbuf point into rx_pkts  */
 		_mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1);
@@ -486,8 +487,10 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
 		mbp2 = _mm_loadu_si128((__m128i *)&mbufp[pos+2]);
 
 		descs0[2] = _mm_loadu_si128((__m128i *)(rxdp + 2));
+		rte_compiler_barrier();
 		/* B.1 load 2 mbuf point */
 		descs0[1] = _mm_loadu_si128((__m128i *)(rxdp + 1));
+		rte_compiler_barrier();
 		descs0[0] = _mm_loadu_si128((__m128i *)(rxdp));
 
 		/* B.2 copy 2 mbuf point into rx_pkts  */
-- 
2.7.4

^ permalink raw reply related

* [PATCH 2/3] net/ixgbe: fix out of order rx read issue
From: Qi Zhang @ 2016-10-17  8:23 UTC (permalink / raw)
  To: jingjing.wu, helin.zhang; +Cc: dev, Qi Zhang
In-Reply-To: <1476692614-35793-1-git-send-email-qi.z.zhang@intel.com>

Volatile point has been cast to non-volatile
point when call _mm_loadu_si128, so add compile
barrier to prevent compiler reorder.

Fixes: c95584dc2b18 ("ixgbe: new vectorized functions for Rx/Tx")

Signed-off-by: Qi Zhang <qi.z.zhang@intel.com>
---
 drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
index ad8a9ff..abbf284 100644
--- a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
+++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
@@ -343,6 +343,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
 		/* Read desc statuses backwards to avoid race condition */
 		/* A.1 load 4 pkts desc */
 		descs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3));
+		rte_compiler_barrier();
 
 		/* B.2 copy 2 mbuf point into rx_pkts  */
 		_mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1);
@@ -351,8 +352,10 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
 		mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos+2]);
 
 		descs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2));
+		rte_compiler_barrier();
 		/* B.1 load 2 mbuf point */
 		descs[1] = _mm_loadu_si128((__m128i *)(rxdp + 1));
+		rte_compiler_barrier();
 		descs[0] = _mm_loadu_si128((__m128i *)(rxdp));
 
 		/* B.2 copy 2 mbuf point into rx_pkts  */
-- 
2.7.4

^ permalink raw reply related

* [PATCH 1/3] net/i40e: fix out of order rx read issue
From: Qi Zhang @ 2016-10-17  8:23 UTC (permalink / raw)
  To: jingjing.wu, helin.zhang; +Cc: dev, Qi Zhang
In-Reply-To: <1476692614-35793-1-git-send-email-qi.z.zhang@intel.com>

Volatile point has been cast to non-volatile
point when call _mm_loadu_si128, so add compile
barrier to prevent compiler reorder.

Fixes: 9ed94e5bb04e ("i40e: add vector Rx")

Signed-off-by: Qi Zhang <qi.z.zhang@intel.com>
---
 drivers/net/i40e/i40e_rxtx_vec.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/drivers/net/i40e/i40e_rxtx_vec.c b/drivers/net/i40e/i40e_rxtx_vec.c
index 0ee0241..ab63501 100644
--- a/drivers/net/i40e/i40e_rxtx_vec.c
+++ b/drivers/net/i40e/i40e_rxtx_vec.c
@@ -305,6 +305,7 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
 		/* Read desc statuses backwards to avoid race condition */
 		/* A.1 load 4 pkts desc */
 		descs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3));
+		rte_compiler_barrier();
 
 		/* B.2 copy 2 mbuf point into rx_pkts  */
 		_mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1);
@@ -313,8 +314,10 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
 		mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos+2]);
 
 		descs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2));
+		rte_compiler_barrier();
 		/* B.1 load 2 mbuf point */
 		descs[1] = _mm_loadu_si128((__m128i *)(rxdp + 1));
+		rte_compiler_barrier();
 		descs[0] = _mm_loadu_si128((__m128i *)(rxdp));
 
 		/* B.2 copy 2 mbuf point into rx_pkts  */
-- 
2.7.4

^ permalink raw reply related

* [PATCH 0/3] net: fix out of order rx read issue
From: Qi Zhang @ 2016-10-17  8:23 UTC (permalink / raw)
  To: jingjing.wu, helin.zhang; +Cc: dev, Qi Zhang

Volatile point has been cast to non-volatile
point when call _mm_loadu_si128, so add compile
barrier to prevent compiler reorder.

Qi Zhang (3):
  net/i40e: fix out of order rx read issue
  net/ixgbe: fix out of order rx read issue
  net/fm10k: fix out of order rx read issue

 drivers/net/fm10k/fm10k_rxtx_vec.c     | 3 +++
 drivers/net/i40e/i40e_rxtx_vec.c       | 3 +++
 drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 3 +++
 3 files changed, 9 insertions(+)

-- 
2.7.4

^ permalink raw reply

* [PATCH] vhost: disable indirect descriptors feature
From: Maxime Coquelin @ 2016-10-17 15:10 UTC (permalink / raw)
  To: yuanhan.liu, dev; +Cc: zhihong.wang, ciara.loftus, Maxime Coquelin

Commit 2304dd73d287 ("vhost: support indirect Tx descriptors")
adds support for indirect descriptors for Tx, but not for Rx.

The problem is that it does not work with windows guests, which
uses indirect descriptors for the Rx, and also with Linux guests
when using kernel driver with mergeable buffers feature disabled.

While indirect descriptors support is also added to the Rx path,
let's disable the feature.

Reported-by: Zhihong Wang <zhihong.wang@intel.com>
Reported-by: Ciara Loftus <ciara.loftus@intel.com>
Cc: Yuanhan Liu <yuanhan.liu@linux.intel.com>
Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 lib/librte_vhost/vhost.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index 469117a..f5f8f92 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -65,8 +65,7 @@
 				(1ULL << VIRTIO_NET_F_CSUM)    | \
 				(1ULL << VIRTIO_NET_F_GUEST_CSUM) | \
 				(1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
-				(1ULL << VIRTIO_NET_F_GUEST_TSO6) | \
-				(1ULL << VIRTIO_RING_F_INDIRECT_DESC))
+				(1ULL << VIRTIO_NET_F_GUEST_TSO6))
 
 uint64_t VHOST_FEATURES = VHOST_SUPPORTED_FEATURES;
 
-- 
2.7.4

^ permalink raw reply related

* Re: [PATCH v2 0/5] i40e: vector poll-mode driver on ARM64
From: Bruce Richardson @ 2016-10-17 14:44 UTC (permalink / raw)
  To: Jianbo Liu; +Cc: helin.zhang, jingjing.wu, jerin.jacob, dev, qi.z.zhang
In-Reply-To: <1476417604-22400-1-git-send-email-jianbo.liu@linaro.org>

On Fri, Oct 14, 2016 at 09:29:59AM +0530, Jianbo Liu wrote:
> This patch set is to implement i40e vector PMD on ARM64.
> For x86, vPMD is only reorganized, there should be no performance loss.
> 
> v1 -> v2
> - rebase to dpdk-next-net/rel_16_11
> 
> Jianbo Liu (5):
>   i40e: extract non-x86 specific code from vector driver
>   i40e: implement vector PMD for ARM architecture
>   i40e: enable i40e vector PMD on ARMv8a platform
>   i40e: make vector driver filenames consistent
>   maintainers: claim i40e vector PMD on ARM
> 
While I haven't reviewed the ARM code, I have run a very quick sanity
check on an IA platform and saw no performance regression there.

Acked-by: Bruce Richardson <bruce.richardson@intel.com>

^ permalink raw reply

* Re: [dpdk-users] Project Governance and Linux Foundation
From: O'Driscoll, Tim @ 2016-10-17 14:40 UTC (permalink / raw)
  To: Thomas Monjalon, users@dpdk.org, dev@dpdk.org; +Cc: Hobywan Kenoby
In-Reply-To: <56798548.UUDuXfq43Z@xps13>



> -----Original Message-----
> From: Thomas Monjalon [mailto:thomas.monjalon@6wind.com]
> Sent: Monday, October 17, 2016 1:41 PM
> To: users@dpdk.org; dev@dpdk.org
> Cc: O'Driscoll, Tim <tim.odriscoll@intel.com>; Hobywan Kenoby
> <hobywank@hotmail.com>
> Subject: Re: [dpdk-users] Project Governance and Linux Foundation
> 
> 2016-10-17 11:52, O'Driscoll, Tim:
> > From: Hobywan Kenoby
> > > The current DPDK version can run on virtually all processors (Intel,
> IBM
> > > and ARM) and leverage all NICs: is there **really** anyone
> questionning
> > > openness of the community?
> >
> > I still hear concerns on this, and based on discussions with others
> who
> > put their names to the post below, they do too.
> > I think it's a perception that we need to address.
> 
> It is simple to address this perception with fact checking.
> The next releases will provide even more code for ARM and NPUs.
> If someone submits some good code and is ignored, it is easy enough
> to ping the mailing list and make it visible.
> If someone sees any regression on his architecture, we care.
> Please let's stop maintaining confusion on this topic.
> 
> DPDK *is* truly open.

Well, to be a little more specific, the concern I've heard on many occasions is that 6WIND control the infrastructure for the project and so effectively have a veto over what's accepted into DPDK. Your argument is that you've never exercised that veto, which is true, but you still have the ability to do so. That's not a characteristic of a truly open project. As stated in the original post on this:

> - The infrastructure for a project like DPDK should not be owned and controlled by any single company.

^ permalink raw reply

* Re: [PATCH v2] net/mlx5: fix init on secondary process
From: Adrien Mazarguil @ 2016-10-17 14:18 UTC (permalink / raw)
  To: Ferruh Yigit; +Cc: Olivier Gournet, dev, Nelio Laranjeiro
In-Reply-To: <bba8c669-2a19-575f-41c0-25b70128cc96@intel.com>

On Mon, Oct 17, 2016 at 02:52:39PM +0100, Ferruh Yigit wrote:
> Hi Adrien,
> 
> On 10/17/2016 1:56 PM, Olivier Gournet wrote:
> > Fixes: 1d88ba171942 ("net/mlx5: refactor Tx data path")
> > Fixes: 21c8bb4928c9 ("net/mlx5: split Tx queue structure")
> > 
> > Signed-off-by: Olivier Gournet <ogournet@corp.free.fr>
> 
> According your comment on previous version of it, I think you have your
> Ack on this patch, but can you please confirm?

Yes it's fine, thanks.

Acked-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>

-- 
Adrien Mazarguil
6WIND

^ permalink raw reply

* Re: [PATCH v4] vhost: Add indirect descriptors support to the TX path
From: Maxime Coquelin @ 2016-10-17 14:14 UTC (permalink / raw)
  To: Yuanhan Liu
  Cc: Wang, Zhihong, Xie, Huawei, dev@dpdk.org, vkaplans@redhat.com,
	mst@redhat.com, stephen@networkplumber.org
In-Reply-To: <20161017132121.GG16751@yliu-dev.sh.intel.com>



On 10/17/2016 03:21 PM, Yuanhan Liu wrote:
> On Mon, Oct 17, 2016 at 01:23:23PM +0200, Maxime Coquelin wrote:
>>> On my side, I just setup 2 Windows 2016 VMs, and confirm the issue.
>>> I'll continue the investigation early next week.
>>
>> The root cause is identified.
>> When INDIRECT_DESC feature is negotiated, Windows guest uses indirect
>> for both Tx and Rx descriptors, whereas Linux guests (Virtio PMD &
>> virtio-net kernel driver) use indirect only for Tx.
>> I'll implement indirect support for the Rx path in vhost lib, but the
>> change will be too big for -rc release.
>> I propose in the mean time to disable INDIRECT_DESC feature in vhost
>> lib, we can still enable it locally for testing.
>>
>> Yuanhan, is it ok for you?
>
> That's okay.
I'll send a patch to disable it then.

>
>>
>>> Has anyone already tested Windows guest with vhost-net, which also has
>>> indirect descs support?
>>
>> I tested and confirm it works with vhost-net.
>
> I'm a bit confused then. IIRC, vhost-net also doesn't support indirect
> for Rx path, right?

No, it does support it actually.
I thought it didn't support too, I misread the Kernel implementation of
vhost-net and virtio-net. Acutally, virtio-net makes use of indirect
in Rx path when mergeable buffers is disabled.

The confusion certainly comes from me, sorry about that.

Maxime

^ permalink raw reply

* Re: [PATCH v2] net/mlx5: fix init on secondary process
From: Ferruh Yigit @ 2016-10-17 13:52 UTC (permalink / raw)
  To: Olivier Gournet, dev; +Cc: Adrien Mazarguil, Nelio Laranjeiro
In-Reply-To: <1476708993-62820-1-git-send-email-ogournet@corp.free.fr>

Hi Adrien,

On 10/17/2016 1:56 PM, Olivier Gournet wrote:
> Fixes: 1d88ba171942 ("net/mlx5: refactor Tx data path")
> Fixes: 21c8bb4928c9 ("net/mlx5: split Tx queue structure")
> 
> Signed-off-by: Olivier Gournet <ogournet@corp.free.fr>

According your comment on previous version of it, I think you have your
Ack on this patch, but can you please confirm?

Thanks,
ferruh

^ permalink raw reply

* Re: [PATCH v11 00/24] Introducing rte_driver/rte_device generalization
From: Ferruh Yigit @ 2016-10-17 13:43 UTC (permalink / raw)
  To: Shreyansh Jain, Thomas Monjalon
  Cc: dev, viktorin, David Marchand, hemant.agrawal
In-Reply-To: <de7b3b3b-7a29-f2a8-bc56-df4e4ec0717d@nxp.com>

On 10/5/2016 12:57 PM, Shreyansh Jain wrote:
> Hi Thomas,
> 
> On Tuesday 04 October 2016 01:12 PM, Thomas Monjalon wrote:
>> 2016-10-04 12:21, Shreyansh Jain:
>>> Hi Thomas,
>>>
>>> On Monday 03 October 2016 07:58 PM, Thomas Monjalon wrote:
>>>> Applied, thanks everybody for the great (re)work!
>>>
>>> Thanks!
>>>
> [...]
> [...]
>>>
>>> It can be merged with changes for:
>>>   - drv_name
>>>   - EAL_ before _REGISTER_ macros
>>>   - eth_driver => rte_driver naming
>>
>> Good.
>> Could you make it this week, please?
>>
> 
> Certainly. At least some of those I can send within this week :)
> 


I caught while running ABI validation script today, I think this patch
should increase LIBABIVER of:
- lib/librte_cryptodev
- lib/librte_eal
- lib/librte_ether

Thanks,
ferruh

^ permalink raw reply

* [RFC v3] latencystats: added new library for latency stats
From: Reshma Pattan @ 2016-10-17 13:39 UTC (permalink / raw)
  To: dev; +Cc: Reshma Pattan
In-Reply-To: <1475856608-15504-1-git-send-email-reshma.pattan@intel.com>

Library is designed to calculate latency stats and report them to the
application when queried. Library measures minimum, average, maximum
latencies and jitter in nano seconds.
Current implementation supports global latency stats, i.e. per application
stats.

Added new field to mbuf struct to mark the packet arrival time on Rx and
use the times tamp to measure the latency on Tx.

Modified dpdk-procinfo process to display the new stats.

APIs:

Added APIs to initialize and un initialize latency stats
calculation.
Added API to retrieve latency stats names and values.

Functionality:

*Library will register ethdev Rx/Tx callbacks for each active port,
queue combinations.
*Library will register latency stats names with new stats library, which is under
design for now.
*Rx packets will be marked with time stamp on each sampling interval.
*On Tx side, packets with time stamp will be considered for calculating
the minimum, maximum, average latencies and jitter.
*Average latency is calculated by summing all the latencies measured for each
time stamped packet and dividing that by total time stamped packets.
*Minimum and maximum latencies will be low and high latency values observed
so far.
*Jitter calculation is done based on inter packet delay variation.
*Measured stats can be retrieved via get API of the libray (or)
by calling generic get API of the new stats library, in this case
callback is provided to update the stats  into new stats library.

Signed-off-by: Reshma Pattan <reshma.pattan@intel.com>
---
 MAINTAINERS                                        |   4 +
 app/proc_info/main.c                               |  60 ++++
 app/test-pmd/testpmd.c                             |   9 +
 config/common_base                                 |  10 +
 lib/Makefile                                       |   1 +
 lib/librte_latencystats/Makefile                   |  57 ++++
 lib/librte_latencystats/rte_latencystats.c         | 378 +++++++++++++++++++++
 lib/librte_latencystats/rte_latencystats.h         | 141 ++++++++
 .../rte_latencystats_version.map                   |  10 +
 lib/librte_mbuf/rte_mbuf.h                         |   3 +
 mk/rte.app.mk                                      |   1 +
 11 files changed, 674 insertions(+)
 create mode 100644 lib/librte_latencystats/Makefile
 create mode 100644 lib/librte_latencystats/rte_latencystats.c
 create mode 100644 lib/librte_latencystats/rte_latencystats.h
 create mode 100644 lib/librte_latencystats/rte_latencystats_version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 8f5fa82..2db3365 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -698,3 +698,7 @@ F: examples/tep_termination/
 F: examples/vmdq/
 F: examples/vmdq_dcb/
 F: doc/guides/sample_app_ug/vmdq_dcb_forwarding.rst
+
+Latency Stats
+M: Reshma Pattan <reshma.pattan@intel.com>
+F: lib/librte_latencystats/
diff --git a/app/proc_info/main.c b/app/proc_info/main.c
index 2c56d10..56c9509 100644
--- a/app/proc_info/main.c
+++ b/app/proc_info/main.c
@@ -57,6 +57,7 @@
 #include <rte_atomic.h>
 #include <rte_branch_prediction.h>
 #include <rte_string_fns.h>
+#include <rte_stats.h>
 
 /* Maximum long option length for option parsing. */
 #define MAX_LONG_OPT_SZ 64
@@ -75,6 +76,8 @@ static uint32_t reset_xstats;
 /**< Enable memory info. */
 static uint32_t mem_info;
 
+static uint32_t enable_latnbit_stats = 1;
+
 /**< display usage */
 static void
 proc_info_usage(const char *prgname)
@@ -301,6 +304,60 @@ nic_xstats_clear(uint8_t port_id)
 	printf("\n  NIC extended statistics for port %d cleared\n", port_id);
 }
 
+static void
+latnbit_stats_display(void)
+{
+	struct rte_stat_value *lat_stats;
+	struct rte_stat_name *names;
+	int len, ret;
+	static const char *nic_stats_border = "########################";
+
+	memset(&lat_stats, 0, sizeof(struct rte_stat_value));
+	len = rte_stats_get_names(NULL, 0);
+	if (len < 0) {
+		printf("Cannot get latency and bitrate stats count\n");
+		return;
+	}
+
+	lat_stats = malloc(sizeof(struct rte_stat_value) * len);
+	if (lat_stats == NULL) {
+		printf("Cannot allocate memory for latency and bitrate stats\n");
+		return;
+	}
+
+	names =  malloc(sizeof(struct rte_stat_name) * len);
+	if (names == NULL) {
+		printf("Cannot allocate memory for latency and bitrate stats names\n");
+		free(lat_stats);
+		return;
+	}
+
+	if (len != rte_stats_get_names(names, len)) {
+		printf("Cannot get latency and bitrate stats names\n");
+		free(lat_stats);
+		free(names);
+		return;
+	}
+
+	printf("###### Latency and bitrate statistics #########\n");
+	printf("%s############################\n", nic_stats_border);
+	ret = rte_stats_get_values(RTE_STATS_NONPORT, lat_stats, len);
+	if (ret < 0 || ret > len) {
+		printf("Cannot get latency and bitrate stats\n");
+		free(lat_stats);
+		free(names);
+		return;
+	}
+
+	int i;
+	for (i = 0; i < len; i++)
+		printf("%s: %"PRIu64"\n", names[i].name, lat_stats[i].value);
+
+	printf("%s############################\n", nic_stats_border);
+	free(lat_stats);
+	free(names);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -363,5 +420,8 @@ main(int argc, char **argv)
 		}
 	}
 
+	if (enable_latnbit_stats)
+		latnbit_stats_display();
+
 	return 0;
 }
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index 6185be6..09dea69 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -78,6 +78,9 @@
 #ifdef RTE_LIBRTE_PDUMP
 #include <rte_pdump.h>
 #endif
+#ifdef RTE_LIBRTE_LATENCY_STATS
+#include <rte_latencystats.h>
+#endif
 
 #include "testpmd.h"
 
@@ -2070,6 +2073,9 @@ signal_handler(int signum)
 		/* uninitialize packet capture framework */
 		rte_pdump_uninit();
 #endif
+#ifdef RTE_LIBRTE_LATENCY_STATS
+		rte_latencystats_uninit();
+#endif
 		force_quit();
 		/* exit with the expected status */
 		signal(signum, SIG_DFL);
@@ -2128,6 +2134,9 @@ main(int argc, char** argv)
 	FOREACH_PORT(port_id, ports)
 		rte_eth_promiscuous_enable(port_id);
 
+#ifdef RTE_LIBRTE_LATENCY_STATS
+	rte_latencystats_init(1, NULL);
+#endif
 #ifdef RTE_LIBRTE_CMDLINE
 	if (interactive == 1) {
 		if (auto_start) {
diff --git a/config/common_base b/config/common_base
index 81b8e1e..9584727 100644
--- a/config/common_base
+++ b/config/common_base
@@ -557,9 +557,19 @@ CONFIG_RTE_LIBRTE_PDUMP=y
 #
 # Compile vhost user library
 #
+# Compile vhost library
+# fuse-devel is needed to run vhost-cuse.
+# fuse-devel enables user space char driver development
+# vhost-user is turned on by default.
+#
 CONFIG_RTE_LIBRTE_VHOST=n
 CONFIG_RTE_LIBRTE_VHOST_NUMA=n
 CONFIG_RTE_LIBRTE_VHOST_DEBUG=n
+#
+# Compile the latency stats library
+#
+CONFIG_RTE_LIBRTE_LATENCY_STATS=y
+
 
 #
 # Compile vhost PMD
diff --git a/lib/Makefile b/lib/Makefile
index cbaeefa..7af7e1c 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -59,6 +59,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += librte_pipeline
 DIRS-$(CONFIG_RTE_LIBRTE_REORDER) += librte_reorder
 DIRS-$(CONFIG_RTE_LIBRTE_PDUMP) += librte_pdump
 DIRS-$(CONFIG_RTE_LIBRTE_STATS) += librte_stats
+DIRS-$(CONFIG_RTE_LIBRTE_LATENCY_STATS) += librte_latencystats
 
 ifeq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 DIRS-$(CONFIG_RTE_LIBRTE_KNI) += librte_kni
diff --git a/lib/librte_latencystats/Makefile b/lib/librte_latencystats/Makefile
new file mode 100644
index 0000000..2dd0381
--- /dev/null
+++ b/lib/librte_latencystats/Makefile
@@ -0,0 +1,57 @@
+#   BSD LICENSE
+#
+#   Copyright(c) 2016 Intel Corporation. All rights reserved.
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Intel Corporation nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_latencystats.a
+
+CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3
+LDLIBS += -lm
+LDLIBS += -lpthread
+
+EXPORT_MAP := rte_latencystats_version.map
+
+LIBABIVER := 1
+
+# all source are stored in SRCS-y
+SRCS-$(CONFIG_RTE_LIBRTE_LATENCY_STATS) := rte_latencystats.c
+
+# install this header file
+SYMLINK-$(CONFIG_RTE_LIBRTE_LATENCY_STATS)-include := rte_latencystats.h
+
+# this lib depends upon:
+DEPDIRS-$(CONFIG_RTE_LIBRTE_LATENCY_STATS) += lib/librte_mbuf
+DEPDIRS-$(CONFIG_RTE_LIBRTE_LATENCY_STATS) += lib/librte_eal
+DEPDIRS-$(CONFIG_RTE_LIBRTE_LATENCY_STATS) += lib/librte_ether
+DEPDIRS-$(CONFIG_RTE_LIBRTE_LATENCY_STATS) += lib/librte_stats
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/lib/librte_latencystats/rte_latencystats.c b/lib/librte_latencystats/rte_latencystats.c
new file mode 100644
index 0000000..3d8b053
--- /dev/null
+++ b/lib/librte_latencystats/rte_latencystats.c
@@ -0,0 +1,378 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <unistd.h>
+#include <sys/types.h>
+#include <stdbool.h>
+#include <math.h>
+#include <pthread.h>
+
+#include <rte_mbuf.h>
+#include <rte_log.h>
+#include <rte_cycles.h>
+#include <rte_ethdev.h>
+#include <rte_stats.h>
+#include <rte_memzone.h>
+#include <rte_lcore.h>
+#include <rte_timer.h>
+
+#include "rte_latencystats.h"
+
+/** Nano seconds per second */
+#define NS_PER_SEC 1E9
+
+/** Clock cycles per nano second */
+#define CYCLES_PER_NS (rte_get_timer_hz() / NS_PER_SEC)
+
+/* Macros for printing using RTE_LOG */
+#define RTE_LOGTYPE_LATENCY_STATS RTE_LOGTYPE_USER1
+
+static pthread_t latency_stats_thread;
+static const char *MZ_RTE_LATENCY_STATS = "rte_latencystats";
+static int latency_stats_index;
+static uint64_t sampIntvl;
+static uint64_t timer_tsc;
+static uint64_t prev_tsc;
+
+struct rte_latency_stats {
+	float min_latency; /**< Minimum latency in nano seconds */
+	float avg_latency; /**< Average latency in nano seconds */
+	float max_latency; /**< Maximum latency in nano seconds */
+	float jitter; /** Latency variation between to packets */
+	uint64_t total_sampl_pkts;
+} *glob_stats;
+
+static struct rxtx_cbs {
+	struct rte_eth_rxtx_callback *cb;
+} rx_cbs[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT],
+	tx_cbs[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];
+
+struct latency_stats_nameoff {
+	char name[RTE_ETH_XSTATS_NAME_SIZE];
+	unsigned int offset;
+};
+
+static const struct latency_stats_nameoff lat_stats_strings[] = {
+	{"min_latency_ns", offsetof(struct rte_latency_stats, min_latency)},
+	{"avg_latency_ns", offsetof(struct rte_latency_stats, avg_latency)},
+	{"max_latency_ns", offsetof(struct rte_latency_stats, max_latency)},
+	{"jitter_ns", offsetof(struct rte_latency_stats, jitter)},
+};
+
+#define NUM_LATENCY_STATS (sizeof(lat_stats_strings) / \
+				sizeof(lat_stats_strings[0]))
+
+static __attribute__((noreturn)) void *
+report_latency_stats(__rte_unused void *arg)
+{
+	for (;;) {
+		unsigned int i;
+		float *stats_ptr = NULL;
+		uint64_t values[NUM_LATENCY_STATS] = {0};
+		int ret;
+
+		for (i = 0; i < NUM_LATENCY_STATS; i++) {
+			stats_ptr = RTE_PTR_ADD(glob_stats,
+					lat_stats_strings[i].offset);
+			if (i == 1)
+				values[i] = (uint64_t)floor((*stats_ptr)/
+				(CYCLES_PER_NS * glob_stats->total_sampl_pkts));
+			else
+				values[i] = (uint64_t)floor((*stats_ptr)/
+						CYCLES_PER_NS);
+		}
+
+		ret = rte_stats_update_metrics(RTE_STATS_NONPORT, latency_stats_index,
+				values, NUM_LATENCY_STATS);
+		if (ret < 0)
+			RTE_LOG(INFO, LATENCY_STATS, "Failed to push the stats\n");
+	}
+}
+
+static void
+rte_latencystats_fill_values(struct rte_stat_value *values)
+{
+	unsigned int i;
+	float *stats_ptr = NULL;
+
+	for (i = 0; i < NUM_LATENCY_STATS; i++) {
+		stats_ptr = RTE_PTR_ADD(glob_stats,
+				lat_stats_strings[i].offset);
+		values[i].key = i;
+		if (i == 1)
+			values[i].value = (uint64_t)floor((*stats_ptr)/
+			(CYCLES_PER_NS * glob_stats->total_sampl_pkts));
+		else
+			values[i].value = (uint64_t)floor((*stats_ptr)/
+						CYCLES_PER_NS);
+	}
+}
+
+static uint16_t
+add_time_stamps(uint8_t pid __rte_unused,
+		uint16_t qid __rte_unused,
+		struct rte_mbuf **pkts,
+		uint16_t nb_pkts,
+		uint16_t max_pkts __rte_unused,
+		void *user_cb __rte_unused)
+{
+	unsigned int i;
+	uint64_t diff_tsc, now;
+
+	/* for every sample interval,
+	 * time stamp is marked on one received packet
+	 */
+	now = rte_rdtsc();
+	for (i = 0; i < nb_pkts; i++) {
+		diff_tsc = now - prev_tsc;
+		timer_tsc += diff_tsc;
+		if (timer_tsc >= sampIntvl) {
+			pkts[i]->time_arrived = now;
+			timer_tsc = 0;
+		}
+		prev_tsc = now;
+		now = rte_rdtsc();
+	}
+
+	return nb_pkts;
+}
+
+static uint16_t
+calc_latency(uint8_t pid __rte_unused,
+		uint16_t qid __rte_unused,
+		struct rte_mbuf **pkts,
+		uint16_t nb_pkts,
+		void *_ __rte_unused)
+{
+	unsigned int i, cnt = 0;
+	uint64_t now;
+	float latency[nb_pkts];
+	static float prev_latency;
+
+	now = rte_rdtsc();
+	for (i = 0; i < nb_pkts; i++) {
+		if (pkts[i]->time_arrived)
+			latency[cnt++] = now - pkts[i]->time_arrived;
+	}
+
+	for (i = 0; i < cnt; i++)
+	{
+		/**
+		*The jitter is calculated as statistical mean of interpacket
+		*delay variation. The "jitter estimate" is computed by taking the
+		*absolute values of the ipdv sequence and applying an exponential
+		*filter with parameter 1/16 to generate the estimate. i.e
+		*J=J+(|D(i-1,i)|-J)/16. Where J is jitter, D(i-1,i) is difference in
+		*latency of two consecutive packets i-1 and i.
+		*Reference: Calculated as per RFC 5481, sec 4.1, RFC 3393 sec 4.5,
+		*RFC 1889 sec.
+		*/
+		glob_stats->jitter +=  (abs(prev_latency - latency[i])
+					- glob_stats->jitter)/16;
+		if (glob_stats->min_latency == 0)
+			glob_stats->min_latency = latency[i];
+		else if (latency[i] < glob_stats->min_latency)
+			glob_stats->min_latency = latency[i];
+		else if (latency[i] > glob_stats->max_latency)
+			glob_stats->max_latency = latency[i];
+		glob_stats->avg_latency += latency[i]/16;
+		glob_stats->total_sampl_pkts++;
+		prev_latency = latency[i];
+	}
+
+	return nb_pkts;
+}
+
+int
+rte_latencystats_init(uint64_t samp_intvl,
+		rte_latency_stats_flow_type_fn user_cb)
+{
+	unsigned int i;
+	uint8_t pid;
+	uint16_t qid;
+	struct rxtx_cbs *cbs = NULL;
+	const uint8_t nb_ports = rte_eth_dev_count();
+	const char *ptr_strings[NUM_LATENCY_STATS] = {0};
+	const struct rte_memzone *mz = NULL;
+	const unsigned int flags = 0;
+
+	/** Allocate stats in shared memory fo muliti process support */
+	mz = rte_memzone_reserve(MZ_RTE_LATENCY_STATS, sizeof(*glob_stats),
+					rte_socket_id(), flags);
+	if (mz == NULL) {
+		RTE_LOG(ERR, LATENCY_STATS, "Cannot reserve memory: %s:%d\n",
+			__func__, __LINE__);
+		return -ENOMEM;
+	}
+
+	glob_stats = mz->addr;
+	samp_intvl *= CYCLES_PER_NS;
+
+	/** Register latency stats with stats library */
+	for (i = 0; i < NUM_LATENCY_STATS; i++)
+		ptr_strings[i] = lat_stats_strings[i].name;
+
+	latency_stats_index = rte_stats_reg_metrics(ptr_strings,
+							NUM_LATENCY_STATS);
+	if (latency_stats_index < 0) {
+		RTE_LOG(DEBUG, LATENCY_STATS,
+			"Failed to register latency stats names\n");
+		return -1;
+	}
+
+	/** Register Rx/Tx callbacks */
+	for (pid = 0; pid < nb_ports; pid++) {
+		struct rte_eth_dev_info dev_info;
+		rte_eth_dev_info_get(pid, &dev_info);
+		for (qid = 0; qid < dev_info.nb_rx_queues; qid++) {
+			cbs = &rx_cbs[pid][qid];
+			cbs->cb = rte_eth_add_first_rx_callback(pid, qid,
+					add_time_stamps, user_cb);
+			if (!cbs->cb)
+				RTE_LOG(INFO, LATENCY_STATS, "Failed to "
+					"register Rx callback for pid=%d, "
+					"qid=%d\n", pid, qid);
+		}
+		for (qid = 0; qid < dev_info.nb_tx_queues; qid++) {
+			cbs = &tx_cbs[pid][qid];
+			cbs->cb =  rte_eth_add_tx_callback(pid, qid,
+					calc_latency, user_cb);
+			if (!cbs->cb)
+				RTE_LOG(INFO, LATENCY_STATS, "Failed to "
+					"register Tx callback for pid=%d, "
+					"qid=%d\n", pid, qid);
+		}
+	}
+
+	int ret = 0;
+	char thread_name[RTE_MAX_THREAD_NAME_LEN];
+
+	/** Create the host thread to update latency stats to stats library */
+	ret = pthread_create(&latency_stats_thread, NULL, report_latency_stats, NULL);
+	if (ret != 0) {
+		RTE_LOG(ERR, LATENCY_STATS,
+			"Failed to create the latency stats thread:%s, %s:%d\n",
+			strerror(errno), __func__, __LINE__);
+		return -1;
+	}
+	/** Set thread_name for aid in debugging */
+	snprintf(thread_name, RTE_MAX_THREAD_NAME_LEN, "latency-stats-thread");
+	ret = rte_thread_setname(latency_stats_thread, thread_name);
+	if (ret != 0)
+		RTE_LOG(DEBUG, LATENCY_STATS,
+			"Failed to set thread name for latency stats handling\n");
+
+	return 0;
+}
+
+int
+rte_latencystats_uninit(void)
+{
+	uint8_t pid;
+	uint16_t qid;
+	int ret = 0;
+	struct rxtx_cbs *cbs = NULL;
+	const uint8_t nb_ports = rte_eth_dev_count();
+
+	/** De register Rx/Tx callbacks */
+	for (pid = 0; pid < nb_ports; pid++) {
+		struct rte_eth_dev_info dev_info;
+		rte_eth_dev_info_get(pid, &dev_info);
+		for (qid = 0; qid < dev_info.nb_rx_queues; qid++) {
+			cbs = &rx_cbs[pid][qid];
+			ret = rte_eth_remove_rx_callback(pid, qid, cbs->cb);
+			if (ret)
+				RTE_LOG(INFO, LATENCY_STATS, "failed to "
+					"remove Rx callback for pid=%d, "
+					"qid=%d\n", pid, qid);
+		}
+		for (qid = 0; qid < dev_info.nb_tx_queues; qid++) {
+			cbs = &tx_cbs[pid][qid];
+			ret = rte_eth_remove_tx_callback(pid, qid, cbs->cb);
+			if (ret)
+				RTE_LOG(INFO, LATENCY_STATS, "failed to "
+					"remove Tx callback for pid=%d, "
+					"qid=%d\n", pid, qid);
+		}
+	}
+
+	/** Cancel the thread */
+	ret = pthread_cancel(latency_stats_thread);
+	if (ret != 0) {
+		RTE_LOG(ERR, LATENCY_STATS,
+			"Failed to cancel latency stats update thread:"
+			"%s,%s:%d\n",
+			strerror(errno), __func__, __LINE__);
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+rte_latencystats_get_names(struct rte_stat_name *names, uint16_t size)
+{
+	unsigned int i;
+
+	if (names == NULL || size < NUM_LATENCY_STATS)
+		return NUM_LATENCY_STATS;
+
+	for (i = 0; i < NUM_LATENCY_STATS; i++)
+		snprintf(names[i].name, sizeof(names[i].name),
+				"%s", lat_stats_strings[i].name);
+
+	return NUM_LATENCY_STATS;
+}
+
+int
+rte_latencystats_get(struct rte_stat_value *values, uint16_t size)
+{
+	if (size < NUM_LATENCY_STATS || values == NULL)
+		return NUM_LATENCY_STATS;
+
+	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
+		const struct rte_memzone *mz;
+		mz = rte_memzone_lookup(MZ_RTE_LATENCY_STATS);
+		if (mz == NULL) {
+			RTE_LOG(ERR, LATENCY_STATS,
+				"Latency stats memzone not found\n");
+			return -ENOMEM;
+		}
+		glob_stats =  mz->addr;
+	}
+
+	/* Retrieve latency stats */
+	rte_latencystats_fill_values(values);
+
+	return NUM_LATENCY_STATS;
+}
diff --git a/lib/librte_latencystats/rte_latencystats.h b/lib/librte_latencystats/rte_latencystats.h
new file mode 100644
index 0000000..9384c4a
--- /dev/null
+++ b/lib/librte_latencystats/rte_latencystats.h
@@ -0,0 +1,141 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_LATENCYSTATS_H_
+#define _RTE_LATENCYSTATS_H_
+
+/**
+ * @file
+ * RTE latency stats
+ *
+ * library to provide application and flow based latency stats.
+ */
+
+#include <rte_stats.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Function type used for identifting flow types of a Rx packet.
+ *
+ * The callback function is called on Rx for each packet.
+ * This function is used for flow based latency calculations.
+ *
+ * @param pkt
+ *   Packet that has to be identified with its flow types.
+ * @param user_param
+ *   The arbitrary user parameter passed in by the application when
+ *   the callback was originally configured.
+ * @return
+ *   The flow_mask, representing the multiple flow types of a packet.
+ */
+typedef uint16_t (*rte_latency_stats_flow_type_fn)(struct rte_mbuf *pkt,
+							void *user_param);
+
+/**
+ * @internal
+ *  Registers Rx/Tx callbacks for each active port, queue.
+ *
+ * @param sampIntvl
+ *  Sampling time period in nano seconds, at which packet
+ *  should be marked with time stamp.
+ * @param user_cb
+ *  User callback to be called to get flow types of a packet.
+ *  Used for flow based latency calculation.
+ *  If the value is NULL, global stats will be calculated,
+ *  else flow based stats will be calculated.
+ *  @return
+ *   -1     : On error
+ *   -ENOMEM: On error
+ *    0     : On sucess
+ */
+int rte_latencystats_init(uint64_t samp_intvl,
+			rte_latency_stats_flow_type_fn user_cb);
+
+/**
+ * @internal
+ *  Removes registered Rx/Tx callbacks for each active port, queue.
+ *  @return
+ *   -1: On error
+ *    0: On suces
+ */
+int rte_latencystats_uninit(void);
+
+/**
+ * Retrieve names of latency statistics
+ *
+ * @param names
+ *  Block of memory to insert names into. Must be at least size in capacity.
+ *  If set to NULL, function returns required capacity.
+ * @param size
+ *  Capacity of latency stats names (number of names).
+ * @return
+ *   - positive value lower or equal to size: success. The return value
+ *     is the number of entries filled in the stats table.
+ *   - positive value higher than size: error, the given statistics table
+ *     is too small. The return value corresponds to the size that should
+ *     be given to succeed. The entries in the table are not valid and
+ *     shall not be used by the caller.
+ */
+int rte_latencystats_get_names(struct rte_stat_name *names,
+				uint16_t size);
+
+/**
+ * Retrieve latency statistics.
+ *
+ * @param values
+ *   A pointer to a table of structure of type *rte_stat_value*
+ *   to be filled with latency statistics ids and values.
+ *   This parameter can be set to NULL if size is 0.
+ * @param size
+ *   The size of the stats table, which should be large enough to store
+ *   all the latency stats.
+ * @return
+ *   - positive value lower or equal to size: success. The return value
+ *     is the number of entries filled in the stats table.
+ *   - positive value higher than size: error, the given statistics table
+ *     is too small. The return value corresponds to the size that should
+ *     be given to succeed. The entries in the table are not valid and
+ *     shall not be used by the caller.
+ *   -ENOMEM: On failure.
+ */
+int rte_latencystats_get(struct rte_stat_value *values,
+			uint16_t size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_LATENCYSTATS_H_ */
diff --git a/lib/librte_latencystats/rte_latencystats_version.map b/lib/librte_latencystats/rte_latencystats_version.map
new file mode 100644
index 0000000..82dc5a7
--- /dev/null
+++ b/lib/librte_latencystats/rte_latencystats_version.map
@@ -0,0 +1,10 @@
+DPDK_16.11 {
+	global:
+
+	rte_latencystats_get;
+	rte_latencystats_get_names;
+	rte_latencystats_init;
+	rte_latencystats_uninit;
+
+	local: *;
+};
diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h
index 109e666..4933bfc 100644
--- a/lib/librte_mbuf/rte_mbuf.h
+++ b/lib/librte_mbuf/rte_mbuf.h
@@ -493,6 +493,9 @@ struct rte_mbuf {
 
 	/** Timesync flags for use with IEEE1588. */
 	uint16_t timesync;
+
+	/** Timestamp for measuring latency stats. */
+	uint64_t time_arrived;
 } __rte_cache_aligned;
 
 /**
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 07ea98f..8c7d3d8 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -69,6 +69,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_TABLE)          += -lrte_table
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PORT)           += -lrte_port
 
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PDUMP)          += -lrte_pdump
+_LDLIBS-$(CONFIG_RTE_LIBRTE_LATENCY_STATS)   += -lrte_latencystats
 _LDLIBS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR)    += -lrte_distributor
 _LDLIBS-$(CONFIG_RTE_LIBRTE_REORDER)        += -lrte_reorder
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IP_FRAG)        += -lrte_ip_frag
-- 
2.7.4

^ permalink raw reply related

* Re: [PATCH v4] vhost: Add indirect descriptors support to the TX path
From: Yuanhan Liu @ 2016-10-17 13:21 UTC (permalink / raw)
  To: Maxime Coquelin
  Cc: Wang, Zhihong, Xie, Huawei, dev@dpdk.org, vkaplans@redhat.com,
	mst@redhat.com, stephen@networkplumber.org
In-Reply-To: <f1129452-af2e-b8ae-600c-492182bec06d@redhat.com>

On Mon, Oct 17, 2016 at 01:23:23PM +0200, Maxime Coquelin wrote:
> >On my side, I just setup 2 Windows 2016 VMs, and confirm the issue.
> >I'll continue the investigation early next week.
> 
> The root cause is identified.
> When INDIRECT_DESC feature is negotiated, Windows guest uses indirect
> for both Tx and Rx descriptors, whereas Linux guests (Virtio PMD &
> virtio-net kernel driver) use indirect only for Tx.
> 
> I'll implement indirect support for the Rx path in vhost lib, but the
> change will be too big for -rc release.
> I propose in the mean time to disable INDIRECT_DESC feature in vhost
> lib, we can still enable it locally for testing.
> 
> Yuanhan, is it ok for you?

That's okay.

> 
> >Has anyone already tested Windows guest with vhost-net, which also has
> >indirect descs support?
> 
> I tested and confirm it works with vhost-net.

I'm a bit confused then. IIRC, vhost-net also doesn't support indirect
for Rx path, right?

	--yliu

^ permalink raw reply

* Re: [PATCH 1/3] eal/drivers: prefix driver REGISTER macros with EAL
From: Neil Horman @ 2016-10-17 13:09 UTC (permalink / raw)
  To: Shreyansh Jain; +Cc: Thomas Monjalon, david.marchand@6wind.com, dev@dpdk.org
In-Reply-To: <DB5PR0401MB205420FE835BE96E45D7838390D90@DB5PR0401MB2054.eurprd04.prod.outlook.com>

On Sat, Oct 08, 2016 at 12:47:59PM +0000, Shreyansh Jain wrote:
> 
> 
> > -----Original Message-----
> > From: Thomas Monjalon [mailto:thomas.monjalon@6wind.com]
> > Sent: Friday, October 07, 2016 7:15 PM
> > To: Shreyansh Jain <shreyansh.jain@nxp.com>
> > Cc: david.marchand@6wind.com; dev@dpdk.org
> > Subject: Re: [PATCH 1/3] eal/drivers: prefix driver REGISTER macros with EAL
> > 
> > 2016-10-07 19:03, Shreyansh Jain:
> > > DRIVER_REGISTER_PCI -> EAL_REGISTER_PCI
> > > DRIVER_REGISTER_PCI_TABLE -> EAL_REGISTER_PCI_TABLE
> > 
> > Why not RTE_ prefix instead of EAL_?
> 
> Because while searching for similar naming pattern I came across other macros like EAL_REGISTER_TAILQ which were within librte_eal/eal/* folders.
> 
This is why we were suggesting to make the regex in the makefile more specific,
to avoid matching on the TAILQ macro.

> > Why 3 patches? As there is no specific comment in each, I think you can
> > squash.
> 
> Because each of the patch are common and tightly related changes. The replacement touches a large number of files, even though the change itself is small. By splitting, it makes review (or minor misses) easier - at least for me.
> Anyways, if as a maintainer you prefer having a single bulk, I have no issues.
> 
> I will send next version with RTE_* and the 'grep -E' suggestion from Neil.
> 
> -
> Shreyansh
> 

^ permalink raw reply


This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox