[PATCH v3 next 3/7] net/vmxnet3: add support for large passthrough BAR register

[Ronak Doshi <doshir@vmware.com>]

For vmxnet3 to work in UPT mode, the BAR sizes have been increased.
The PT page has been extended to 2 pages and also includes OOB pages
as a part of PT BAR. This patch enhances vmxnet3 to use appropriate
BAR offsets based on the capability registered. To use new offsets,
VMXNET3_CAP_LARGE_BAR needs to be set by the device. If it is not set
then the device will use legacy PT page layout.

Signed-off-by: Ronak Doshi <doshir@vmware.com>
Acked-by: Jochen Behrens <jbehrens@vmware.com>
---
 drivers/net/vmxnet3/base/vmxnet3_defs.h | 13 +++++++++++--
 drivers/net/vmxnet3/vmxnet3_ethdev.c    | 11 +++++++++++
 drivers/net/vmxnet3/vmxnet3_ethdev.h    |  2 ++
 drivers/net/vmxnet3/vmxnet3_rxtx.c      | 13 +++++++------
 4 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/drivers/net/vmxnet3/base/vmxnet3_defs.h b/drivers/net/vmxnet3/base/vmxnet3_defs.h
index 759fdb6e4a..27f35a0062 100644
--- a/drivers/net/vmxnet3/base/vmxnet3_defs.h
+++ b/drivers/net/vmxnet3/base/vmxnet3_defs.h
@@ -43,8 +43,16 @@
 #define VMXNET3_REG_RXPROD   0x800 /* Rx Producer Index for ring 1 */
 #define VMXNET3_REG_RXPROD2  0xA00 /* Rx Producer Index for ring 2 */
 
-#define VMXNET3_PT_REG_SIZE     4096    /* BAR 0 */
-#define VMXNET3_VD_REG_SIZE     4096    /* BAR 1 */
+/* For Large PT BAR, the following offset to DB register */
+#define VMXNET3_REG_LB_TXPROD   0x1000 /* Tx Producer Index */
+#define VMXNET3_REG_LB_RXPROD   0x1400 /* Rx Producer Index for ring 1 */
+#define VMXNET3_REG_LB_RXPROD2  0x1800 /* Rx Producer Index for ring 2 */
+
+#define VMXNET3_PT_REG_SIZE         4096           /* BAR 0 */
+#define VMXNET3_LARGE_PT_REG_SIZE   8192           /* large PT pages */
+#define VMXNET3_VD_REG_SIZE         4096           /* BAR 1 */
+#define VMXNET3_LARGE_BAR0_REG_SIZE (4096 * 4096)  /* LARGE BAR 0 */
+#define VMXNET3_OOB_REG_SIZE        (4094 * 4096)  /* OOB pages */
 
 /*
  * The two Vmxnet3 MMIO Register PCI BARs (BAR 0 at offset 10h and BAR 1 at
@@ -56,6 +64,7 @@
  * VMXNET3_MSIX_BAR_SIZE is defined in "vmxnet3Int.h"
  */
 #define VMXNET3_PHYSMEM_PAGES   4
+#define VMXNET3_PHYSMEM_LB_PAGES 4099 /* 4096 + 1 + 2 */
 
 #define VMXNET3_REG_ALIGN       8  /* All registers are 8-byte aligned. */
 #define VMXNET3_REG_ALIGN_MASK  0x7
diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.c b/drivers/net/vmxnet3/vmxnet3_ethdev.c
index 05b6d54cb9..62c7e048f5 100644
--- a/drivers/net/vmxnet3/vmxnet3_ethdev.c
+++ b/drivers/net/vmxnet3/vmxnet3_ethdev.c
@@ -456,6 +456,17 @@ eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev)
 		hw->uptv2_enabled = TRUE;
 		eth_vmxnet3_setup_capabilities(hw, eth_dev);
 	}
+
+	if (hw->used_DCR_capabilities[0] & (1 << VMXNET3_CAP_LARGE_BAR)) {
+		hw->tx_prod_offset = VMXNET3_REG_LB_TXPROD;
+		hw->rx_prod_offset[0] = VMXNET3_REG_LB_RXPROD;
+		hw->rx_prod_offset[1] = VMXNET3_REG_LB_RXPROD2;
+	} else {
+		hw->tx_prod_offset = VMXNET3_REG_TXPROD;
+		hw->rx_prod_offset[0] = VMXNET3_REG_RXPROD;
+		hw->rx_prod_offset[1] = VMXNET3_REG_RXPROD2;
+	}
+
 	/* Getting MAC Address */
 	mac_lo = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACL);
 	mac_hi = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACH);
diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.h b/drivers/net/vmxnet3/vmxnet3_ethdev.h
index 1bbf2b4465..cabd83e7e1 100644
--- a/drivers/net/vmxnet3/vmxnet3_ethdev.h
+++ b/drivers/net/vmxnet3/vmxnet3_ethdev.h
@@ -122,6 +122,8 @@ struct vmxnet3_hw {
 	UPT1_RxStats	      saved_rx_stats[VMXNET3_EXT_MAX_RX_QUEUES];
 	UPT1_TxStats          snapshot_tx_stats[VMXNET3_MAX_TX_QUEUES];
 	UPT1_RxStats          snapshot_rx_stats[VMXNET3_MAX_RX_QUEUES];
+	uint16_t              tx_prod_offset;
+	uint16_t              rx_prod_offset[2];
 	/* device capability bit map */
 	uint32_t	      DCR_capabilities[8];
 	/* pass-through capability bit map */
diff --git a/drivers/net/vmxnet3/vmxnet3_rxtx.c b/drivers/net/vmxnet3/vmxnet3_rxtx.c
index a875ffec07..83daac02c4 100644
--- a/drivers/net/vmxnet3/vmxnet3_rxtx.c
+++ b/drivers/net/vmxnet3/vmxnet3_rxtx.c
@@ -57,8 +57,6 @@
 #define	VMXNET3_TX_OFFLOAD_NOTSUP_MASK	\
 	(RTE_MBUF_F_TX_OFFLOAD_MASK ^ VMXNET3_TX_OFFLOAD_MASK)
 
-static const uint32_t rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2};
-
 static int vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t*, uint8_t);
 static void vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *);
 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
@@ -577,7 +575,7 @@ vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	if (deferred >= rte_le_to_cpu_32(txq_ctrl->txThreshold)) {
 		txq_ctrl->txNumDeferred = 0;
 		/* Notify vSwitch that packets are available. */
-		VMXNET3_WRITE_BAR0_REG(hw, (VMXNET3_REG_TXPROD + txq->queue_id * VMXNET3_REG_ALIGN),
+		VMXNET3_WRITE_BAR0_REG(hw, (hw->tx_prod_offset + txq->queue_id * VMXNET3_REG_ALIGN),
 				       txq->cmd_ring.next2fill);
 	}
 
@@ -1000,7 +998,8 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		/* It's time to renew descriptors */
 		vmxnet3_renew_desc(rxq, ring_idx, newm);
 		if (unlikely(rxq->shared->ctrl.updateRxProd)) {
-			VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
+			VMXNET3_WRITE_BAR0_REG(hw, hw->rx_prod_offset[ring_idx] +
+					       (rxq->queue_id * VMXNET3_REG_ALIGN),
 					       rxq->cmd_ring[ring_idx].next2fill);
 		}
 
@@ -1027,7 +1026,8 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		}
 		if (unlikely(rxq->shared->ctrl.updateRxProd)) {
 			for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) {
-				VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
+				VMXNET3_WRITE_BAR0_REG(hw, hw->rx_prod_offset[ring_idx] +
+						       (rxq->queue_id * VMXNET3_REG_ALIGN),
 						       rxq->cmd_ring[ring_idx].next2fill);
 			}
 		}
@@ -1322,7 +1322,8 @@ vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev)
 			 * mbufs for coming packets.
 			 */
 			if (unlikely(rxq->shared->ctrl.updateRxProd)) {
-				VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[j] + (rxq->queue_id * VMXNET3_REG_ALIGN),
+				VMXNET3_WRITE_BAR0_REG(hw, hw->rx_prod_offset[j] +
+						       (rxq->queue_id * VMXNET3_REG_ALIGN),
 						       rxq->cmd_ring[j].next2fill);
 			}
 		}
-- 
2.11.0