* [PATCH v0 1/3] common/cnxk: add O20 DPI DMA support
2026-08-29 10:54 [PATCH v0 0/3] dma/cnxk: add CN20K (O20) DPI DMA and inter-process domain support Vamsi Krishna
@ 2026-08-29 10:54 ` Vamsi Krishna
2026-08-29 10:54 ` [PATCH v0 2/3] dma/cnxk: " Vamsi Krishna
2026-08-29 10:54 ` [PATCH v0 3/3] dma/cnxk: enable inter process domain " Vamsi Krishna
2 siblings, 0 replies; 4+ messages in thread
From: Vamsi Krishna @ 2026-08-29 10:54 UTC (permalink / raw)
To: dev, vattunuru; +Cc: jerinj, fengchengwen, ndabilpuram
From: Vamsi Attunuru <vattunuru@marvell.com>
Adds ROC support for O20 DPI DMA hardware.
Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>
---
drivers/common/cnxk/hw/dpi.h | 72 +-
drivers/common/cnxk/hw/rvu.h | 2 +
drivers/common/cnxk/roc_constants.h | 2 +
drivers/common/cnxk/roc_dev.c | 1 +
drivers/common/cnxk/roc_dpi.c | 683 +++++++++++++++++-
drivers/common/cnxk/roc_dpi.h | 85 +++
drivers/common/cnxk/roc_dpi_priv.h | 34 +
drivers/common/cnxk/roc_mbox.h | 174 ++++-
drivers/common/cnxk/roc_platform.h | 8 +
.../common/cnxk/roc_platform_base_symbols.c | 18 +
10 files changed, 1065 insertions(+), 14 deletions(-)
diff --git a/drivers/common/cnxk/hw/dpi.h b/drivers/common/cnxk/hw/dpi.h
index a34713dde6..8d5a5bd0e1 100644
--- a/drivers/common/cnxk/hw/dpi.h
+++ b/drivers/common/cnxk/hw/dpi.h
@@ -23,6 +23,27 @@
#define DPI_VF_INT_ENA_W1C (0x110)
#define DPI_VF_INT_ENA_W1S (0x118)
+/* DPI CN20K LF register offsets from VF_BAR2 */
+#define DPI_LF_CTL (0ull)
+#define DPI_LF_RINGX_CFG(x) ((0x20ull | (uint64_t)(x) << 3))
+#define DPI_LF_RINGX_BASE(x) ((0x30ull | (uint64_t)(x) << 3))
+#define DPI_LF_RINGX_RIDX(x) ((0x40ull | (uint64_t)(x) << 3))
+#define DPI_LF_RINGX_WIDX(x) ((0x50ull | (uint64_t)(x) << 3))
+#define DPI_LF_RINGX_RST(x) ((0x70ull | (uint64_t)(x) << 3))
+#define DPI_LF_RINGX_ISTAT(x) ((0x80ull | (uint64_t)(x) << 3))
+#define DPI_LF_RINGX_CMPL(x) ((0x90ull | (uint64_t)(x) << 3))
+#define DPI_LF_RINGX_INT (0x100ull)
+#define DPI_LF_RINGX_INT_W1S (0x108ull)
+#define DPI_LF_RINGX_INT_ENA_W1C (0x110ull)
+#define DPI_LF_RINGX_INT_ENA_W1S (0x118ull)
+#define DPI_LF_RINGX_ERR_STAT(x) ((0x120ull | (uint64_t)(x) << 3))
+#define DPI_LF_RINGX_ERR (0x200ull)
+#define DPI_LF_RINGX_ERR_W1S (0x208ull)
+#define DPI_LF_RINGX_ERR_ENA_W1C (0x210ull)
+#define DPI_LF_RINGX_ERR_ENA_W1S (0x218ull)
+#define DPI_LF_RINGX_DMA_CNT(x) ((0x220ull | (uint64_t)(x) << 3))
+#define DPI_LF_RINGX_DMA_BCNT(x) ((0x230ull | (uint64_t)(x) << 3))
+
/**
* Enumeration dpi_hdr_xtype_e
*
@@ -54,6 +75,26 @@
#define DPI_MIN_CMD_SIZE 8
#define DPI_MAX_CMD_SIZE 64
+#define DPI_CMD_SIZE_64B 64
+#define DPI_CMD_SIZE_128B 128
+
+#define DPI_CMD_VLD_BIT BIT_ULL(63)
+
+#define DPI_LF_QCFG_QEN BIT_ULL(63)
+#define DPI_LF_QCFG_ISIZE BIT(11)
+#define DPI_LF_QUEUE_RST BIT(0)
+
+#define DPI_LF_QIDX_WRAP_MASK 0x8000
+#define DPI_LF_QSIZE_MASK 0xFF
+#define DPI_LF_QIDX_MASK 0xFFF
+#define DPI_LF_QSIZE_SHIFT 56
+
+#define DPI_Q_RIDX(x) ((x)&0xFFF)
+#define DPI_Q_WIDX(x) ((x)&0xFFF)
+
+#define DPI_Q_RIDX_WRAP(x) (((x) >> 15) & 0x1)
+#define DPI_Q_WIDX_WRAP(x) (((x) >> 15) & 0x1)
+
/**
* Structure dpi_instr_hdr_s for CN9K
*
@@ -61,7 +102,7 @@
*/
union dpi_instr_hdr_s {
uint64_t u[4];
- struct dpi_cn9k_instr_hdr_s_s {
+ struct dpi_cn9k_instr_hdr_s {
uint64_t tag : 32;
uint64_t tt : 2;
uint64_t grp : 10;
@@ -95,7 +136,7 @@ union dpi_instr_hdr_s {
/* Word 3 - End */
} cn9k;
- struct dpi_cn10k_instr_hdr_s_s {
+ struct dpi_cn10k_instr_hdr_s {
uint64_t nfst : 4;
uint64_t reserved_4_5 : 2;
uint64_t nlst : 4;
@@ -128,6 +169,33 @@ union dpi_instr_hdr_s {
uint64_t reserved_192_255 : 64;
/* Word 3 - End */
} cn10k;
+
+ struct dpi_cn20k_instr_hdr_s {
+ uint64_t nfst : 3;
+ uint64_t reserved_3 : 1;
+ uint64_t nlst : 3;
+ uint64_t reserved_7 : 1;
+ uint64_t msix_int : 1;
+ uint64_t ct : 3;
+ uint64_t chan : 14;
+ uint64_t reserved_26_29 : 4;
+ uint64_t aura : 20;
+ uint64_t xt : 2;
+ uint64_t ivec : 9;
+ uint64_t fe : 1;
+ uint64_t reserved_62 : 1;
+ uint64_t vld : 1;
+ /* Word 0 - End */
+ uint64_t ptr : 64;
+ /* Word 1 - End */
+ uint64_t tag : 32;
+ uint64_t tt : 2;
+ uint64_t grp : 10;
+ uint64_t reserved_107_127 : 20;
+ /* Word 2 - End */
+ uint64_t reserved_128_191 : 64;
+ /* Word 3 - End */
+ } cn20k;
};
#endif /*__DEV_DPI_HW_H__*/
diff --git a/drivers/common/cnxk/hw/rvu.h b/drivers/common/cnxk/hw/rvu.h
index 6f02d1e3d2..01f19b0a16 100644
--- a/drivers/common/cnxk/hw/rvu.h
+++ b/drivers/common/cnxk/hw/rvu.h
@@ -169,6 +169,8 @@
#define RVU_BLOCK_ADDR_R_START (0x14ull)
#define RVU_BLOCK_ADDR_REE0 (0x14ull)
#define RVU_BLOCK_ADDR_REE1 (0x15ull)
+#define RVU_BLOCK_ADDR_DPI0 (0x18ULL)
+#define RVU_BLOCK_ADDR_DPI1 (0x19ULL)
#define RVU_BLOCK_ADDR_MBOX (0x1bULL)
#define RVU_VF_INT_VEC_MBOX (0x0ull)
diff --git a/drivers/common/cnxk/roc_constants.h b/drivers/common/cnxk/roc_constants.h
index 795869c9df..3218e641f0 100644
--- a/drivers/common/cnxk/roc_constants.h
+++ b/drivers/common/cnxk/roc_constants.h
@@ -47,6 +47,8 @@
#define PCI_DEVID_CNXK_RVU_ESWITCH_VF 0xA0E1
#define PCI_DEVID_CNXK_RVU_BPHY_PF 0xA0E4
#define PCI_DEVID_CNXK_RVU_BPHY_VF 0xA0E5
+#define PCI_DEVID_CN20K_DPI_PF 0xA0E8
+#define PCI_DEVID_CN20K_DPI_VF 0xA0E9
#define PCI_DEVID_CN9K_CGX 0xA059
#define PCI_DEVID_CN10K_RPM 0xA060
diff --git a/drivers/common/cnxk/roc_dev.c b/drivers/common/cnxk/roc_dev.c
index 61aa4b3075..e2b717a214 100644
--- a/drivers/common/cnxk/roc_dev.c
+++ b/drivers/common/cnxk/roc_dev.c
@@ -1401,6 +1401,7 @@ dev_vf_hwcap_update(struct plt_pci_device *pci_dev, struct dev *dev)
case PCI_DEVID_CNXK_RVU_NIX_INL_VF:
case PCI_DEVID_CNXK_RVU_BPHY_VF:
case PCI_DEVID_CNXK_RVU_ESWITCH_VF:
+ case PCI_DEVID_CN20K_DPI_VF:
dev->hwcap |= DEV_HWCAP_F_VF;
break;
}
diff --git a/drivers/common/cnxk/roc_dpi.c b/drivers/common/cnxk/roc_dpi.c
index 71edfcbf9b..39d20accb0 100644
--- a/drivers/common/cnxk/roc_dpi.c
+++ b/drivers/common/cnxk/roc_dpi.c
@@ -6,10 +6,14 @@
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
+#include <stdarg.h>
#include "roc_api.h"
#include "roc_priv.h"
+#define ROC_DPI_DEV_NAME "roc_dpi_dev_"
+#define ROC_DPI_DEV_NAME_LEN (sizeof(ROC_DPI_DEV_NAME) + PCI_PRI_STR_SIZE)
+
#define DPI_PF_MBOX_SYSFS_ENTRY "dpi_device_config"
static inline int
@@ -56,17 +60,225 @@ roc_dpi_wait_queue_idle(struct roc_dpi *roc_dpi)
return 0;
}
+void
+dpi_lf_ena_dis(struct roc_dpi_lf *lf, uint8_t enb)
+{
+ uint64_t reg;
+ int ring_idx;
+
+ for (ring_idx = 0; ring_idx < ROC_DPI_LF_RINGS; ring_idx++) {
+ reg = plt_read64(lf->rbase + DPI_LF_RINGX_CFG(ring_idx));
+
+ if (enb)
+ reg |= DPI_LF_QCFG_QEN;
+ else
+ reg &= ~DPI_LF_QCFG_QEN;
+
+ plt_write64(reg, lf->rbase + DPI_LF_RINGX_CFG(ring_idx));
+ }
+}
+
+int
+dpi_lf_reset(struct roc_dpi_lf *lf)
+{
+ uint64_t start_cycle;
+ uint64_t wait_cycles;
+ uintptr_t reg_addr;
+ int ring_idx;
+
+ wait_cycles = (DPI_LF_RESET_TMO_US * plt_tsc_hz()) / 1000000;
+
+ for (ring_idx = 0; ring_idx < ROC_DPI_LF_RINGS; ring_idx++) {
+ reg_addr = lf->rbase + DPI_LF_RINGX_RST(ring_idx);
+ plt_write64(DPI_LF_QUEUE_RST, reg_addr);
+
+ start_cycle = plt_tsc_cycles();
+ while (plt_read64(reg_addr) & DPI_LF_QUEUE_RST) {
+ if (plt_tsc_cycles() - start_cycle >= wait_cycles) {
+ plt_err("DPI LF[%u]: ring[%u] reset timed out", lf->slot, ring_idx);
+ return -ETIMEDOUT;
+ }
+ }
+ }
+
+ return 0;
+}
+
+int
+roc_dpi_access_pair_group_create(struct roc_dpi_lf *lf, rte_uuid_t domain_id, rte_uuid_t token,
+ int16_t *group_id)
+{
+ struct mbox *mbox = mbox_get(lf->dev->mbox);
+ struct dpi_lf_access_group_alloc_req *req;
+ struct dpi_lf_access_group_alloc_rsp *rsp;
+ int rc;
+
+ req = mbox_alloc_msg_dpi_lf_access_group_alloc(mbox);
+ if (req == NULL) {
+ rc = -ENOSPC;
+ goto exit;
+ }
+
+ req->dpi_blkaddr = lf->blk_addr;
+ mbox_memcpy(req->lf_handle, domain_id, sizeof(rte_uuid_t));
+ mbox_memcpy(req->access_key, token, sizeof(rte_uuid_t));
+
+ rc = mbox_process_msg(mbox, (void **)&rsp);
+ if (rc)
+ goto exit;
+
+ *group_id = rsp->group_id;
+ lf->group_id = rsp->group_id;
+exit:
+ mbox_put(mbox);
+ return rc;
+}
+
+int
+roc_dpi_access_pair_group_destroy(struct roc_dpi_lf *lf, int16_t group_id)
+{
+ struct mbox *mbox = mbox_get(lf->dev->mbox);
+ struct dpi_lf_access_group_free_req *req;
+ int rc;
+
+ req = mbox_alloc_msg_dpi_lf_access_group_free(mbox);
+ if (req == NULL) {
+ rc = -ENOSPC;
+ goto exit;
+ }
+
+ req->dpi_blkaddr = lf->blk_addr;
+ req->group_id = group_id;
+ rc = mbox_process(mbox);
+exit:
+ mbox_put(mbox);
+ return rc;
+}
+
+int
+roc_dpi_access_pair_group_join(struct roc_dpi_lf *lf, rte_uuid_t domain_id, rte_uuid_t token,
+ int16_t group_id)
+{
+ struct mbox *mbox = mbox_get(lf->dev->mbox);
+ struct dpi_lf_access_group_join_req *req;
+ int rc;
+
+ req = mbox_alloc_msg_dpi_lf_access_group_join(mbox);
+ if (req == NULL) {
+ rc = -ENOSPC;
+ goto exit;
+ }
+
+ mbox_memcpy(req->lf_handle, domain_id, sizeof(rte_uuid_t));
+ mbox_memcpy(req->access_key, token, sizeof(rte_uuid_t));
+ req->dpi_blkaddr = lf->blk_addr;
+ req->group_id = group_id;
+
+ rc = mbox_process(mbox);
+ if (rc)
+ goto exit;
+
+ lf->group_id = group_id;
+exit:
+ mbox_put(mbox);
+ return rc;
+}
+
+int
+roc_dpi_access_pair_group_leave(struct roc_dpi_lf *lf, int16_t group_id)
+{
+ struct mbox *mbox = mbox_get(lf->dev->mbox);
+ struct dpi_lf_access_group_leave_req *req;
+ int rc;
+
+ req = mbox_alloc_msg_dpi_lf_access_group_leave(mbox);
+ if (req == NULL) {
+ rc = -ENOSPC;
+ goto exit;
+ }
+
+ req->dpi_blkaddr = lf->blk_addr;
+ req->group_id = group_id;
+
+ rc = mbox_process(mbox);
+exit:
+ mbox_put(mbox);
+ return rc;
+}
+
+int
+roc_dpi_access_pair_group_handler_get(struct roc_dpi_lf *lf, int16_t group_id, rte_uuid_t domain_id,
+ uint16_t *handler)
+{
+ struct mbox *mbox = mbox_get(lf->dev->mbox);
+ struct dpi_lf_access_group_handle_get_req *req;
+ struct dpi_lf_access_group_handle_get_rsp *rsp;
+ int rc;
+
+ req = mbox_alloc_msg_dpi_lf_access_group_handle_get(mbox);
+ if (req == NULL) {
+ rc = -ENOSPC;
+ goto exit;
+ }
+
+ mbox_memcpy(req->lf_handle, domain_id, sizeof(rte_uuid_t));
+ req->dpi_blkaddr = lf->blk_addr;
+ req->group_id = group_id;
+
+ rc = mbox_process_msg(mbox, (void **)&rsp);
+ if (rc)
+ goto exit;
+
+ *handler = rsp->handle;
+exit:
+ mbox_put(mbox);
+ return rc;
+}
+
int
-roc_dpi_enable(struct roc_dpi *dpi)
+roc_dpi_reset(struct roc_dpi *dpi)
{
- plt_write64(0x1, dpi->rbase + DPI_VDMA_EN);
+ uint16_t i;
+ int rc = 0;
+
+ if (roc_model_is_cn20k()) {
+ for (i = 0; i < dpi->nr_lfs; i++) {
+ rc |= dpi_lf_reset(&dpi->lfs[i]);
+ if (rc)
+ plt_err("Reset failed for DPI LF - %u", i);
+ }
+ }
+
+ return rc;
+}
+
+int
+roc_dpi_enable(struct roc_dpi *roc_dpi)
+{
+ uint16_t i;
+
+ if (roc_model_is_cn20k()) {
+ for (i = 0; i < roc_dpi->nr_lfs; i++)
+ dpi_lf_ena_dis(&roc_dpi->lfs[i], true);
+ } else {
+ plt_write64(0x1, roc_dpi->rbase + DPI_VDMA_EN);
+ }
+
return 0;
}
int
-roc_dpi_disable(struct roc_dpi *dpi)
+roc_dpi_disable(struct roc_dpi *roc_dpi)
{
- plt_write64(0x0, dpi->rbase + DPI_VDMA_EN);
+ uint16_t i;
+
+ if (roc_model_is_cn20k()) {
+ for (i = 0; i < roc_dpi->nr_lfs; i++)
+ dpi_lf_ena_dis(&roc_dpi->lfs[i], false);
+ } else {
+ plt_write64(0x0, roc_dpi->rbase + DPI_VDMA_EN);
+ }
+
return 0;
}
@@ -154,28 +366,481 @@ roc_dpi_configure_v2(struct roc_dpi *roc_dpi, uint32_t chunk_sz, uint64_t aura,
return rc;
}
+int
+dpi_lf_attach(struct dev *dev, uint8_t blk_addr, bool modify, uint16_t nb_lf)
+{
+ struct mbox *mbox = mbox_get(dev->mbox);
+ struct dpi_rsrc_attach_req *req;
+ int rc;
+
+ req = mbox_alloc_msg_dpi_attach_resources(mbox);
+ if (req == NULL) {
+ rc = -ENOSPC;
+ goto exit;
+ }
+
+ req->dpi_lfs = nb_lf;
+ req->dpilfs = 1;
+ req->modify = modify;
+ req->dpi_blkaddr = blk_addr;
+
+ rc = mbox_process(mbox);
+exit:
+ mbox_put(mbox);
+ return rc;
+}
+
+int
+dpi_lf_detach(struct dev *dev)
+{
+ struct mbox *mbox = mbox_get(dev->mbox);
+ uint8_t blk_addr = RVU_BLOCK_ADDR_DPI0;
+ struct dpi_rsrc_detach *req;
+ int rc;
+
+ req = mbox_alloc_msg_dpi_detach_resources(mbox);
+ if (req == NULL) {
+ rc = -ENOSPC;
+ goto exit;
+ }
+
+ req->dpi_blkaddr = blk_addr;
+ req->dpilfs = 1;
+ req->partial = 1;
+
+ rc = mbox_process(mbox);
+exit:
+ mbox_put(mbox);
+ return rc;
+}
+
+int
+dpi_chan_tbl_alloc(struct dev *dev, uint8_t blk_addr, uint16_t tbl_sz)
+{
+ struct mbox *mbox = mbox_get(dev->mbox);
+ struct dpi_lf_chan_tbl_alloc_req *req;
+ struct dpi_lf_chan_tbl_alloc_rsp *rsp;
+ int rc;
+
+ req = mbox_alloc_msg_dpi_lf_chan_tbl_alloc(mbox);
+ if (req == NULL) {
+ rc = -ENOSPC;
+ goto exit;
+ }
+
+ req->tbl_size = tbl_sz;
+ req->dpi_blkaddr = blk_addr;
+
+ rc = mbox_process_msg(mbox, (void **)&rsp);
+ if (rc)
+ goto exit;
+ rc = rsp->tbl_num;
+exit:
+ mbox_put(mbox);
+ return rc;
+}
+
+int
+dpi_chan_tbl_free(struct dev *dev, uint8_t blk_addr, uint16_t tbl_num)
+{
+ struct mbox *mbox = mbox_get(dev->mbox);
+ struct dpi_lf_chan_tbl_free_req *req;
+ int rc;
+
+ req = mbox_alloc_msg_dpi_lf_chan_tbl_free(mbox);
+ if (req == NULL) {
+ rc = -ENOSPC;
+ goto exit;
+ }
+
+ req->tbl_num = tbl_num;
+ req->dpi_blkaddr = blk_addr;
+
+ rc = mbox_process(mbox);
+
+exit:
+ mbox_put(mbox);
+ return rc;
+}
+
+int
+roc_dpi_lf_chan_tbl_select(struct roc_dpi_lf *lf)
+{
+ struct mbox *mbox = mbox_get(lf->dev->mbox);
+ struct dpi_lf_chan_tbl_select_req *req;
+ int rc;
+
+ req = mbox_alloc_msg_dpi_lf_chan_tbl_select(mbox);
+ if (req == NULL) {
+ rc = -ENOSPC;
+ goto exit;
+ }
+
+ req->lf_slot = lf->slot;
+ req->chan_tbl = lf->chan_tbl;
+ req->dpi_blkaddr = lf->blk_addr;
+ req->ena = true;
+
+ rc = mbox_process(mbox);
+exit:
+ mbox_put(mbox);
+ return rc;
+}
+
+int
+roc_dpi_lf_chan_tbl_ena_dis(struct roc_dpi_lf *lf, bool ena)
+{
+ struct mbox *mbox = mbox_get(lf->dev->mbox);
+ struct dpi_lf_chan_tbl_ena_dis_req *req;
+ int rc;
+
+ req = mbox_alloc_msg_dpi_lf_chan_tbl_ena_dis(mbox);
+ if (req == NULL) {
+ rc = -ENOSPC;
+ goto exit;
+ }
+
+ req->lf_slot = lf->slot;
+ req->dpi_blkaddr = lf->blk_addr;
+ req->ena_dis = ena;
+
+ rc = mbox_process(mbox);
+
+exit:
+ mbox_put(mbox);
+ return rc;
+}
+
+int
+dpi_chan_tbl_update(struct dev *dev, uint8_t blk_addr, uint16_t chan_tbl, uint64_t *tbl,
+ uint16_t off, uint16_t nb_entries)
+{
+ struct mbox *mbox = mbox_get(dev->mbox);
+ struct dpi_lf_chan_tbl_update_req *req;
+ int rc;
+
+ req = mbox_alloc_msg_dpi_lf_chan_tbl_update(mbox);
+ if (req == NULL) {
+ rc = -ENOSPC;
+ goto exit;
+ }
+
+ mbox_memcpy(req->config, tbl, nb_entries * sizeof(uint64_t));
+
+ req->chan_tbl = chan_tbl;
+ req->num_entries = nb_entries;
+ req->idx_offset = off;
+ req->dpi_blkaddr = blk_addr;
+
+ rc = mbox_process(mbox);
+exit:
+ mbox_put(mbox);
+ return rc;
+}
+
+int
+roc_dpi_lf_chan_tbl_update(struct roc_dpi_lf *lf, uint64_t *config, uint16_t offset,
+ uint16_t entries)
+{
+ if (entries > DPI_LF_CHAN_TBL_UPDATE_SIZE)
+ return -EINVAL;
+
+ return dpi_chan_tbl_update(lf->dev, lf->blk_addr, lf->chan_tbl, config, offset, entries);
+}
+
+int
+dpi_chan_tbl_ena_dis(struct dev *dev, uint32_t blkaddr, uint16_t lfid, uint16_t chan_tbl,
+ bool enable)
+{
+ struct mbox *mbox = mbox_get(dev->mbox);
+ struct dpi_lf_chan_tbl_select_req *req;
+ int rc;
+
+ req = mbox_alloc_msg_dpi_lf_chan_tbl_select(mbox);
+ if (req == NULL) {
+ rc = -ENOSPC;
+ goto exit;
+ }
+
+ req->chan_tbl = chan_tbl;
+ req->lf_slot = lfid;
+ req->ena = enable;
+ req->dpi_blkaddr = blkaddr;
+
+ rc = mbox_process(mbox);
+exit:
+ mbox_put(mbox);
+ return rc;
+}
+
+int
+roc_dpi_lf_chan_tbl_alloc(struct roc_dpi_lf *lf, uint16_t tbl_sz)
+{
+ int rc;
+
+ rc = dpi_chan_tbl_alloc(lf->dev, lf->blk_addr, tbl_sz);
+ if (rc < 0)
+ return rc;
+
+ lf->chan_tbl_sz = tbl_sz;
+ lf->chan_tbl = rc;
+ return 0;
+}
+
+int
+roc_dpi_lf_chan_tbl_free(struct roc_dpi_lf *lf)
+{
+ return dpi_chan_tbl_free(lf->dev, lf->blk_addr, lf->chan_tbl);
+}
+
+int
+roc_dpi_lf_pffunc_cfg(struct roc_dpi_lf *lf)
+{
+ struct mbox *mbox = mbox_get(lf->dev->mbox);
+ struct dpi_lf_pf_func_cfg_req *req;
+ int rc = -ENOSPC;
+
+ req = mbox_alloc_msg_dpi_lf_pf_func_cfg(mbox);
+ if (req == NULL) {
+ rc = -ENOSPC;
+ goto exit;
+ }
+
+ req->dpi_blkaddr = lf->blk_addr;
+ req->sso_pf_func = idev_sso_pffunc_get();
+ req->npa_pf_func = idev_npa_pffunc_get();
+ req->lf_slot = lf->slot;
+
+ rc = mbox_process(mbox);
+
+exit:
+ mbox_put(mbox);
+ return rc;
+}
+
+static int
+dpi_lf_queue_configure(struct roc_dpi_lf_que *que, struct roc_dpi_lf_ring_cfg *rcfg)
+{
+ char nm[ROC_DPI_DEV_NAME_LEN] = {'\0'};
+ struct roc_dpi_lf *lf = que->lf;
+ const struct plt_memzone *mz;
+ uint64_t reg;
+
+ snprintf(nm, sizeof(nm), "%s_%u_%u_%x", "dpi_lf_q", lf->slot, rcfg->ring_idx,
+ lf->dev->pf_func);
+ mz = plt_memzone_reserve_aligned(nm, que->qsize * que->cmd_len, 0, 128);
+ if (!mz) {
+ plt_err("Cannot alloc buffer for DPI LF ring command buffer: %s", nm);
+ return -ENOMEM;
+ }
+
+ que->mz = mz;
+ que->cmd_base = (uint64_t *)mz->addr;
+
+ reg = plt_read64(lf->rbase + DPI_LF_RINGX_CFG(rcfg->ring_idx));
+
+ if (rcfg->isize)
+ reg |= DPI_LF_QCFG_ISIZE;
+ else
+ reg &= ~DPI_LF_QCFG_ISIZE;
+
+ reg |= (((uint64_t)que->first_skip << 20) | ((uint64_t)que->later_skip << 28));
+ reg |= BIT_ULL(7);
+
+ plt_write64(reg, lf->rbase + DPI_LF_RINGX_CFG(rcfg->ring_idx));
+
+ reg = plt_read64(lf->rbase + DPI_LF_RINGX_BASE(rcfg->ring_idx));
+ reg = (uint64_t)que->cmd_base;
+ reg |= ((((que->mz->len >> 10) - 1) & DPI_LF_QSIZE_MASK) << DPI_LF_QSIZE_SHIFT);
+ plt_write64(reg, lf->rbase + DPI_LF_RINGX_BASE(rcfg->ring_idx));
+
+ return 0;
+}
+
+int
+roc_dpi_lf_ring_init(struct roc_dpi_lf_que *que, struct roc_dpi_lf_ring_cfg *rcfg)
+{
+ struct roc_dpi_lf *lf = que->lf;
+ struct mbox *mbox = mbox_get(lf->dev->mbox);
+ struct dpi_lf_ring_cfg_req *req;
+ int rc = -ENOSPC;
+
+ req = mbox_alloc_msg_dpi_lf_ring_cfg(mbox);
+ if (req == NULL)
+ goto fail;
+
+ req->dpi_blkaddr = lf->blk_addr;
+ req->lf_slot = lf->slot;
+ req->xtype = rcfg->xtype;
+ req->rport = rcfg->rport;
+ req->wport = rcfg->wport;
+ req->ring_idx = rcfg->ring_idx;
+ req->pri = rcfg->pri;
+
+ rc = mbox_process(mbox);
+ if (rc)
+ goto fail;
+
+ rc = dpi_lf_queue_configure(que, rcfg);
+
+fail:
+ mbox_put(mbox);
+ return rc;
+}
+
+void
+roc_dpi_lf_ring_fini(struct roc_dpi_lf_que *que)
+{
+ if (que->mz) {
+ plt_memzone_free(que->mz);
+ que->mz = NULL;
+ }
+}
+
+int
+roc_dpi_lf_ring_chan_cfg(struct roc_dpi_lf_que *que, union roc_dpi_lf_ccfg *ccfg)
+{
+ struct roc_dpi_lf *lf = que->lf;
+ struct mbox *mbox = mbox_get(lf->dev->mbox);
+ struct dpi_lf_chan_cfg_req *req;
+ int rc = -ENOSPC;
+
+ req = mbox_alloc_msg_dpi_lf_chan_cfg(mbox);
+ if (req == NULL)
+ goto fail;
+
+ req->dpi_blkaddr = lf->blk_addr;
+ req->def_config = ccfg->u;
+ req->lf_slot = lf->slot;
+ req->ring_idx = que == lf->queue ? 0 : 1;
+
+ rc = mbox_process(mbox);
+
+fail:
+ mbox_put(mbox);
+ return rc;
+}
+
+int
+dpi_lf_init(struct roc_dpi_lf *lf, struct dev *dev, uint8_t slot)
+{
+ uint8_t blk_addr = RVU_BLOCK_ADDR_DPI0;
+
+ lf->dev = dev;
+ lf->slot = slot;
+ lf->rbase = dev->bar2 + (RVU_BLOCK_ADDR_DPI0 << 20 | slot << 12);
+ lf->queue[0].lf = lf;
+ lf->queue[1].lf = lf;
+ lf->blk_addr = blk_addr;
+ return 0;
+}
+
+int
+roc_dpi_rsrc_init(struct roc_dpi *roc_dpi)
+{
+ struct plt_pci_device *pci_dev = roc_dpi->pci_dev;
+ struct dpi *dpi = roc_dpi_to_dpi_priv(roc_dpi);
+ uint8_t blk_addr = RVU_BLOCK_ADDR_DPI0;
+ char name[ROC_DPI_DEV_NAME_LEN];
+ const struct plt_memzone *mz;
+ struct dev *dev = &dpi->dev;
+ uint16_t slot;
+ int rc;
+
+ mz = plt_memzone_reserve_cache_align(plt_pci_dev_name(name, ROC_DPI_DEV_NAME, pci_dev),
+ roc_dpi->nr_lfs * sizeof(struct roc_dpi_lf));
+ if (!mz)
+ return -ENOMEM;
+
+ roc_dpi->mz = mz;
+ roc_dpi->lfs = mz->addr;
+
+ rc = dpi_lf_attach(dev, blk_addr, true, roc_dpi->nr_lfs);
+ if (rc) {
+ plt_err("Could not attach LFs");
+ plt_memzone_free(mz);
+ roc_dpi->mz = NULL;
+ roc_dpi->lfs = NULL;
+ return rc;
+ }
+
+ for (slot = 0; slot < roc_dpi->nr_lfs; slot++)
+ dpi_lf_init(&(roc_dpi->lfs[slot]), dev, slot);
+
+ return rc;
+}
+
+int
+roc_dpi_rsrc_fini(struct roc_dpi *roc_dpi)
+{
+ struct dpi *dpi = roc_dpi_to_dpi_priv(roc_dpi);
+ struct dev *dev = &dpi->dev;
+ struct roc_dpi_lf_que *que;
+ struct roc_dpi_lf *lf;
+ uint16_t slot, qid;
+ int rc;
+
+ roc_dpi_disable(roc_dpi);
+
+ for (slot = 0; slot < roc_dpi->nr_lfs; slot++) {
+ lf = &roc_dpi->lfs[slot];
+
+ for (qid = 0; qid < ROC_DPI_LF_RINGS; qid++) {
+ que = &lf->queue[qid];
+ if (que->mz) {
+ plt_memzone_free(que->mz);
+ que->mz = NULL;
+ }
+ }
+ }
+
+ rc = dpi_lf_detach(dev);
+ plt_memzone_free(roc_dpi->mz);
+ roc_dpi->mz = NULL;
+ roc_dpi->lfs = NULL;
+
+ return rc;
+}
+
int
roc_dpi_dev_init(struct roc_dpi *roc_dpi, uint8_t offset)
{
struct plt_pci_device *pci_dev = roc_dpi->pci_dev;
+ struct dpi *dpi = roc_dpi_to_dpi_priv(roc_dpi);
+ struct dev *dev = &dpi->dev;
uint16_t vfid;
+ int rc = 0;
roc_dpi->rbase = pci_dev->mem_resource[0].addr;
- vfid = ((pci_dev->addr.devid & 0x1F) << 3) | (pci_dev->addr.function & 0x7);
- vfid -= 1;
- roc_dpi->vfid = vfid;
- idev_dma_cs_offset_set(offset);
- return 0;
+ if (roc_model_is_cn20k()) {
+ rc = dev_init(dev, pci_dev);
+ if (rc)
+ plt_err("Failed to init dpi roc device");
+ } else {
+ vfid = ((pci_dev->addr.devid & 0x1F) << 3) | (pci_dev->addr.function & 0x7);
+ vfid -= 1;
+ roc_dpi->vfid = vfid;
+ idev_dma_cs_offset_set(offset);
+ }
+
+ return rc;
}
int
roc_dpi_dev_fini(struct roc_dpi *roc_dpi)
{
struct plt_pci_device *pci_dev = roc_dpi->pci_dev;
+ struct dpi *dpi = roc_dpi_to_dpi_priv(roc_dpi);
+ struct dev *dev = &dpi->dev;
dpi_mbox_msg_t mbox_msg;
int rc;
+ if (roc_model_is_cn20k()) {
+ rc = dev_fini(dev, pci_dev);
+ return rc;
+ }
+
rc = roc_dpi_wait_queue_idle(roc_dpi);
if (rc)
return rc;
diff --git a/drivers/common/cnxk/roc_dpi.h b/drivers/common/cnxk/roc_dpi.h
index 3a11559df9..6f12ad2c4b 100644
--- a/drivers/common/cnxk/roc_dpi.h
+++ b/drivers/common/cnxk/roc_dpi.h
@@ -5,16 +5,80 @@
#ifndef _ROC_DPI_H_
#define _ROC_DPI_H_
+#define ROC_DPI_LF_RINGS 2
+#define ROC_DPI_MAX_LFS 256
+#define ROC_DPI_LF_CHAN_TBL_SZ 4096
+
+struct roc_dpi_lf_ring_cfg {
+ uint8_t ring_idx;
+ uint8_t xtype;
+ uint8_t rport;
+ uint8_t wport;
+ uint8_t pri;
+ uint8_t isize;
+};
+
+union roc_dpi_lf_ccfg {
+ uint64_t u;
+ struct {
+ uint64_t vf_func : 12;
+ uint64_t pf_func : 4;
+ uint64_t st : 8;
+ uint64_t rsvd_24_31 : 8;
+ uint64_t pasid : 20;
+ uint64_t pasid_ctrl : 2;
+ uint64_t th : 1;
+ uint64_t ph : 2;
+ uint64_t rsvd_57_62 : 6;
+ uint64_t valid : 1;
+ };
+ struct {
+ uint32_t type;
+ uint16_t src_key;
+ uint16_t dst_key;
+ };
+};
+
+struct roc_dpi_lf_que {
+ const struct plt_memzone *mz;
+ uint64_t *cmd_base;
+ struct roc_dpi_lf *lf;
+ uint16_t qsize;
+ uint16_t widx;
+ uint8_t cmd_len;
+ uint8_t first_skip;
+ uint8_t later_skip;
+} __plt_cache_aligned;
+
+struct roc_dpi_lf {
+ struct roc_dpi_lf_que queue[ROC_DPI_LF_RINGS];
+ uintptr_t rbase;
+ struct dev *dev;
+ uint16_t chan_tbl;
+ uint16_t chan_tbl_sz;
+ uint16_t slot;
+ uint16_t blk_addr;
+ int group_id;
+};
+
struct roc_dpi {
struct plt_pci_device *pci_dev;
+ const struct plt_memzone *mz;
+ struct roc_dpi_lf *lfs;
uint8_t *rbase;
uint16_t vfid;
uint8_t priority;
+ uint16_t nr_lfs;
+
+#define ROC_DPI_MEM_SZ (4 * 1024)
+ uint8_t reserved[ROC_DPI_MEM_SZ] __plt_cache_aligned;
} __plt_cache_aligned;
int __roc_api roc_dpi_dev_init(struct roc_dpi *roc_dpi, uint8_t offset);
int __roc_api roc_dpi_dev_fini(struct roc_dpi *roc_dpi);
+int __roc_api roc_dpi_rsrc_init(struct roc_dpi *roc_dpi);
+int __roc_api roc_dpi_rsrc_fini(struct roc_dpi *roc_dpi);
int __roc_api roc_dpi_configure(struct roc_dpi *dpi, uint32_t chunk_sz, uint64_t aura,
uint64_t chunk_base);
int __roc_api roc_dpi_configure_v2(struct roc_dpi *roc_dpi, uint32_t chunk_sz, uint64_t aura,
@@ -22,5 +86,26 @@ int __roc_api roc_dpi_configure_v2(struct roc_dpi *roc_dpi, uint32_t chunk_sz, u
int __roc_api roc_dpi_enable(struct roc_dpi *dpi);
int __roc_api roc_dpi_wait_queue_idle(struct roc_dpi *dpi);
int __roc_api roc_dpi_disable(struct roc_dpi *dpi);
+int __roc_api roc_dpi_reset(struct roc_dpi *dpi);
+
+int __roc_api roc_dpi_lf_ring_init(struct roc_dpi_lf_que *que, struct roc_dpi_lf_ring_cfg *rcfg);
+void __roc_api roc_dpi_lf_ring_fini(struct roc_dpi_lf_que *que);
+int __roc_api roc_dpi_lf_pffunc_cfg(struct roc_dpi_lf *lf);
+int __roc_api roc_dpi_lf_ring_chan_cfg(struct roc_dpi_lf_que *que, union roc_dpi_lf_ccfg *cfg);
+int __roc_api roc_dpi_lf_chan_tbl_alloc(struct roc_dpi_lf *lf, uint16_t tbl_sz);
+int __roc_api roc_dpi_lf_chan_tbl_free(struct roc_dpi_lf *lf);
+int __roc_api roc_dpi_lf_chan_tbl_select(struct roc_dpi_lf *lf);
+int __roc_api roc_dpi_lf_chan_tbl_ena_dis(struct roc_dpi_lf *lf, bool ena);
+int __roc_api roc_dpi_lf_chan_tbl_update(struct roc_dpi_lf *lf, uint64_t *config, uint16_t offset,
+ uint16_t entries);
+int __roc_api roc_dpi_lf_dump(struct roc_dpi_lf *lf, FILE *file);
+int __roc_api roc_dpi_access_pair_group_create(struct roc_dpi_lf *lf, rte_uuid_t domain_id,
+ rte_uuid_t token, int16_t *group_id);
+int __roc_api roc_dpi_access_pair_group_destroy(struct roc_dpi_lf *lf, int16_t group_id);
+int __roc_api roc_dpi_access_pair_group_join(struct roc_dpi_lf *lf, rte_uuid_t domain_id,
+ rte_uuid_t token, int16_t group_id);
+int __roc_api roc_dpi_access_pair_group_leave(struct roc_dpi_lf *lf, int16_t group_id);
+int __roc_api roc_dpi_access_pair_group_handler_get(struct roc_dpi_lf *lf, int16_t group_id,
+ rte_uuid_t domain_id, uint16_t *handler);
#endif
diff --git a/drivers/common/cnxk/roc_dpi_priv.h b/drivers/common/cnxk/roc_dpi_priv.h
index 05b6751ca6..3051abc979 100644
--- a/drivers/common/cnxk/roc_dpi_priv.h
+++ b/drivers/common/cnxk/roc_dpi_priv.h
@@ -18,6 +18,7 @@
#define DPI_QUEUE_OPEN_V2 0x5
#define DPI_QUEUE_IDLE_TMO_MS 1E3
+#define DPI_LF_RESET_TMO_US 10000
typedef union dpi_mbox_msg_t {
uint64_t u[2];
@@ -43,4 +44,37 @@ typedef union dpi_mbox_msg_t {
} s;
} dpi_mbox_msg_t;
+struct dpi {
+ struct plt_pci_device *pci_dev;
+ struct dev dev;
+ uint16_t lf_msix_off[ROC_DPI_MAX_LFS];
+ uint8_t lf_blkaddr[ROC_DPI_MAX_LFS];
+};
+
+static inline struct dpi *
+roc_dpi_to_dpi_priv(struct roc_dpi *roc_dpi)
+{
+ return (struct dpi *)&roc_dpi->reserved[0];
+}
+
+static inline struct roc_dpi *
+dpi_priv_to_roc_dpi(struct dpi *dpi)
+{
+ return (struct roc_dpi *)((char *)dpi - offsetof(struct roc_dpi, reserved));
+}
+
+int dpi_lf_reset(struct roc_dpi_lf *lf);
+void dpi_lf_ena_dis(struct roc_dpi_lf *lf, uint8_t enb);
+int dpi_lfs_attach(struct dev *dev, uint8_t blkaddr, bool modify, uint16_t nb_lf);
+int dpi_lfs_detach(struct dev *dev);
+int dpi_lf_attach(struct dev *dev, uint8_t blkaddr, bool modify, uint16_t nb_lf);
+int dpi_lf_detach(struct dev *dev);
+int dpi_lf_init(struct roc_dpi_lf *lf, struct dev *dev, uint8_t slot);
+int dpi_chan_tbl_alloc(struct dev *dev, uint8_t blk_addr, uint16_t tbl_sz);
+int dpi_chan_tbl_free(struct dev *dev, uint8_t blk_addr, uint16_t tbl_num);
+int dpi_chan_tbl_ena_dis(struct dev *dev, uint32_t dpi_blkaddr, uint16_t lfid, uint16_t chan_tbl,
+ bool enable);
+int dpi_chan_tbl_update(struct dev *dev, uint8_t blk_addr, uint16_t chan_tbl, uint64_t *tbl,
+ uint16_t off, uint16_t nb_entries);
+
#endif
diff --git a/drivers/common/cnxk/roc_mbox.h b/drivers/common/cnxk/roc_mbox.h
index 52ecde6563..48d0b6f895 100644
--- a/drivers/common/cnxk/roc_mbox.h
+++ b/drivers/common/cnxk/roc_mbox.h
@@ -198,7 +198,7 @@ struct mbox_msghdr {
M(CPT_GET_CAPS, 0xBFD, cpt_caps_get, msg_req, cpt_caps_rsp_msg) \
M(CPT_GET_ENG_GRP, 0xBFF, cpt_eng_grp_get, cpt_eng_grp_req, \
cpt_eng_grp_rsp) \
- M(CPT_SET_QUEUE_PRI, 0xBFB, cpt_set_que_pri, cpt_queue_pri_req_msg, \
+ M(CPT_SET_QUEUE_PRI, 0xBFB, cpt_set_que_pri, cpt_queue_pri_req_msg, \
msg_rsp) \
/* REE mbox IDs (range 0xE00 - 0xFFF) */ \
M(REE_CONFIG_LF, 0xE01, ree_config_lf, ree_lf_req_msg, msg_rsp) \
@@ -401,7 +401,34 @@ struct mbox_msghdr {
M(MCS_FIPS_KEY_SET, 0xa045, mcs_fips_key_set, mcs_fips_key_req, msg_rsp) \
M(MCS_FIPS_BLOCK_SET, 0xa046, mcs_fips_block_set, mcs_fips_block_req, msg_rsp) \
M(MCS_FIPS_START, 0xa047, mcs_fips_start, mcs_fips_req, msg_rsp) \
- M(MCS_FIPS_RESULT_GET, 0xa048, mcs_fips_result_get, mcs_fips_req, mcs_fips_result_rsp)
+ M(MCS_FIPS_RESULT_GET, 0xa048, mcs_fips_result_get, mcs_fips_req, mcs_fips_result_rsp) \
+ /* DPI mbox IDs (range 0xc000 - 0xcfff) */ \
+ M(DPI_ATTACH_RESOURCES, 0xc000, dpi_attach_resources, dpi_rsrc_attach_req, msg_rsp) \
+ M(DPI_DETACH_RESOURCES, 0xc001, dpi_detach_resources, dpi_rsrc_detach, msg_rsp) \
+ M(DPI_LF_RING_CFG, 0xc002, dpi_lf_ring_cfg, dpi_lf_ring_cfg_req, msg_rsp) \
+ M(DPI_LF_PF_FUNC_CFG, 0xc003, dpi_lf_pf_func_cfg, dpi_lf_pf_func_cfg_req, msg_rsp) \
+ M(DPI_LF_FREE, 0xc004, dpi_lf_free, msg_req, msg_rsp) \
+ M(DPI_FREE_RSRC_CNT, 0xc005, dpi_free_rsrc_cnt, msg_req, dpi_free_rsrcs_rsp) \
+ M(DPI_LF_CHAN_CFG, 0xc006, dpi_lf_chan_cfg, dpi_lf_chan_cfg_req, msg_rsp) \
+ M(DPI_LF_CHAN_TBL_ALLOC, 0xc007, dpi_lf_chan_tbl_alloc, dpi_lf_chan_tbl_alloc_req, \
+ dpi_lf_chan_tbl_alloc_rsp) \
+ M(DPI_LF_CHAN_TBL_FREE, 0xc008, dpi_lf_chan_tbl_free, dpi_lf_chan_tbl_free_req, msg_rsp) \
+ M(DPI_LF_CHAN_TBL_SELECT, 0xc009, dpi_lf_chan_tbl_select, dpi_lf_chan_tbl_select_req, \
+ msg_rsp) \
+ M(DPI_LF_CHAN_TBL_ENA_DIS, 0xc00a, dpi_lf_chan_tbl_ena_dis, dpi_lf_chan_tbl_ena_dis_req, \
+ msg_rsp) \
+ M(DPI_LF_CHAN_TBL_UPDATE, 0xc00b, dpi_lf_chan_tbl_update, dpi_lf_chan_tbl_update_req, \
+ msg_rsp) \
+ M(DPI_LF_ACCESS_GROUP_ALLOC, 0xc00d, dpi_lf_access_group_alloc, \
+ dpi_lf_access_group_alloc_req, dpi_lf_access_group_alloc_rsp) \
+ M(DPI_LF_ACCESS_GROUP_JOIN, 0xc00e, dpi_lf_access_group_join, dpi_lf_access_group_join_req,\
+ msg_rsp) \
+ M(DPI_LF_ACCESS_GROUP_LEAVE, 0xc00f, dpi_lf_access_group_leave, \
+ dpi_lf_access_group_leave_req, msg_rsp) \
+ M(DPI_LF_ACCESS_GROUP_HANDLE_GET, 0xc010, dpi_lf_access_group_handle_get, \
+ dpi_lf_access_group_handle_get_req, dpi_lf_access_group_handle_get_rsp) \
+ M(DPI_LF_ACCESS_GROUP_FREE, 0xc011, dpi_lf_access_group_free, dpi_lf_access_group_free_req,\
+ msg_rsp)
/* Messages initiated by AF (range 0xC00 - 0xDFF) */
#define MBOX_UP_CGX_MESSAGES \
@@ -1318,7 +1345,6 @@ struct mcs_fips_result_rsp {
uint64_t __io icv_bits63_0;
uint8_t __io result_pass;
};
-
/* NPA mbox message formats */
/* NPA mailbox error codes
@@ -3318,4 +3344,146 @@ struct rep_event {
struct rep_evt_data evt_data;
};
+/* DPI mbox message formats */
+
+struct dpi_lf_chan_tbl_alloc_req {
+ struct mbox_msghdr hdr;
+
+ uint32_t __io dpi_blkaddr;
+ uint32_t __io tbl_size; /* No of table entries */
+};
+
+struct dpi_lf_chan_tbl_alloc_rsp {
+ struct mbox_msghdr hdr;
+
+ uint16_t __io tbl_num; /* Allocated channel table num */
+};
+
+struct dpi_lf_chan_tbl_free_req {
+ struct mbox_msghdr hdr;
+
+ uint32_t __io dpi_blkaddr;
+ uint16_t __io tbl_num;
+};
+
+struct dpi_lf_ring_cfg_req {
+ struct mbox_msghdr hdr;
+
+ uint32_t __io dpi_blkaddr;
+ uint16_t __io lf_slot;
+ uint8_t __io xtype; /* Transfer type */
+ uint8_t __io pri; /* Queue priority */
+ uint8_t __io ring_idx;
+ uint8_t __io err_rsp_en;
+ uint8_t __io wport; /* Write port */
+ uint8_t __io rport; /* Read port */
+};
+
+struct dpi_rsrc_attach_req {
+ struct mbox_msghdr hdr;
+ uint32_t __io dpi_blkaddr;
+ uint8_t __io modify : 1;
+ uint8_t __io dpilfs : 1;
+ uint16_t __io dpi_lfs;
+};
+
+struct dpi_rsrc_detach {
+ struct mbox_msghdr hdr;
+ uint32_t __io dpi_blkaddr;
+ uint8_t __io partial : 1;
+ uint8_t __io dpilfs : 1;
+ uint8_t __io dpi1_lfs : 1;
+};
+
+struct dpi_free_rsrcs_rsp {
+ struct mbox_msghdr hdr;
+ uint8_t __io dpi;
+ uint8_t __io dpi1;
+};
+
+struct dpi_lf_pf_func_cfg_req {
+ struct mbox_msghdr hdr;
+ uint32_t __io dpi_blkaddr;
+ uint16_t __io npa_pf_func;
+ uint16_t __io sso_pf_func;
+ uint16_t __io lf_slot;
+};
+
+struct dpi_lf_chan_cfg_req {
+ struct mbox_msghdr hdr;
+ uint64_t __io def_config; /* DPI_CHANNEL_TABLE_S value */
+ uint32_t __io dpi_blkaddr;
+ uint16_t __io lf_slot;
+ uint16_t __io ring_idx;
+};
+
+struct dpi_lf_chan_tbl_select_req {
+ struct mbox_msghdr hdr;
+ uint32_t __io dpi_blkaddr;
+ uint16_t __io lf_slot;
+ uint16_t __io chan_tbl; /* Channel table */
+ uint8_t __io ena;
+};
+
+struct dpi_lf_chan_tbl_ena_dis_req {
+ struct mbox_msghdr hdr;
+ uint32_t __io dpi_blkaddr;
+ uint16_t __io lf_slot;
+ uint8_t __io ena_dis;
+};
+
+struct dpi_lf_chan_tbl_update_req {
+ struct mbox_msghdr hdr;
+#define DPI_LF_CHAN_TBL_UPDATE_SIZE 64
+ uint64_t __io config[DPI_LF_CHAN_TBL_UPDATE_SIZE]; /* DPI_CHANNEL_TABLE_S value */
+ uint32_t __io dpi_blkaddr;
+ uint16_t __io idx_offset; /* Offset within the channel table */
+ uint16_t __io num_entries; /* Num of entries to be updated from idx_offset */
+ uint16_t __io chan_tbl;
+};
+
+struct dpi_lf_access_group_alloc_req {
+ struct mbox_msghdr hdr;
+ uint32_t __io dpi_blkaddr;
+ uint8_t __io lf_handle[16];
+ uint8_t __io access_key[16];
+};
+
+struct dpi_lf_access_group_alloc_rsp {
+ struct mbox_msghdr hdr;
+ uint16_t __io group_id;
+};
+
+struct dpi_lf_access_group_join_req {
+ struct mbox_msghdr hdr;
+ uint32_t __io dpi_blkaddr;
+ uint8_t __io lf_handle[16];
+ uint8_t __io access_key[16];
+ uint16_t __io group_id;
+};
+
+struct dpi_lf_access_group_leave_req {
+ struct mbox_msghdr hdr;
+ uint32_t __io dpi_blkaddr;
+ uint16_t __io group_id;
+};
+
+struct dpi_lf_access_group_handle_get_req {
+ struct mbox_msghdr hdr;
+ uint32_t __io dpi_blkaddr;
+ uint8_t __io lf_handle[16];
+ uint16_t __io group_id;
+};
+
+struct dpi_lf_access_group_handle_get_rsp {
+ struct mbox_msghdr hdr;
+ uint16_t __io handle;
+};
+
+struct dpi_lf_access_group_free_req {
+ struct mbox_msghdr hdr;
+ uint32_t __io dpi_blkaddr;
+ uint16_t __io group_id;
+};
+
#endif /* __ROC_MBOX_H__ */
diff --git a/drivers/common/cnxk/roc_platform.h b/drivers/common/cnxk/roc_platform.h
index ac4f76473f..082397ed4e 100644
--- a/drivers/common/cnxk/roc_platform.h
+++ b/drivers/common/cnxk/roc_platform.h
@@ -347,6 +347,14 @@ extern int cnxk_logtype_esw;
RTE_LOG_DP_LINE_PREFIX(DEBUG, CNXK, "%s():%u ", __func__ RTE_LOG_COMMA __LINE__, \
__VA_ARGS__)
+/* append dbdf to name */
+#define plt_pci_dev_name(devname, name, dev) \
+ ({ \
+ snprintf((devname), sizeof(devname), "%s" PCI_PRI_FMT, (name), (dev)->addr.domain, \
+ (dev)->addr.bus, (dev)->addr.devid, (dev)->addr.function); \
+ devname; \
+ })
+
#ifdef __cplusplus
#define CNXK_PCI_ID(subsystem_dev, dev) \
{ \
diff --git a/drivers/common/cnxk/roc_platform_base_symbols.c b/drivers/common/cnxk/roc_platform_base_symbols.c
index 063cb21aae..232129d903 100644
--- a/drivers/common/cnxk/roc_platform_base_symbols.c
+++ b/drivers/common/cnxk/roc_platform_base_symbols.c
@@ -70,6 +70,24 @@ RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_configure)
RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_configure_v2)
RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_dev_init)
RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_dev_fini)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_lf_ring_init)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_lf_ring_fini)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_lf_pffunc_cfg)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_lf_ring_chan_cfg)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_lf_chan_tbl_select)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_lf_chan_tbl_ena_dis)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_lf_chan_tbl_update)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_access_pair_group_create)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_access_pair_group_destroy)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_access_pair_group_join)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_access_pair_group_leave)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_access_pair_group_handler_get)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_reset)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_rsrc_init)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_rsrc_fini)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_lf_chan_tbl_alloc)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_lf_chan_tbl_free)
+RTE_EXPORT_INTERNAL_SYMBOL(roc_dpi_lf_chan_tbl_update)
RTE_EXPORT_INTERNAL_SYMBOL(roc_eswitch_npc_mcam_tx_rule)
RTE_EXPORT_INTERNAL_SYMBOL(roc_eswitch_npc_mcam_delete_rule)
RTE_EXPORT_INTERNAL_SYMBOL(roc_eswitch_npc_mcam_rx_rule)
--
2.34.1
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH v0 2/3] dma/cnxk: add O20 DPI DMA support
2026-08-29 10:54 [PATCH v0 0/3] dma/cnxk: add CN20K (O20) DPI DMA and inter-process domain support Vamsi Krishna
2026-08-29 10:54 ` [PATCH v0 1/3] common/cnxk: add O20 DPI DMA support Vamsi Krishna
@ 2026-08-29 10:54 ` Vamsi Krishna
2026-08-29 10:54 ` [PATCH v0 3/3] dma/cnxk: enable inter process domain " Vamsi Krishna
2 siblings, 0 replies; 4+ messages in thread
From: Vamsi Krishna @ 2026-08-29 10:54 UTC (permalink / raw)
To: dev, vattunuru; +Cc: jerinj, fengchengwen, ndabilpuram
From: Vamsi Attunuru <vattunuru@marvell.com>
Adds DMA driver support for O20 hardware.
Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>
---
drivers/dma/cnxk/cnxk_dmadev.c | 527 +++++++++++++++++++++++++++++-
drivers/dma/cnxk/cnxk_dmadev.h | 72 +++-
drivers/dma/cnxk/cnxk_dmadev_fp.c | 214 ++++++++++++
3 files changed, 791 insertions(+), 22 deletions(-)
diff --git a/drivers/dma/cnxk/cnxk_dmadev.c b/drivers/dma/cnxk/cnxk_dmadev.c
index 6ae7fdca3b..3970abe1c3 100644
--- a/drivers/dma/cnxk/cnxk_dmadev.c
+++ b/drivers/dma/cnxk/cnxk_dmadev.c
@@ -2,6 +2,9 @@
* Copyright (C) 2021 Marvell International Ltd.
*/
+#include <errno.h>
+#include <stdlib.h>
+
#include <rte_event_dma_adapter.h>
#include <cnxk_dmadev.h>
@@ -9,6 +12,68 @@
static int cnxk_stats_reset(struct rte_dma_dev *dev, uint16_t vchan);
static void cnxk_set_fp_ops(struct rte_dma_dev *dev, uint8_t enable_enq_deq);
+static int
+parse_val_u16(const char *key, const char *value, void *extra_args)
+{
+ unsigned long val;
+ char *end;
+
+ RTE_SET_USED(key);
+
+ if (value == NULL || extra_args == NULL)
+ return -EINVAL;
+
+ errno = 0;
+ val = strtoul(value, &end, 0);
+ if (errno != 0 || end == value || *end != '\0' || val > UINT16_MAX)
+ return -EINVAL;
+
+ *(uint16_t *)extra_args = (uint16_t)val;
+
+ return 0;
+}
+
+static int
+cn20k_dmadev_parse_devargs(struct rte_devargs *devargs, struct cnxk_dpi_vf_s *dpivf)
+{
+ uint16_t num_vchans = CN20K_DPI_DEF_VCHANS;
+ uint16_t num_lfs = num_vchans >> 1; /* Each LF has 2 rings */
+ struct rte_kvargs *kvlist = NULL;
+
+ if (devargs == NULL) {
+ dpivf->max_vchans = num_vchans;
+ dpivf->max_lfs = num_lfs;
+ return 0;
+ }
+
+ kvlist = rte_kvargs_parse(devargs->args, NULL);
+ if (kvlist == NULL)
+ goto exit;
+
+ if (rte_kvargs_process(kvlist, CN20K_DPI_NUM_VCHANS, &parse_val_u16, &num_vchans) < 0)
+ goto exit;
+ if (rte_kvargs_process(kvlist, CN20K_DPI_NUM_LFS, &parse_val_u16, &num_lfs) < 0)
+ goto exit;
+
+ if (!num_vchans || !num_lfs || num_vchans > CN20K_DPI_MAX_VCHANS ||
+ num_lfs > CN20K_DPI_MAX_LFS)
+ goto exit;
+
+ if (!rte_is_power_of_2(num_vchans) || !rte_is_power_of_2(num_lfs)) {
+ plt_err("num_vchans or num_lfs is not a power of 2");
+ goto exit;
+ }
+
+ dpivf->max_vchans = num_vchans;
+ dpivf->max_lfs = num_lfs;
+
+ rte_kvargs_free(kvlist);
+ return 0;
+exit:
+ rte_kvargs_free(kvlist);
+ return -EINVAL;
+}
+
static int
cnxk_dmadev_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *dev_info, uint32_t size)
{
@@ -25,9 +90,21 @@ cnxk_dmadev_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *dev_inf
dev_info->dev_capa |= RTE_DMA_CAPA_PRI_POLICY_SP;
dev_info->nb_priorities = CN10K_DPI_MAX_PRI;
}
- dev_info->max_desc = CNXK_DPI_MAX_DESC;
- dev_info->min_desc = CNXK_DPI_MIN_DESC;
- dev_info->max_sges = CNXK_DPI_MAX_POINTER;
+
+ if (roc_model_is_cn20k()) {
+ const uint16_t vpr = dpivf->vchans_per_ring ? dpivf->vchans_per_ring : 1;
+
+ dev_info->max_desc = CN20K_DPI_MAX_DESC / vpr;
+ dev_info->min_desc = CN20K_DPI_MIN_DESC;
+ dev_info->max_sges = CN20K_DPI_MAX_POINTER;
+ dev_info->max_vchans = dpivf->max_vchans;
+ dev_info->dev_capa |= RTE_DMA_CAPA_OPS_FILL;
+ } else {
+ dev_info->max_desc = CNXK_DPI_MAX_DESC;
+ dev_info->min_desc = CNXK_DPI_MIN_DESC;
+ dev_info->max_sges = CNXK_DPI_MAX_POINTER;
+ dev_info->max_vchans = CNXK_DPI_MAX_VCHANS_PER_QUEUE;
+ }
return 0;
}
@@ -52,8 +129,10 @@ cnxk_dmadev_vchan_free(struct cnxk_dpi_vf_s *dpivf, uint16_t vchan)
for (; i < num_vchans; i++) {
dpi_conf = &dpivf->conf[i];
- rte_free(dpi_conf->c_desc.compl_ptr);
- dpi_conf->c_desc.compl_ptr = NULL;
+ if (dpi_conf->c_desc.compl_ptr) {
+ rte_free(dpi_conf->c_desc.compl_ptr);
+ dpi_conf->c_desc.compl_ptr = NULL;
+ }
}
return 0;
@@ -99,10 +178,42 @@ cnxk_dmadev_chunk_pool_create(struct rte_dma_dev *dev, uint32_t nb_chunks, uint3
return rc;
}
+static int
+cnxk_dmadev_vchan_rsrc_free(struct cnxk_dpi_vf_s *dpivf)
+{
+ struct roc_dpi *rdpi = &dpivf->rdpi;
+ int rc;
+
+ if (dpivf->ring_conf) {
+ rte_free(dpivf->ring_conf);
+ dpivf->ring_conf = NULL;
+ }
+
+ if (rdpi->lfs == NULL)
+ return 0;
+
+ rc = roc_dpi_lf_chan_tbl_free(&(rdpi->lfs[0]));
+ if (rc < 0) {
+ plt_err("Failed to free dpi lf channel table");
+ goto error;
+ }
+
+ rc = roc_dpi_rsrc_fini(rdpi);
+ if (rc < 0)
+ plt_err("Failed to free dpi lfs");
+
+ dpivf->is_ring_conf_done = false;
+error:
+ return rc;
+}
+
static int
cnxk_dmadev_configure(struct rte_dma_dev *dev, const struct rte_dma_conf *conf, uint32_t conf_sz)
{
struct cnxk_dpi_vf_s *dpivf = NULL;
+ struct roc_dpi_lf *lf;
+ uint16_t num_rings;
+ int rc = 0;
RTE_SET_USED(conf_sz);
dpivf = dev->fp_obj->dev_private;
@@ -111,13 +222,78 @@ cnxk_dmadev_configure(struct rte_dma_dev *dev, const struct rte_dma_conf *conf,
* Free up vchan memory if any, before configuring num_vchans.
*/
cnxk_dmadev_vchan_free(dpivf, RTE_DMA_ALL_VCHAN);
+
dpivf->num_vchans = conf->nb_vchans;
+ if (roc_model_is_cn20k()) {
+ if (!rte_is_power_of_2(dpivf->num_vchans))
+ dpivf->num_vchans = rte_align32pow2(dpivf->num_vchans);
+
+ if (dpivf->num_vchans > dpivf->max_vchans) {
+ plt_info("Limiting vchans from %u to max_vchans %u",
+ dpivf->num_vchans, dpivf->max_vchans);
+ dpivf->num_vchans = dpivf->max_vchans;
+ }
+ }
if (roc_feature_dpi_has_priority())
dpivf->rdpi.priority = conf->priority;
cnxk_set_fp_ops(dev, conf->flags & RTE_DMA_CFG_FLAG_ENQ_DEQ);
- return 0;
+ if (roc_model_is_cn20k()) {
+ rc = cnxk_dmadev_vchan_rsrc_free(dpivf);
+ if (rc < 0)
+ goto error;
+
+ dpivf->rdpi.nr_lfs = dpivf->max_lfs;
+ num_rings = dpivf->rdpi.nr_lfs << 1;
+
+ if (dpivf->num_vchans > num_rings) {
+ dpivf->vchans_per_ring = dpivf->num_vchans / num_rings;
+ } else {
+ /* Each vchan has got it's own hardware ring */
+ dpivf->vchans_per_ring = 1;
+ dpivf->rdpi.nr_lfs = dpivf->num_vchans >> 1;
+ if (!dpivf->rdpi.nr_lfs)
+ dpivf->rdpi.nr_lfs = 1;
+ }
+
+ if ((CN20K_DPI_MAX_DESC / dpivf->vchans_per_ring) < CN20K_DPI_MIN_DESC) {
+ rc = -EINVAL;
+ plt_err("Very few LFs are requested than required");
+ goto error;
+ }
+
+ num_rings = dpivf->rdpi.nr_lfs << 1;
+ dpivf->ring_conf = rte_zmalloc("dpi_ring_conf", sizeof(struct cn20k_ring_conf) *
+ num_rings, 0);
+ if (dpivf->ring_conf == NULL) {
+ plt_err("Failed to allocate memory for ring conf");
+ rc = -ENOMEM;
+ goto error;
+ }
+
+ rc = roc_dpi_rsrc_init(&dpivf->rdpi);
+ if (rc < 0) {
+ plt_err("rsrc alloc failed");
+ goto error;
+ }
+
+ lf = &(dpivf->rdpi.lfs[0]);
+ rc = roc_dpi_lf_chan_tbl_alloc(lf, dpivf->num_vchans);
+ if (rc < 0) {
+ plt_err("Failed to allocate chan tbl");
+ goto error;
+ }
+
+ dpivf->chan_tbl = lf->chan_tbl;
+ dpivf->is_ring_conf_done = false;
+ }
+
+error:
+ if (roc_model_is_cn20k() && rc < 0)
+ cnxk_dmadev_vchan_rsrc_free(dpivf);
+
+ return rc;
}
static int
@@ -232,6 +408,87 @@ cn10k_dmadev_setup_hdr(union cnxk_dpi_instr_cmd *header, const struct rte_dma_vc
return 0;
}
+static int
+cn20k_dmadev_setup(struct cnxk_dpi_vf_s *dpivf, uint16_t vchan,
+ const struct rte_dma_vchan_conf *conf)
+{
+ struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan];
+ struct cn20k_ring_conf *rconf = dpivf->ring_conf;
+ uint16_t ridx, max_desc, num_rings;
+ union cnxk_dpi_instr_cmd *header;
+ int rc = 0;
+ int aura;
+
+ header = (union cnxk_dpi_instr_cmd *)&dpi_conf->cmd.u;
+ header->cn20k.ct = DPI_HDR_PT_ZBW_CA;
+ header->cn20k.xt = 0;
+
+ switch (conf->direction) {
+ case RTE_DMA_DIR_DEV_TO_MEM:
+ dpi_conf->cfg.xtype = DPI_XTYPE_INBOUND;
+ dpi_conf->cfg.rport = conf->src_port.pcie.coreid;
+ dpi_conf->cfg.wport = 0;
+ dpi_conf->chan_cfg.pf_func = conf->src_port.pcie.pfid;
+ dpi_conf->chan_cfg.vf_func = conf->src_port.pcie.vfid;
+ dpi_conf->chan_cfg.valid = 1;
+ break;
+ case RTE_DMA_DIR_MEM_TO_DEV:
+ dpi_conf->cfg.xtype = DPI_XTYPE_OUTBOUND;
+ dpi_conf->cfg.rport = 0;
+ dpi_conf->cfg.wport = conf->dst_port.pcie.coreid;
+ dpi_conf->chan_cfg.pf_func = conf->dst_port.pcie.pfid;
+ dpi_conf->chan_cfg.vf_func = conf->dst_port.pcie.vfid;
+ dpi_conf->chan_cfg.valid = 1;
+ aura = dmadev_src_buf_aura_get(conf->auto_free.m2d.pool, "cn20k_mempool_ops");
+ if (aura < 0)
+ return aura;
+ header->cn20k.aura = aura;
+ break;
+ case RTE_DMA_DIR_MEM_TO_MEM:
+ dpi_conf->cfg.xtype = DPI_XTYPE_INTERNAL_ONLY;
+ dpi_conf->cfg.rport = 0;
+ dpi_conf->cfg.wport = 0;
+ break;
+ case RTE_DMA_DIR_DEV_TO_DEV:
+ dpi_conf->cfg.xtype = DPI_XTYPE_EXTERNAL_ONLY;
+ dpi_conf->cfg.rport = conf->src_port.pcie.coreid;
+ dpi_conf->cfg.wport = conf->dst_port.pcie.coreid;
+ };
+
+ max_desc = conf->nb_desc;
+ if (!rte_is_power_of_2(max_desc))
+ max_desc = rte_align32pow2(max_desc);
+
+ num_rings = dpivf->rdpi.nr_lfs << 1;
+
+ for (ridx = 0; ridx < num_rings; ridx++) {
+ if (!rconf[ridx].used) {
+ if (!rconf[ridx].num_vchans) {
+ rconf[ridx].direction = conf->direction;
+ rconf[ridx].num_desc = max_desc;
+ } else if ((rconf[ridx].direction == conf->direction) &&
+ (rconf[ridx].num_vchans < dpivf->vchans_per_ring)) {
+ rconf[ridx].num_desc += max_desc;
+ } else {
+ continue;
+ }
+
+ rconf[ridx].num_vchans++;
+ dpi_conf->ridx = ridx;
+
+ if (rconf[ridx].num_vchans == dpivf->vchans_per_ring)
+ rconf[ridx].used = true;
+
+ break;
+ }
+ }
+
+ if (ridx == num_rings)
+ rc = -ENODEV;
+
+ return rc;
+}
+
static int
cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan,
const struct rte_dma_vchan_conf *conf, uint32_t conf_sz)
@@ -245,9 +502,14 @@ cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan,
RTE_SET_USED(conf_sz);
+ if (conf->auto_free.m2d.pool != NULL && conf->direction != RTE_DMA_DIR_MEM_TO_DEV)
+ return -EINVAL;
+
header = (union cnxk_dpi_instr_cmd *)&dpi_conf->cmd.u;
- if (dpivf->is_cn10k)
+ if (roc_model_is_cn20k())
+ ret = cn20k_dmadev_setup(dpivf, vchan, conf);
+ else if (roc_model_is_cn10k())
ret = cn10k_dmadev_setup_hdr(header, conf);
else
ret = cn9k_dmadev_setup_hdr(header, conf);
@@ -262,7 +524,7 @@ cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan,
if (!rte_is_power_of_2(max_desc))
max_desc = rte_align32pow2(max_desc);
- if (max_desc > CNXK_DPI_MAX_DESC)
+ if (!roc_model_is_cn20k() && (max_desc > CNXK_DPI_MAX_DESC))
max_desc = CNXK_DPI_MAX_DESC;
size = (max_desc * sizeof(uint8_t) * CNXK_DPI_COMPL_OFFSET);
@@ -285,10 +547,121 @@ cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan,
dpi_conf->c_desc.compl_ptr[i * CNXK_DPI_COMPL_OFFSET] = CNXK_DPI_REQ_CDATA;
dpi_conf->c_desc.max_cnt = (max_desc - 1);
+ dpi_conf->cfg_done = true;
return 0;
}
+static int
+cn20k_dmadev_queue_setup(struct cnxk_dpi_vf_s *dpivf)
+{
+ struct cn20k_ring_conf *rconf = dpivf->ring_conf;
+ struct roc_dpi *rdpi = &dpivf->rdpi;
+ struct cnxk_dpi_conf *dpi_conf;
+ struct roc_dpi_lf_que *que;
+ struct roc_dpi_lf_ring_cfg cfg;
+ struct roc_dpi_lf *lf;
+ uint16_t idx, qidx, ridx, vchan;
+ int rc = 0;
+
+ for (idx = 0; idx < rdpi->nr_lfs; idx++) {
+ lf = &rdpi->lfs[idx];
+ lf->chan_tbl = dpivf->chan_tbl;
+
+ for (qidx = 0; qidx < 2; qidx++) {
+ ridx = (idx << 1) + qidx;
+
+ if (!rconf[ridx].used)
+ continue;
+
+ que = &lf->queue[qidx];
+
+ que->qsize = rconf[ridx].num_desc;
+ que->cmd_len = DPI_CMD_SIZE_128B;
+ que->first_skip = 0;
+ que->later_skip = 0;
+ memset(&cfg, 0, sizeof(cfg));
+ cfg.ring_idx = qidx;
+ cfg.isize = que->cmd_len / DPI_CMD_SIZE_128B;
+ cfg.xtype = DPI_XTYPE_INTERNAL_ONLY;
+ cfg.rport = 0; /* Default is PEM:0 */
+ cfg.wport = 0; /* Default is PEM:0 */
+ cfg.pri = 0; /* 0 - High Priority */
+ for (vchan = 0; vchan < dpivf->num_vchans; vchan++) {
+ dpi_conf = &dpivf->conf[vchan];
+
+ if (dpi_conf->ridx == ridx)
+ cfg.xtype = dpi_conf->cfg.xtype;
+ }
+
+ rc = roc_dpi_lf_ring_init(que, &cfg);
+ if (rc)
+ return rc;
+
+ for (vchan = 0; vchan < dpivf->num_vchans; vchan++) {
+ dpi_conf = &dpivf->conf[vchan];
+
+ if (dpi_conf->ridx == ridx) {
+ dpi_conf->que = que;
+ dpi_conf->dbell = lf->rbase + DPI_LF_RINGX_WIDX(qidx);
+ }
+ }
+ }
+
+ rc = roc_dpi_lf_chan_tbl_select(lf);
+ if (rc)
+ return rc;
+
+ /* FIX ME */
+ rc = roc_dpi_lf_pffunc_cfg(lf);
+ if (rc)
+ return rc;
+ }
+
+ return rc;
+}
+
+static int
+cn20k_dmadev_chan_tbl_setup(struct cnxk_dpi_vf_s *dpivf)
+{
+ uint16_t vchan, idx = 0, offset = 0;
+ struct cnxk_dpi_conf *dpi_conf;
+ uint64_t config[64] = {0};
+ struct roc_dpi_lf *lf;
+ int rc = 0;
+
+ lf = &(dpivf->rdpi.lfs[0]);
+
+ for (vchan = 0; vchan < dpivf->num_vchans; vchan++) {
+ dpi_conf = &dpivf->conf[vchan];
+ if (!dpi_conf->cfg_done || (dpi_conf->cfg.xtype == DPI_XTYPE_INTERNAL_ONLY))
+ continue;
+
+ dpi_conf->cmd.cn20k.chan = idx;
+ config[idx++] = dpi_conf->chan_cfg.u;
+
+ if (idx == 64) {
+ rc = roc_dpi_lf_chan_tbl_update(lf, config, offset, idx);
+ if (rc < 0) {
+ plt_err("Failed to update chan tbl = %d", rc);
+ return rc;
+ }
+ offset += idx;
+ idx = 0;
+ }
+ }
+
+ if (idx) {
+ rc = roc_dpi_lf_chan_tbl_update(lf, config, offset, idx);
+ if (rc < 0) {
+ plt_err("Failed to update chan tbl = %d", rc);
+ return rc;
+ }
+ }
+
+ return rc;
+}
+
static int
cnxk_dmadev_start(struct rte_dma_dev *dev)
{
@@ -306,6 +679,10 @@ cnxk_dmadev_start(struct rte_dma_dev *dev)
dpi_conf->c_desc.head = 0;
dpi_conf->c_desc.tail = 0;
dpi_conf->desc_idx = 0;
+
+ if (dpi_conf->c_desc.compl_ptr == NULL)
+ continue;
+
for (j = 0; j < dpi_conf->c_desc.max_cnt + 1; j++)
dpi_conf->c_desc.compl_ptr[j * CNXK_DPI_COMPL_OFFSET] = CNXK_DPI_REQ_CDATA;
nb_desc += dpi_conf->c_desc.max_cnt + 1;
@@ -313,6 +690,27 @@ cnxk_dmadev_start(struct rte_dma_dev *dev)
dpi_conf->completed_offset = 0;
}
+ if (roc_model_is_cn20k()) {
+ for (i = 0; i < (dpivf->rdpi.nr_lfs << 1); i++)
+ dpivf->ring_conf[i].pending = 0;
+ roc_dpi_reset(&dpivf->rdpi);
+
+ if (dpivf->is_ring_conf_done)
+ goto enable_dpi;
+
+ rc = cn20k_dmadev_queue_setup(dpivf);
+ if (rc)
+ goto error;
+
+ rc = cn20k_dmadev_chan_tbl_setup(dpivf);
+ if (rc)
+ goto error;
+
+ dpivf->is_ring_conf_done = true;
+
+ goto enable_dpi;
+ }
+
queue_buf_sz = CNXK_DPI_QUEUE_BUF_SIZE_V2;
/* Max block size allowed by cnxk mempool driver is (128 * 1024).
* Block size = elt_size + mp->header + mp->trailer.
@@ -349,6 +747,7 @@ cnxk_dmadev_start(struct rte_dma_dev *dev)
dpivf->chunk_head = 0;
dpivf->chunk_size_m1 = (queue_buf_sz >> 3) - 2;
+enable_dpi:
roc_dpi_enable(&dpivf->rdpi);
error:
return rc;
@@ -359,14 +758,18 @@ cnxk_dmadev_stop(struct rte_dma_dev *dev)
{
struct cnxk_dpi_vf_s *dpivf = dev->fp_obj->dev_private;
- if (roc_dpi_wait_queue_idle(&dpivf->rdpi))
- return -EAGAIN;
+ if (!roc_model_is_cn20k()) {
+ if (roc_dpi_wait_queue_idle(&dpivf->rdpi))
+ return -EAGAIN;
+ }
roc_dpi_disable(&dpivf->rdpi);
- rte_mempool_free(dpivf->chunk_pool);
- dpivf->chunk_pool = NULL;
- dpivf->chunk_base = NULL;
- dpivf->chunk_size_m1 = 0;
+ if (!roc_model_is_cn20k()) {
+ rte_mempool_free(dpivf->chunk_pool);
+ dpivf->chunk_pool = NULL;
+ dpivf->chunk_base = NULL;
+ dpivf->chunk_size_m1 = 0;
+ }
return 0;
}
@@ -378,6 +781,10 @@ cnxk_dmadev_close(struct rte_dma_dev *dev)
roc_dpi_disable(&dpivf->rdpi);
cnxk_dmadev_vchan_free(dpivf, RTE_DMA_ALL_VCHAN);
+
+ if (roc_model_is_cn20k())
+ cnxk_dmadev_vchan_rsrc_free(dpivf);
+
roc_dpi_dev_fini(&dpivf->rdpi);
/* Clear all flags as we close the device. */
@@ -454,11 +861,17 @@ cnxk_damdev_burst_capacity(const void *dev_private, uint16_t vchan)
{
const struct cnxk_dpi_vf_s *dpivf = (const struct cnxk_dpi_vf_s *)dev_private;
const struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan];
+ struct cn20k_ring_conf *ring_conf;
uint16_t burst_cap;
burst_cap = dpi_conf->c_desc.max_cnt -
(dpi_conf->stats.submitted - dpi_conf->stats.completed) + 1;
+ if (roc_model_is_cn20k()) {
+ ring_conf = &dpivf->ring_conf[dpi_conf->ridx];
+ burst_cap -= ring_conf->pending;
+ }
+
return burst_cap;
}
@@ -480,6 +893,26 @@ cnxk_dmadev_submit(void *dev_private, uint16_t vchan)
return 0;
}
+static int
+cn20k_dmadev_submit(void *dev_private, uint16_t vchan)
+{
+ struct cnxk_dpi_vf_s *dpivf = dev_private;
+ struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan];
+ struct cn20k_ring_conf *ring_conf = &dpivf->ring_conf[dpi_conf->ridx];
+ uint16_t num_words = ring_conf->pending;
+
+ if (!num_words)
+ return 0;
+
+ rte_wmb();
+ plt_write64(num_words, dpi_conf->dbell);
+ dpi_conf->stats.submitted += num_words;
+
+ ring_conf->pending = 0;
+
+ return 0;
+}
+
static int
cnxk_stats_get(const struct rte_dma_dev *dev, uint16_t vchan, struct rte_dma_stats *rte_stats,
uint32_t size)
@@ -552,11 +985,16 @@ cnxk_set_fp_ops(struct rte_dma_dev *dev, uint8_t ena_enq_deq)
dev->fp_obj->submit = cnxk_dmadev_submit;
dev->fp_obj->completed = cnxk_dmadev_completed;
dev->fp_obj->completed_status = cnxk_dmadev_completed_status;
- dev->fp_obj->burst_capacity = cnxk_damdev_burst_capacity;
+ dev->fp_obj->burst_capacity = cnxk_damdev_burst_capacity;
if (roc_model_is_cn10k()) {
dev->fp_obj->copy = cn10k_dmadev_copy;
dev->fp_obj->copy_sg = cn10k_dmadev_copy_sg;
+ } else if (roc_model_is_cn20k()) {
+ dev->fp_obj->submit = cn20k_dmadev_submit;
+ dev->fp_obj->copy = cn20k_dmadev_copy;
+ dev->fp_obj->copy_sg = cn20k_dmadev_copy_sg;
+ dev->fp_obj->fill = cn20k_dmadev_fill;
}
if (ena_enq_deq) {
@@ -571,6 +1009,9 @@ cnxk_set_fp_ops(struct rte_dma_dev *dev, uint8_t ena_enq_deq)
if (roc_model_is_cn10k())
dev->fp_obj->enqueue = cn10k_dma_ops_enqueue;
+
+ if (roc_model_is_cn20k())
+ dev->fp_obj->enqueue = cn20k_dma_ops_enqueue;
}
}
@@ -594,8 +1035,10 @@ cnxk_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_de
struct roc_dpi *rdpi = NULL;
int rc;
- if (!pci_dev->mem_resource[0].addr)
- return -ENODEV;
+ if (!roc_model_is_cn20k()) {
+ if (!pci_dev->mem_resource[0].addr)
+ return -ENODEV;
+ }
rc = roc_plt_init();
if (rc) {
@@ -612,19 +1055,65 @@ cnxk_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_de
}
dpivf = dmadev->data->dev_private;
+ dmadev->device = &pci_dev->device;
+
+ if (roc_model_is_cn20k()) {
+ /* Parse devargs string */
+ rc = cn20k_dmadev_parse_devargs(dmadev->device->devargs, dpivf);
+ if (rc) {
+ plt_err("Failed to parse devargs rc=%d", rc);
+ goto err_out_free;
+ }
+ dpivf->num_vchans = dpivf->max_vchans;
+ }
dmadev->device = &pci_dev->device;
dmadev->fp_obj->dev_private = dpivf;
dmadev->dev_ops = &cnxk_dmadev_ops;
+ /*
+ * dev_private (and the roc_dpi it holds) lives in shared memory. Only
+ * the primary owns the HW and the per-process pci_dev pointer; a
+ * secondary must not run roc_dpi_dev_init nor overwrite rdpi->pci_dev,
+ * otherwise the primary later dereferences the secondary's stale
+ * pointer during rte_eal_cleanup() and crashes.
+ */
if (rte_eal_process_type() != RTE_PROC_PRIMARY)
return 0;
- dpivf->is_cn10k = roc_model_is_cn10k();
+ dmadev->fp_obj->submit = cnxk_dmadev_submit;
+ dmadev->fp_obj->completed = cnxk_dmadev_completed;
+ dmadev->fp_obj->completed_status = cnxk_dmadev_completed_status;
+ dmadev->fp_obj->burst_capacity = cnxk_damdev_burst_capacity;
+
+ if (roc_model_is_cn10k()) {
+ dmadev->fp_obj->copy = cn10k_dmadev_copy;
+ dmadev->fp_obj->copy_sg = cn10k_dmadev_copy_sg;
+ dpivf->num_vchans = CNXK_DPI_MAX_VCHANS_PER_QUEUE;
+ } else if (roc_model_is_cn20k()) {
+ dmadev->fp_obj->submit = cn20k_dmadev_submit;
+ dmadev->fp_obj->copy = cn20k_dmadev_copy;
+ dmadev->fp_obj->copy_sg = cn20k_dmadev_copy_sg;
+ dmadev->fp_obj->fill = cn20k_dmadev_fill;
+ } else {
+ dmadev->fp_obj->copy = cnxk_dmadev_copy;
+ dmadev->fp_obj->copy_sg = cnxk_dmadev_copy_sg;
+ dpivf->num_vchans = CNXK_DPI_MAX_VCHANS_PER_QUEUE;
+ }
+
+ dpivf->conf = rte_zmalloc("dpi_vchan_conf", dpivf->num_vchans *
+ sizeof(struct cnxk_dpi_conf), 0);
+ if (dpivf->conf == NULL) {
+ plt_err("Failed to allocate memory for vchan conf");
+ rc = -ENOMEM;
+ goto err_out_free;
+ }
+
dpivf->mcs_lock = NULL;
rdpi = &dpivf->rdpi;
rdpi->pci_dev = pci_dev;
+
rc = roc_dpi_dev_init(rdpi, offsetof(struct rte_dma_op, impl_opaque));
if (rc < 0)
goto err_out_free;
@@ -653,6 +1142,8 @@ cnxk_dmadev_remove(struct rte_pci_device *pci_dev)
static const struct rte_pci_id cnxk_dma_pci_map[] = {
{RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CNXK_DPI_VF)},
+ {RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN20K_DPI_PF)},
+ {RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN20K_DPI_VF)},
{
.vendor_id = 0,
},
diff --git a/drivers/dma/cnxk/cnxk_dmadev.h b/drivers/dma/cnxk/cnxk_dmadev.h
index 18039e43fb..3ba404dda8 100644
--- a/drivers/dma/cnxk/cnxk_dmadev.h
+++ b/drivers/dma/cnxk/cnxk_dmadev.h
@@ -9,9 +9,11 @@
#include <bus_pci_driver.h>
#include <rte_common.h>
+#include <rte_devargs.h>
#include <rte_dmadev.h>
#include <rte_dmadev_pmd.h>
#include <rte_eal.h>
+#include <rte_kvargs.h>
#include <rte_lcore.h>
#include <rte_mbuf_pool_ops.h>
#include <rte_mcslock.h>
@@ -31,15 +33,31 @@
#define CN10K_DPI_MAX_PRI 2
#define CNXK_DPI_MAX_VCHANS_PER_QUEUE 128
#define CNXK_DPI_QUEUE_BUF_SIZE 16256
-#define CNXK_DPI_QUEUE_BUF_SIZE_V2 130944
+/* Maximum pool size supported by device is 128 * 1024. When RTE_LIBRTE_MEMPOOL_DEBUG is enabled
+ * mempool->trailer size will be increased by 8B. Additionally if the pool is not created with
+ * RTE_MEMPOOL_F_NO_CACHE_ALIGN, trailer will be expanded to cache line size.
+ * To allow future needs, limit the max size to 127KB
+ */
+#define CNXK_DPI_QUEUE_BUF_SIZE_V2 130048
#define CNXK_DPI_POOL_MAX_CACHE_SZ (16)
#define CNXK_DPI_DW_PER_SINGLE_CMD 8
#define CNXK_DPI_HDR_LEN 4
#define CNXK_DPI_CMD_LEN(src, dst) (CNXK_DPI_HDR_LEN + ((src) << 1) + ((dst) << 1))
-#define CNXK_DPI_MAX_CMD_SZ CNXK_DPI_CMD_LEN(CNXK_DPI_MAX_POINTER, \
+#define CNXK_DPI_MAX_CMD_SZ CNXK_DPI_CMD_LEN(CNXK_DPI_MAX_POINTER, \
CNXK_DPI_MAX_POINTER)
#define CNXK_DPI_CHUNKS_FROM_DESC(cz, desc) (((desc) / (((cz) / 8) / CNXK_DPI_MAX_CMD_SZ)) + 1)
#define CNXK_DPI_COMPL_OFFSET ROC_CACHE_LINE_SZ
+
+#define CN20K_DPI_MAX_POINTER 4
+#define CN20K_DPI_MAX_DESC 2048
+#define CN20K_DPI_MIN_DESC 128
+#define CN20K_DPI_MAX_VCHANS 512
+#define CN20K_DPI_DEF_VCHANS 8
+#define CN20K_DPI_MAX_LFS 256
+
+#define CN20K_DPI_NUM_VCHANS "num_vchans"
+#define CN20K_DPI_NUM_LFS "num_lfs"
+
/* Set Completion data to 0xFF when request submitted,
* upon successful request completion engine reset to completion status
*/
@@ -86,6 +104,32 @@ union cnxk_dpi_instr_cmd {
uint64_t reserved_62_63 : 2;
/* Word 0 - End */
} cn10k;
+
+ struct cn20k_dpi_instr_cmd {
+ uint64_t nfst : 3;
+ uint64_t reserved_3 : 1;
+ uint64_t nlst : 3;
+ uint64_t reserved_7 : 1;
+ uint64_t msix_int : 1;
+ uint64_t ct : 3;
+ uint64_t chan : 14;
+ uint64_t reserved_26_29 : 4;
+ uint64_t aura : 20;
+ uint64_t xt : 2;
+ uint64_t ivec : 9;
+ uint64_t fe : 1;
+ uint64_t reserved_62 : 1;
+ uint64_t vld : 1;
+ /* Word 0 - End */
+ } cn20k;
+};
+
+struct cn20k_ring_conf {
+ enum rte_dma_direction direction;
+ uint16_t pending;
+ uint16_t num_desc;
+ uint8_t num_vchans;
+ bool used;
};
struct cnxk_dpi_cdesc_data_s {
@@ -100,9 +144,15 @@ struct cnxk_dpi_conf {
union cnxk_dpi_instr_cmd cmd;
struct cnxk_dpi_cdesc_data_s c_desc;
uint16_t desc_idx;
+ uintptr_t dbell;
struct rte_dma_stats stats;
uint64_t completed_offset;
+ struct roc_dpi_lf_que *que;
+ union roc_dpi_lf_ccfg chan_cfg;
+ struct roc_dpi_lf_ring_cfg cfg;
+ uint16_t ridx;
bool adapter_enabled;
+ bool cfg_done;
};
struct cnxk_dpi_vf_s {
@@ -111,15 +161,21 @@ struct cnxk_dpi_vf_s {
uint16_t chunk_head;
uint16_t chunk_size_m1;
uint16_t total_pnum_words;
+ uint16_t vchans_per_ring;
struct rte_mempool *chunk_pool;
- struct cnxk_dpi_conf conf[CNXK_DPI_MAX_VCHANS_PER_QUEUE];
+ struct cnxk_dpi_conf *conf;
+ struct cn20k_ring_conf *ring_conf;
RTE_ATOMIC(rte_mcslock_t *) mcs_lock;
/* Slow path */
struct roc_dpi rdpi;
uint32_t aura;
+ uint16_t max_lfs;
+ uint16_t max_vchans;
uint16_t num_vchans;
+ uint16_t chan_tbl;
uint16_t flag;
uint8_t is_cn10k;
+ uint8_t is_ring_conf_done;
} __plt_cache_aligned;
int cnxk_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t dst,
@@ -138,5 +194,13 @@ uint16_t cn10k_dma_ops_enqueue(void *dev_private, uint16_t vchan, struct rte_dma
uint16_t nb_ops);
uint16_t cnxk_dma_ops_dequeue(void *dev_private, uint16_t vchan, struct rte_dma_op **ops,
uint16_t nb_ops);
-
+int cn20k_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t dst,
+ uint32_t length, uint64_t flags);
+int cn20k_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge *src,
+ const struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst,
+ uint64_t flags);
+int cn20k_dmadev_fill(void *dev_private, uint16_t vchan, uint64_t pattern, rte_iova_t dst,
+ uint32_t length, uint64_t flags);
+uint16_t cn20k_dma_ops_enqueue(void *dev_private, uint16_t vchan, struct rte_dma_op **ops,
+ uint16_t nb_ops);
#endif
diff --git a/drivers/dma/cnxk/cnxk_dmadev_fp.c b/drivers/dma/cnxk/cnxk_dmadev_fp.c
index 4435adc38b..29ce13c8e4 100644
--- a/drivers/dma/cnxk/cnxk_dmadev_fp.c
+++ b/drivers/dma/cnxk/cnxk_dmadev_fp.c
@@ -440,6 +440,163 @@ cn10k_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge
return dpi_conf->desc_idx++;
}
+int
+cn20k_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t dst,
+ uint32_t length, uint64_t flags)
+{
+ struct cnxk_dpi_vf_s *dpivf = dev_private;
+ struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan];
+ const uint16_t max_cnt = dpi_conf->c_desc.max_cnt;
+ struct roc_dpi_lf_que *queue = dpi_conf->que;
+ struct cn20k_ring_conf *ring_conf;
+ uint8_t *comp_ptr;
+ uint64_t *cmd;
+
+ if (unlikely(((dpi_conf->c_desc.tail + 1) & max_cnt) == (dpi_conf->c_desc.head & max_cnt)))
+ return -ENOSPC;
+
+ if (dpivf->vchans_per_ring == 1) {
+ cmd = queue->cmd_base + ((dpi_conf->c_desc.tail & max_cnt) << 4);
+ } else {
+ cmd = queue->cmd_base + (queue->widx << 4);
+ queue->widx = (queue->widx + 1) & (queue->qsize - 1);
+ }
+
+ ring_conf = &(dpivf->ring_conf[dpi_conf->ridx]);
+ comp_ptr = &dpi_conf->c_desc
+ .compl_ptr[(dpi_conf->c_desc.tail & max_cnt) * CNXK_DPI_COMPL_OFFSET];
+ dpi_conf->c_desc.tail++;
+
+ cmd[1] = (uint64_t)comp_ptr;
+ cmd[4] = ((uint64_t)length << 32) | length | ((flags & RTE_DMA_OP_FLAG_AUTO_FREE) << 28);
+ cmd[5] = src;
+ cmd[6] = dst;
+ cmd[0] = DPI_CMD_VLD_BIT | dpi_conf->cmd.u | 0x11U;
+
+ if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+ rte_wmb();
+ plt_write64(ring_conf->pending + 1, dpi_conf->dbell);
+ dpi_conf->stats.submitted += (ring_conf->pending + 1);
+ ring_conf->pending = 0;
+ } else {
+ ring_conf->pending++;
+ }
+
+ return dpi_conf->desc_idx++;
+}
+
+/* Helper macro to write length and address */
+#define DPI_WRITE_SEGMENT(ptr, seg, i, idx, eidx, tmp) \
+ do { \
+ if ((tmp) % 2 == 0) { \
+ ptr[eidx] = seg[i].length; \
+ idx++; \
+ } else { \
+ ptr[eidx] |= ((uint64_t)seg[i].length << 32); \
+ eidx += 3; \
+ } \
+ ptr[idx++] = (uint64_t)seg[i].addr; \
+ tmp++; \
+ } while (0)
+
+int
+cn20k_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge *src,
+ const struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst,
+ uint64_t flags)
+{
+ struct cnxk_dpi_vf_s *dpivf = dev_private;
+ struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan];
+ const uint16_t max_cnt = dpi_conf->c_desc.max_cnt;
+ struct roc_dpi_lf_que *queue = dpi_conf->que;
+ struct cn20k_ring_conf *ring_conf = &dpivf->ring_conf[dpi_conf->ridx];
+ uint16_t idx = 4, eidx = 4, tmp = 0;
+ uint8_t *comp_ptr, i;
+ uint64_t *cmd;
+
+ if (unlikely(((dpi_conf->c_desc.tail + 1) & max_cnt) == (dpi_conf->c_desc.head & max_cnt)))
+ return -ENOSPC;
+
+ if (dpivf->vchans_per_ring == 1) {
+ cmd = queue->cmd_base + ((dpi_conf->c_desc.tail & max_cnt) << 4);
+ } else {
+ cmd = queue->cmd_base + (queue->widx << 4);
+ queue->widx = (queue->widx + 1) & (queue->qsize - 1);
+ }
+ comp_ptr = &dpi_conf->c_desc
+ .compl_ptr[(dpi_conf->c_desc.tail & max_cnt) * CNXK_DPI_COMPL_OFFSET];
+ dpi_conf->c_desc.tail++;
+
+ cmd[1] = (uint64_t)comp_ptr;
+
+ /* Fill source segments */
+ for (i = 0; i < nb_src; i++)
+ DPI_WRITE_SEGMENT(cmd, src, i, idx, eidx, tmp);
+
+ /* Fill destination segments */
+ for (i = 0; i < nb_dst; i++)
+ DPI_WRITE_SEGMENT(cmd, dst, i, idx, eidx, tmp);
+
+ cmd[0] = DPI_CMD_VLD_BIT | dpi_conf->cmd.u | (nb_dst << 4) | nb_src;
+
+ if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+ rte_wmb();
+ plt_write64(ring_conf->pending + 1, dpi_conf->dbell);
+ dpi_conf->stats.submitted += ring_conf->pending + 1;
+ ring_conf->pending = 0;
+ } else {
+ ring_conf->pending++;
+ }
+
+ return dpi_conf->desc_idx++;
+}
+
+int
+cn20k_dmadev_fill(void *dev_private, uint16_t vchan, uint64_t pattern, rte_iova_t dst,
+ uint32_t length, uint64_t flags)
+{
+ struct cnxk_dpi_vf_s *dpivf = dev_private;
+ struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan];
+ const uint16_t max_cnt = dpi_conf->c_desc.max_cnt;
+ struct roc_dpi_lf_que *queue = dpi_conf->que;
+ struct cn20k_ring_conf *ring_conf;
+ uint8_t *comp_ptr;
+ uint64_t *cmd;
+
+#define DPI_XT_TYPE_FILL BIT_ULL(50)
+
+ if (unlikely(((dpi_conf->c_desc.tail + 1) & max_cnt) == (dpi_conf->c_desc.head & max_cnt)))
+ return -ENOSPC;
+
+ if (dpivf->vchans_per_ring == 1) {
+ cmd = queue->cmd_base + ((dpi_conf->c_desc.tail & max_cnt) << 4);
+ } else {
+ cmd = queue->cmd_base + (queue->widx << 4);
+ queue->widx = (queue->widx + 1) & (queue->qsize - 1);
+ }
+
+ ring_conf = &(dpivf->ring_conf[dpi_conf->ridx]);
+ comp_ptr = &dpi_conf->c_desc
+ .compl_ptr[(dpi_conf->c_desc.tail & max_cnt) * CNXK_DPI_COMPL_OFFSET];
+ dpi_conf->c_desc.tail++;
+
+ cmd[1] = (uint64_t)comp_ptr;
+ cmd[4] = (uint64_t)length << 32;
+ cmd[5] = pattern;
+ cmd[6] = dst;
+ cmd[0] = DPI_CMD_VLD_BIT | DPI_XT_TYPE_FILL | dpi_conf->cmd.u | 0x10U;
+
+ if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+ rte_wmb();
+ plt_write64(ring_conf->pending + 1, dpi_conf->dbell);
+ dpi_conf->stats.submitted += (ring_conf->pending + 1);
+ ring_conf->pending = 0;
+ } else {
+ ring_conf->pending++;
+ }
+
+ return dpi_conf->desc_idx++;
+}
+
static inline uint64_t
cnxk_dma_adapter_format_event(uint64_t event)
{
@@ -450,6 +607,63 @@ cnxk_dma_adapter_format_event(uint64_t event)
return w0;
}
+uint16_t
+cn20k_dma_ops_enqueue(void *dev_private, uint16_t vchan, struct rte_dma_op **ops, uint16_t nb_ops)
+{
+ struct cnxk_dpi_vf_s *dpivf = dev_private;
+ struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan];
+ const uint16_t max_cnt = dpi_conf->c_desc.max_cnt;
+ struct roc_dpi_lf_que *queue = dpi_conf->que;
+ uint16_t idx, eidx, tmp;
+ struct rte_dma_op *op;
+ uint16_t space, i, j;
+ uint16_t src, dst;
+ uint8_t *comp_ptr;
+ uint64_t *cmd;
+
+ space = (max_cnt + dpi_conf->c_desc.head - dpi_conf->c_desc.tail) & max_cnt;
+ space = RTE_MIN(space, nb_ops);
+
+ for (j = 0; j < space; j++) {
+ op = ops[j];
+ src = op->nb_src;
+ dst = op->nb_dst;
+
+ idx = 4; eidx = 4; tmp = 0;
+
+ if (dpivf->vchans_per_ring == 1) {
+ cmd = queue->cmd_base + ((dpi_conf->c_desc.tail & max_cnt) << 4);
+ } else {
+ cmd = queue->cmd_base + (queue->widx << 4);
+ queue->widx = (queue->widx + 1) & (queue->qsize - 1);
+ }
+ comp_ptr = &dpi_conf->c_desc.compl_ptr[(dpi_conf->c_desc.tail & max_cnt) *
+ CNXK_DPI_COMPL_OFFSET];
+ dpi_conf->c_desc.ops[dpi_conf->c_desc.tail & max_cnt] = op;
+ dpi_conf->c_desc.tail++;
+
+ cmd[1] = (uint64_t)comp_ptr;
+
+ /* Fill source segments */
+ for (i = 0; i < src; i++)
+ DPI_WRITE_SEGMENT(cmd, op->src_dst_seg, i, idx, eidx, tmp);
+
+ /* Fill destination segments */
+ for (i = 0; i < dst; i++)
+ DPI_WRITE_SEGMENT(cmd, (op->src_dst_seg + src), i, idx, eidx, tmp);
+
+ cmd[0] = DPI_CMD_VLD_BIT | dpi_conf->cmd.u | (dst << 4) | src;
+ }
+
+ if (space) {
+ rte_wmb();
+ plt_write64(space, dpi_conf->dbell);
+ dpi_conf->stats.submitted += space;
+ }
+
+ return j;
+}
+
RTE_EXPORT_INTERNAL_SYMBOL(cn10k_dma_adapter_enqueue)
uint16_t
cn10k_dma_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events)
--
2.34.1
^ permalink raw reply related [flat|nested] 4+ messages in thread