* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-11-29 16:03 ` [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode Koichiro Den
@ 2025-12-01 4:38 ` kernel test robot
2025-12-01 5:49 ` kernel test robot
` (3 subsequent siblings)
4 siblings, 0 replies; 19+ messages in thread
From: kernel test robot @ 2025-12-01 4:38 UTC (permalink / raw)
To: Koichiro Den; +Cc: oe-kbuild-all
Hi Koichiro,
[This is a private test report for your RFC patch.]
kernel test robot noticed the following build warnings:
[auto build test WARNING on pci/next]
[also build test WARNING on next-20251128]
[cannot apply to jonmason-ntb/ntb-next pci/for-linus geert-renesas-devel/next linus/master v6.18]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Koichiro-Den/PCI-endpoint-pci-epf-vntb-Use-array_index_nospec-on-mws_size-access/20251130-001424
base: https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git next
patch link: https://lore.kernel.org/r/20251129160405.2568284-21-den%40valinux.co.jp
patch subject: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
config: riscv-randconfig-r071-20251201 (https://download.01.org/0day-ci/archive/20251201/202512011208.AnoqH1AC-lkp@intel.com/config)
compiler: clang version 22.0.0git (https://github.com/llvm/llvm-project b3428bb966f1de8aa48375ffee0eba04ede133b7)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251201/202512011208.AnoqH1AC-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202512011208.AnoqH1AC-lkp@intel.com/
All warnings (new ones prefixed by >>):
drivers/ntb/ntb_transport_core.c:1286:15: warning: variable 'mw_num' set but not used [-Wunused-but-set-variable]
1286 | unsigned int mw_num, mw_count, qp_count;
| ^
drivers/ntb/ntb_transport_core.c:1286:23: warning: variable 'mw_count' set but not used [-Wunused-but-set-variable]
1286 | unsigned int mw_num, mw_count, qp_count;
| ^
drivers/ntb/ntb_transport_core.c:1286:33: warning: variable 'qp_count' set but not used [-Wunused-but-set-variable]
1286 | unsigned int mw_num, mw_count, qp_count;
| ^
>> drivers/ntb/ntb_transport_core.c:1367:47: warning: unused variable 'edma_backend_ops' [-Wunused-const-variable]
1367 | static const struct ntb_transport_backend_ops edma_backend_ops;
| ^~~~~~~~~~~~~~~~
4 warnings generated.
Kconfig warnings: (for reference only)
WARNING: unmet direct dependencies detected for ARCH_HAS_ELF_CORE_EFLAGS
Depends on [n]: BINFMT_ELF [=y] && ELF_CORE [=n]
Selected by [y]:
- RISCV [=y]
smatch warnings:
drivers/ntb/ntb_transport_core.c:1425 ntb_transport_probe() warn: inconsistent indenting
drivers/ntb/ntb_transport_core.c:2412 ntb_transport_rx_enqueue() warn: variable dereferenced before check 'qp' (see line 2409)
drivers/ntb/ntb_transport_core.c:2451 ntb_transport_tx_enqueue() warn: variable dereferenced before check 'qp' (see line 2447)
vim +/edma_backend_ops +1367 drivers/ntb/ntb_transport_core.c
1281
1282 static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,
1283 unsigned int qp_num)
1284 {
1285 struct ntb_transport_qp *qp;
> 1286 unsigned int mw_num, mw_count, qp_count;
1287
1288 mw_count = nt->mw_count;
1289 qp_count = nt->qp_count;
1290
1291 mw_num = QP_TO_MW(nt, qp_num);
1292
1293 qp = &nt->qp_vec[qp_num];
1294 qp->qp_num = qp_num;
1295 qp->transport = nt;
1296 qp->ndev = nt->ndev;
1297 qp->client_ready = false;
1298 qp->event_handler = NULL;
1299 ntb_qp_link_context_reset(qp);
1300
1301 if (nt->debugfs_node_dir) {
1302 char debugfs_name[8];
1303
1304 snprintf(debugfs_name, sizeof(debugfs_name), "qp%d", qp_num);
1305 qp->debugfs_dir = debugfs_create_dir(debugfs_name,
1306 nt->debugfs_node_dir);
1307
1308 qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR,
1309 qp->debugfs_dir, qp,
1310 &ntb_qp_debugfs_stats_fops);
1311 } else {
1312 qp->debugfs_dir = NULL;
1313 qp->debugfs_stats = NULL;
1314 }
1315
1316 INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work);
1317 INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work);
1318
1319 spin_lock_init(&qp->ntb_rx_q_lock);
1320 spin_lock_init(&qp->ntb_tx_free_q_lock);
1321
1322 INIT_LIST_HEAD(&qp->rx_post_q);
1323 INIT_LIST_HEAD(&qp->rx_pend_q);
1324 INIT_LIST_HEAD(&qp->rx_free_q);
1325 INIT_LIST_HEAD(&qp->tx_free_q);
1326
1327 tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db,
1328 (unsigned long)qp);
1329
1330 return 0;
1331 }
1332
1333 static unsigned int ntb_transport_default_tx_free_entry(struct ntb_transport_qp *qp)
1334 {
1335 unsigned int head = qp->tx_index;
1336 unsigned int tail = qp->remote_rx_info->entry;
1337
1338 return tail >= head ? tail - head : qp->tx_max_entry + tail - head;
1339 }
1340
1341 static int ntb_transport_default_rx_enqueue(struct ntb_transport_qp *qp,
1342 struct ntb_queue_entry *entry)
1343 {
1344 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q);
1345
1346 if (qp->active)
1347 tasklet_schedule(&qp->rxc_db_work);
1348
1349 return 0;
1350 }
1351
1352 static void ntb_transport_default_rx_poll(struct ntb_transport_qp *qp);
1353 static int ntb_transport_default_tx_enqueue(struct ntb_transport_qp *qp,
1354 struct ntb_queue_entry *entry,
1355 void *cb, void *data, unsigned int len,
1356 unsigned int flags);
1357
1358 static const struct ntb_transport_backend_ops default_backend_ops = {
1359 .setup_qp_mw = ntb_transport_default_setup_qp_mw,
1360 .tx_free_entry = ntb_transport_default_tx_free_entry,
1361 .tx_enqueue = ntb_transport_default_tx_enqueue,
1362 .rx_enqueue = ntb_transport_default_rx_enqueue,
1363 .rx_poll = ntb_transport_default_rx_poll,
1364 .debugfs_stats_show = ntb_transport_default_debugfs_stats_show,
1365 };
1366
> 1367 static const struct ntb_transport_backend_ops edma_backend_ops;
1368
1369 static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
1370 {
1371 struct ntb_transport_ctx *nt;
1372 struct ntb_transport_mw *mw;
1373 unsigned int mw_count, qp_count, spad_count, max_mw_count_for_spads;
1374 u64 qp_bitmap;
1375 int node;
1376 int rc, i;
1377
1378 mw_count = ntb_peer_mw_count(ndev);
1379
1380 if (!ndev->ops->mw_set_trans) {
1381 dev_err(&ndev->dev, "Inbound MW based NTB API is required\n");
1382 return -EINVAL;
1383 }
1384
1385 if (ntb_db_is_unsafe(ndev))
1386 dev_dbg(&ndev->dev,
1387 "doorbell is unsafe, proceed anyway...\n");
1388 if (ntb_spad_is_unsafe(ndev))
1389 dev_dbg(&ndev->dev,
1390 "scratchpad is unsafe, proceed anyway...\n");
1391
1392 if (ntb_peer_port_count(ndev) != NTB_DEF_PEER_CNT)
1393 dev_warn(&ndev->dev, "Multi-port NTB devices unsupported\n");
1394
1395 node = dev_to_node(&ndev->dev);
1396
1397 nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node);
1398 if (!nt)
1399 return -ENOMEM;
1400
1401 nt->ndev = ndev;
1402
1403 #ifdef CONFIG_NTB_TRANSPORT_EDMA
1404 if (use_remote_edma) {
1405 rc = ntb_transport_edma_init(nt, &mw_count);
1406 if (rc) {
1407 nt->mw_count = 0;
1408 goto err;
1409 }
1410 nt->backend_ops = edma_backend_ops;
1411
1412 /*
1413 * On remote eDMA mode, we reserve a read channel for Host->EP
1414 * interruption.
1415 */
1416 use_msi = false;
1417 } else
1418 #endif
1419 nt->backend_ops = default_backend_ops;
1420
1421 /*
1422 * If we are using MSI, and have at least one extra memory window,
1423 * we will reserve the last MW for the MSI window.
1424 */
> 1425 if (use_msi && mw_count > 1) {
1426 rc = ntb_msi_init(ndev, ntb_transport_msi_desc_changed);
1427 if (!rc) {
1428 mw_count -= 1;
1429 nt->use_msi = true;
1430 }
1431 }
1432
1433 spad_count = ntb_spad_count(ndev);
1434
1435 /* Limit the MW's based on the availability of scratchpads */
1436
1437 if (spad_count < NTB_TRANSPORT_MIN_SPADS) {
1438 nt->mw_count = 0;
1439 rc = -EINVAL;
1440 goto err;
1441 }
1442
1443 max_mw_count_for_spads = (spad_count - MW0_SZ_HIGH) / 2;
1444 nt->mw_count = min(mw_count, max_mw_count_for_spads);
1445
1446 nt->msi_spad_offset = nt->mw_count * 2 + MW0_SZ_HIGH;
1447
1448 nt->mw_vec = kcalloc_node(mw_count, sizeof(*nt->mw_vec),
1449 GFP_KERNEL, node);
1450 if (!nt->mw_vec) {
1451 rc = -ENOMEM;
1452 goto err;
1453 }
1454
1455 for (i = 0; i < mw_count; i++) {
1456 mw = &nt->mw_vec[i];
1457
1458 rc = ntb_peer_mw_get_addr(ndev, i, &mw->phys_addr,
1459 &mw->phys_size);
1460 if (rc)
1461 goto err1;
1462
1463 mw->vbase = ioremap_wc(mw->phys_addr, mw->phys_size);
1464 if (!mw->vbase) {
1465 rc = -ENOMEM;
1466 goto err1;
1467 }
1468
1469 mw->buff_size = 0;
1470 mw->xlat_size = 0;
1471 mw->virt_addr = NULL;
1472 mw->dma_addr = 0;
1473 }
1474
1475 qp_bitmap = ntb_db_valid_mask(ndev);
1476
1477 qp_count = ilog2(qp_bitmap);
1478 if (nt->use_msi) {
1479 qp_count -= 1;
1480 nt->msi_db_mask = BIT_ULL(qp_count);
1481 ntb_db_clear_mask(ndev, nt->msi_db_mask);
1482 }
1483
1484 if (max_num_clients && max_num_clients < qp_count)
1485 qp_count = max_num_clients;
1486
1487 qp_bitmap &= BIT_ULL(qp_count) - 1;
1488
1489 nt->qp_count = qp_count;
1490 nt->qp_bitmap = qp_bitmap;
1491 nt->qp_bitmap_free = qp_bitmap;
1492
1493 nt->qp_vec = kcalloc_node(qp_count, sizeof(*nt->qp_vec),
1494 GFP_KERNEL, node);
1495 if (!nt->qp_vec) {
1496 rc = -ENOMEM;
1497 goto err1;
1498 }
1499
1500 if (nt_debugfs_dir) {
1501 nt->debugfs_node_dir =
1502 debugfs_create_dir(pci_name(ndev->pdev),
1503 nt_debugfs_dir);
1504 }
1505
1506 for (i = 0; i < qp_count; i++) {
1507 rc = ntb_transport_init_queue(nt, i);
1508 if (rc)
1509 goto err2;
1510
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-11-29 16:03 ` [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode Koichiro Den
2025-12-01 4:38 ` kernel test robot
@ 2025-12-01 5:49 ` kernel test robot
2025-12-01 18:45 ` kernel test robot
` (2 subsequent siblings)
4 siblings, 0 replies; 19+ messages in thread
From: kernel test robot @ 2025-12-01 5:49 UTC (permalink / raw)
To: Koichiro Den; +Cc: oe-kbuild-all
Hi Koichiro,
[This is a private test report for your RFC patch.]
kernel test robot noticed the following build warnings:
[auto build test WARNING on pci/next]
[also build test WARNING on next-20251128]
[cannot apply to jonmason-ntb/ntb-next pci/for-linus geert-renesas-devel/next linus/master v6.18]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Koichiro-Den/PCI-endpoint-pci-epf-vntb-Use-array_index_nospec-on-mws_size-access/20251130-001424
base: https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git next
patch link: https://lore.kernel.org/r/20251129160405.2568284-21-den%40valinux.co.jp
patch subject: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
config: powerpc-randconfig-001-20251201 (https://download.01.org/0day-ci/archive/20251201/202512011355.xYtxLg6d-lkp@intel.com/config)
compiler: powerpc-linux-gcc (GCC) 13.4.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251201/202512011355.xYtxLg6d-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202512011355.xYtxLg6d-lkp@intel.com/
All warnings (new ones prefixed by >>):
drivers/ntb/ntb_transport_core.c: In function 'ntb_transport_init_queue':
>> drivers/ntb/ntb_transport_core.c:1286:40: warning: variable 'qp_count' set but not used [-Wunused-but-set-variable]
1286 | unsigned int mw_num, mw_count, qp_count;
| ^~~~~~~~
>> drivers/ntb/ntb_transport_core.c:1286:30: warning: variable 'mw_count' set but not used [-Wunused-but-set-variable]
1286 | unsigned int mw_num, mw_count, qp_count;
| ^~~~~~~~
>> drivers/ntb/ntb_transport_core.c:1286:22: warning: variable 'mw_num' set but not used [-Wunused-but-set-variable]
1286 | unsigned int mw_num, mw_count, qp_count;
| ^~~~~~
drivers/ntb/ntb_transport_core.c: At top level:
>> drivers/ntb/ntb_transport_core.c:1367:47: warning: 'edma_backend_ops' defined but not used [-Wunused-const-variable=]
1367 | static const struct ntb_transport_backend_ops edma_backend_ops;
| ^~~~~~~~~~~~~~~~
vim +/qp_count +1286 drivers/ntb/ntb_transport_core.c
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1281
e26a5843f7f501 drivers/ntb/ntb_transport.c Allen Hubbe 2015-04-09 1282 static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1283 unsigned int qp_num)
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1284 {
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1285 struct ntb_transport_qp *qp;
e26a5843f7f501 drivers/ntb/ntb_transport.c Allen Hubbe 2015-04-09 @1286 unsigned int mw_num, mw_count, qp_count;
948d3a65b6d164 drivers/ntb/ntb_transport.c Jon Mason 2013-04-18 1287
e26a5843f7f501 drivers/ntb/ntb_transport.c Allen Hubbe 2015-04-09 1288 mw_count = nt->mw_count;
e26a5843f7f501 drivers/ntb/ntb_transport.c Allen Hubbe 2015-04-09 1289 qp_count = nt->qp_count;
e26a5843f7f501 drivers/ntb/ntb_transport.c Allen Hubbe 2015-04-09 1290
e26a5843f7f501 drivers/ntb/ntb_transport.c Allen Hubbe 2015-04-09 1291 mw_num = QP_TO_MW(nt, qp_num);
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1292
e26a5843f7f501 drivers/ntb/ntb_transport.c Allen Hubbe 2015-04-09 1293 qp = &nt->qp_vec[qp_num];
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1294 qp->qp_num = qp_num;
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1295 qp->transport = nt;
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1296 qp->ndev = nt->ndev;
e26a5843f7f501 drivers/ntb/ntb_transport.c Allen Hubbe 2015-04-09 1297 qp->client_ready = false;
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1298 qp->event_handler = NULL;
cc79bd2738c2d4 drivers/ntb/ntb_transport.c Dave Jiang 2023-08-22 1299 ntb_qp_link_context_reset(qp);
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1300
c8650fd03d320e drivers/ntb/ntb_transport.c Dave Jiang 2015-07-13 1301 if (nt->debugfs_node_dir) {
2d1987992b40a8 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1302 char debugfs_name[8];
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1303
2d1987992b40a8 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1304 snprintf(debugfs_name, sizeof(debugfs_name), "qp%d", qp_num);
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1305 qp->debugfs_dir = debugfs_create_dir(debugfs_name,
c8650fd03d320e drivers/ntb/ntb_transport.c Dave Jiang 2015-07-13 1306 nt->debugfs_node_dir);
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1307
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1308 qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR,
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1309 qp->debugfs_dir, qp,
2d1987992b40a8 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1310 &ntb_qp_debugfs_stats_fops);
e26a5843f7f501 drivers/ntb/ntb_transport.c Allen Hubbe 2015-04-09 1311 } else {
e26a5843f7f501 drivers/ntb/ntb_transport.c Allen Hubbe 2015-04-09 1312 qp->debugfs_dir = NULL;
e26a5843f7f501 drivers/ntb/ntb_transport.c Allen Hubbe 2015-04-09 1313 qp->debugfs_stats = NULL;
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1314 }
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1315
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1316 INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work);
fca4d5188c1123 drivers/ntb/ntb_transport.c Jon Mason 2013-09-09 1317 INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work);
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1318
da2e5ae56164b8 drivers/ntb/ntb_transport.c Allen Hubbe 2015-07-13 1319 spin_lock_init(&qp->ntb_rx_q_lock);
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1320 spin_lock_init(&qp->ntb_tx_free_q_lock);
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1321
da2e5ae56164b8 drivers/ntb/ntb_transport.c Allen Hubbe 2015-07-13 1322 INIT_LIST_HEAD(&qp->rx_post_q);
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1323 INIT_LIST_HEAD(&qp->rx_pend_q);
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1324 INIT_LIST_HEAD(&qp->rx_free_q);
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1325 INIT_LIST_HEAD(&qp->tx_free_q);
282a2feeb9bfb1 drivers/ntb/ntb_transport.c Jon Mason 2013-02-12 1326
e26a5843f7f501 drivers/ntb/ntb_transport.c Allen Hubbe 2015-04-09 1327 tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db,
e26a5843f7f501 drivers/ntb/ntb_transport.c Allen Hubbe 2015-04-09 1328 (unsigned long)qp);
e26a5843f7f501 drivers/ntb/ntb_transport.c Allen Hubbe 2015-04-09 1329
282a2feeb9bfb1 drivers/ntb/ntb_transport.c Jon Mason 2013-02-12 1330 return 0;
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1331 }
fce8a7bb5b4bfb drivers/ntb/ntb_transport.c Jon Mason 2012-11-16 1332
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1333 static unsigned int ntb_transport_default_tx_free_entry(struct ntb_transport_qp *qp)
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1334 {
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1335 unsigned int head = qp->tx_index;
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1336 unsigned int tail = qp->remote_rx_info->entry;
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1337
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1338 return tail >= head ? tail - head : qp->tx_max_entry + tail - head;
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1339 }
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1340
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1341 static int ntb_transport_default_rx_enqueue(struct ntb_transport_qp *qp,
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1342 struct ntb_queue_entry *entry)
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1343 {
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1344 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q);
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1345
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1346 if (qp->active)
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1347 tasklet_schedule(&qp->rxc_db_work);
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1348
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1349 return 0;
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1350 }
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1351
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1352 static void ntb_transport_default_rx_poll(struct ntb_transport_qp *qp);
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1353 static int ntb_transport_default_tx_enqueue(struct ntb_transport_qp *qp,
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1354 struct ntb_queue_entry *entry,
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1355 void *cb, void *data, unsigned int len,
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1356 unsigned int flags);
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1357
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1358 static const struct ntb_transport_backend_ops default_backend_ops = {
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1359 .setup_qp_mw = ntb_transport_default_setup_qp_mw,
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1360 .tx_free_entry = ntb_transport_default_tx_free_entry,
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1361 .tx_enqueue = ntb_transport_default_tx_enqueue,
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1362 .rx_enqueue = ntb_transport_default_rx_enqueue,
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1363 .rx_poll = ntb_transport_default_rx_poll,
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1364 .debugfs_stats_show = ntb_transport_default_debugfs_stats_show,
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1365 };
5c2671d8c2c274 drivers/ntb/ntb_transport.c Koichiro Den 2025-11-30 1366
2c52b6a2a63cec drivers/ntb/ntb_transport_core.c Koichiro Den 2025-11-30 @1367 static const struct ntb_transport_backend_ops edma_backend_ops;
2c52b6a2a63cec drivers/ntb/ntb_transport_core.c Koichiro Den 2025-11-30 1368
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-11-29 16:03 ` [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode Koichiro Den
2025-12-01 4:38 ` kernel test robot
2025-12-01 5:49 ` kernel test robot
@ 2025-12-01 18:45 ` kernel test robot
2025-12-01 21:41 ` Frank Li
2025-12-01 21:46 ` Dave Jiang
4 siblings, 0 replies; 19+ messages in thread
From: kernel test robot @ 2025-12-01 18:45 UTC (permalink / raw)
To: Koichiro Den; +Cc: oe-kbuild-all
Hi Koichiro,
[This is a private test report for your RFC patch.]
kernel test robot noticed the following build errors:
[auto build test ERROR on pci/next]
[also build test ERROR on next-20251201]
[cannot apply to jonmason-ntb/ntb-next pci/for-linus geert-renesas-devel/next linus/master v6.18]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Koichiro-Den/PCI-endpoint-pci-epf-vntb-Use-array_index_nospec-on-mws_size-access/20251130-001424
base: https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git next
patch link: https://lore.kernel.org/r/20251129160405.2568284-21-den%40valinux.co.jp
patch subject: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
config: mips-allyesconfig (https://download.01.org/0day-ci/archive/20251202/202512020042.oqBiIiDK-lkp@intel.com/config)
compiler: mips-linux-gcc (GCC) 15.1.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251202/202512020042.oqBiIiDK-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202512020042.oqBiIiDK-lkp@intel.com/
All error/warnings (new ones prefixed by >>):
drivers/ntb/ntb_transport_core.c: In function 'ntb_transport_init_queue':
drivers/ntb/ntb_transport_core.c:1286:40: warning: variable 'qp_count' set but not used [-Wunused-but-set-variable]
1286 | unsigned int mw_num, mw_count, qp_count;
| ^~~~~~~~
drivers/ntb/ntb_transport_core.c:1286:30: warning: variable 'mw_count' set but not used [-Wunused-but-set-variable]
1286 | unsigned int mw_num, mw_count, qp_count;
| ^~~~~~~~
drivers/ntb/ntb_transport_core.c:1286:22: warning: variable 'mw_num' set but not used [-Wunused-but-set-variable]
1286 | unsigned int mw_num, mw_count, qp_count;
| ^~~~~~
In file included from include/linux/device.h:15,
from include/linux/dmaengine.h:8,
from drivers/ntb/ntb_transport_core.c:55:
drivers/ntb/ntb_transport_core.c: In function 'ntb_transport_edma_setup_qp_mw':
>> drivers/ntb/ntb_transport_core.c:2837:37: warning: format '%lu' expects argument of type 'long unsigned int', but argument 3 has type 'size_t' {aka 'unsigned int'} [-Wformat=]
2837 | dev_err(&ndev->dev, "Ring space requires %luB (>=%uB)\n",
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
include/linux/dev_printk.h:110:30: note: in definition of macro 'dev_printk_index_wrap'
110 | _p_func(dev, fmt, ##__VA_ARGS__); \
| ^~~
include/linux/dev_printk.h:154:56: note: in expansion of macro 'dev_fmt'
154 | dev_printk_index_wrap(_dev_err, KERN_ERR, dev, dev_fmt(fmt), ##__VA_ARGS__)
| ^~~~~~~
drivers/ntb/ntb_transport_core.c:2837:17: note: in expansion of macro 'dev_err'
2837 | dev_err(&ndev->dev, "Ring space requires %luB (>=%uB)\n",
| ^~~~~~~
drivers/ntb/ntb_transport_core.c:2837:60: note: format string is defined here
2837 | dev_err(&ndev->dev, "Ring space requires %luB (>=%uB)\n",
| ~~^
| |
| long unsigned int
| %u
drivers/ntb/ntb_transport_core.c: In function 'ntb_transport_edma_ep_read_complete':
>> drivers/ntb/ntb_transport_core.c:2890:17: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
2890 | entry = (struct ntb_queue_entry *)(in->data);
| ^
drivers/ntb/ntb_transport_core.c: In function 'ntb_transport_edma_ep_write_complete':
drivers/ntb/ntb_transport_core.c:2927:17: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
2927 | entry = (struct ntb_queue_entry *)(in->data);
| ^
drivers/ntb/ntb_transport_core.c: In function 'ntb_transport_edma_rc_write_complete_work':
drivers/ntb/ntb_transport_core.c:2987:25: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
2987 | entry = (struct ntb_queue_entry *)READ_ONCE(in->data);
| ^
drivers/ntb/ntb_transport_core.c: In function 'ntb_transport_edma_rc_read_complete_work':
drivers/ntb/ntb_transport_core.c:3061:25: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
3061 | entry = (struct ntb_queue_entry *)in->data;
| ^
drivers/ntb/ntb_transport_core.c: In function 'ntb_transport_edma_ep_tx_enqueue':
>> drivers/ntb/ntb_transport_core.c:3427:17: error: implicit declaration of function 'iowrite64'; did you mean 'iowrite32'? [-Wimplicit-function-declaration]
3427 | iowrite64(ep_src, &out->addr);
| ^~~~~~~~~
| iowrite32
vim +3427 drivers/ntb/ntb_transport_core.c
3389
3390 static int ntb_transport_edma_ep_tx_enqueue(struct ntb_transport_qp *qp,
3391 struct ntb_queue_entry *entry)
3392 {
3393 struct device *dma_dev = get_dma_dev(qp->ndev);
3394 struct ntb_edma_desc *in, __iomem *out;
3395 unsigned int len = entry->len;
3396 dma_addr_t ep_src = 0;
3397 u32 idx;
3398 int rc;
3399
3400 if (likely(len)) {
3401 ep_src = dma_map_single(dma_dev, entry->buf, len,
3402 DMA_TO_DEVICE);
3403 rc = dma_mapping_error(dma_dev, ep_src);
3404 if (rc)
3405 return rc;
3406 }
3407
3408 scoped_guard(spinlock_irqsave, &qp->ep_tx_lock) {
3409 if (ntb_edma_ring_full(qp->wr_prod, qp->wr_cons)) {
3410 rc = -ENOSPC;
3411 qp->tx_ring_full++;
3412 goto out_unmap;
3413 }
3414
3415 idx = ntb_edma_ring_idx(qp->wr_prod);
3416 in = NTB_DESC_WR_EP_I(qp, idx);
3417 out = NTB_DESC_WR_EP_O(qp, idx);
3418
3419 WARN_ON(in->flags & DESC_DONE_FLAG);
3420 WARN_ON(entry->flags & DESC_DONE_FLAG);
3421 in->flags = 0;
3422 in->data = (uintptr_t)entry;
3423 entry->addr = ep_src;
3424
3425 iowrite32(len, &out->len);
3426 iowrite32(entry->flags, &out->flags);
> 3427 iowrite64(ep_src, &out->addr);
3428 WRITE_ONCE(qp->wr_prod, qp->wr_prod + 1);
3429
3430 dma_wmb();
3431 iowrite32(qp->wr_prod, NTB_HEAD_WR_EP_O(qp));
3432
3433 qp->tx_bytes += len;
3434 qp->tx_pkts++;
3435 }
3436
3437 ntb_transport_edma_notify_peer(qp);
3438
3439 return 0;
3440 out_unmap:
3441 if (likely(len))
3442 dma_unmap_single(dma_dev, ep_src, len, DMA_TO_DEVICE);
3443 return rc;
3444 }
3445
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-11-29 16:03 ` [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode Koichiro Den
` (2 preceding siblings ...)
2025-12-01 18:45 ` kernel test robot
@ 2025-12-01 21:41 ` Frank Li
2025-12-02 6:43 ` Koichiro Den
2025-12-01 21:46 ` Dave Jiang
4 siblings, 1 reply; 19+ messages in thread
From: Frank Li @ 2025-12-01 21:41 UTC (permalink / raw)
To: Koichiro Den
Cc: ntb, linux-pci, dmaengine, linux-kernel, mani, kwilczynski,
kishon, bhelgaas, corbet, vkoul, jdmason, dave.jiang, allenbh,
Basavaraj.Natikar, Shyam-sundar.S-k, kurt.schwemmer, logang,
jingoohan1, lpieralisi, robh, jbrunet, fancer.lancer, arnd,
pstanner, elfring
On Sun, Nov 30, 2025 at 01:03:58AM +0900, Koichiro Den wrote:
> Add a new transport backend that uses a remote DesignWare eDMA engine
> located on the NTB endpoint to move data between host and endpoint.
>
> In this mode:
>
> - The endpoint exposes a dedicated memory window that contains the
> eDMA register block followed by a small control structure (struct
> ntb_edma_info) and per-channel linked-list (LL) rings.
>
> - On the endpoint side, ntb_edma_setup_mws() allocates the control
> structure and LL rings in endpoint memory, then programs an inbound
> iATU region so that the host can access them via a peer MW.
>
> - On the host side, ntb_edma_setup_peer() ioremaps the peer MW, reads
> ntb_edma_info and configures a dw-edma DMA device to use the LL
> rings provided by the endpoint.
>
> - ntb_transport is extended with a new backend_ops implementation that
> routes TX and RX enqueue/poll operations through the remote eDMA
> rings while keeping the existing shared-memory backend intact.
>
> - The host signals the endpoint via a dedicated DMA read channel.
> 'use_msi' module option is ignored when 'use_remote_edma=1'.
>
> The new mode is guarded by a Kconfig option (NTB_TRANSPORT_EDMA) and a
> module parameter (use_remote_edma). When disabled, the existing
> ntb_transport behaviour is unchanged.
>
> Signed-off-by: Koichiro Den <den@valinux.co.jp>
> ---
> drivers/ntb/Kconfig | 11 +
> drivers/ntb/Makefile | 3 +
> drivers/ntb/ntb_edma.c | 628 ++++++++
> drivers/ntb/ntb_edma.h | 128 ++
> .../{ntb_transport.c => ntb_transport_core.c} | 1281 ++++++++++++++++-
> 5 files changed, 2048 insertions(+), 3 deletions(-)
> create mode 100644 drivers/ntb/ntb_edma.c
> create mode 100644 drivers/ntb/ntb_edma.h
> rename drivers/ntb/{ntb_transport.c => ntb_transport_core.c} (65%)
>
> diff --git a/drivers/ntb/Kconfig b/drivers/ntb/Kconfig
> index df16c755b4da..db63f02bb116 100644
> --- a/drivers/ntb/Kconfig
> +++ b/drivers/ntb/Kconfig
> @@ -37,4 +37,15 @@ config NTB_TRANSPORT
>
> If unsure, say N.
>
> +config NTB_TRANSPORT_EDMA
> + bool "NTB Transport backed by remote eDMA"
> + depends on NTB_TRANSPORT
> + depends on PCI
> + select DMA_ENGINE
> + help
> + Enable a transport backend that uses a remote DesignWare eDMA engine
> + exposed through a dedicated NTB memory window. The host uses the
> + endpoint's eDMA engine to move data in both directions.
> + Say Y here if you intend to use the 'use_remote_edma' module parameter.
> +
> endif # NTB
> diff --git a/drivers/ntb/Makefile b/drivers/ntb/Makefile
> index 3a6fa181ff99..51f0e1e3aec7 100644
> --- a/drivers/ntb/Makefile
> +++ b/drivers/ntb/Makefile
> @@ -4,3 +4,6 @@ obj-$(CONFIG_NTB_TRANSPORT) += ntb_transport.o
>
> ntb-y := core.o
> ntb-$(CONFIG_NTB_MSI) += msi.o
> +
> +ntb_transport-y := ntb_transport_core.o
> +ntb_transport-$(CONFIG_NTB_TRANSPORT_EDMA) += ntb_edma.o
> diff --git a/drivers/ntb/ntb_edma.c b/drivers/ntb/ntb_edma.c
> new file mode 100644
> index 000000000000..cb35e0d56aa8
> --- /dev/null
> +++ b/drivers/ntb/ntb_edma.c
> @@ -0,0 +1,628 @@
> +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
> +
> +#include <linux/module.h>
> +#include <linux/device.h>
> +#include <linux/pci.h>
> +#include <linux/ntb.h>
> +#include <linux/io.h>
> +#include <linux/iommu.h>
> +#include <linux/dmaengine.h>
> +#include <linux/pci-epc.h>
> +#include <linux/dma/edma.h>
> +#include <linux/irq.h>
> +#include <linux/irqdomain.h>
> +#include <linux/of.h>
> +#include <linux/of_irq.h>
> +#include <dt-bindings/interrupt-controller/arm-gic.h>
> +
> +#include "ntb_edma.h"
> +
> +/*
> + * The interrupt register offsets below are taken from the DesignWare
> + * eDMA "unrolled" register map (EDMA_MF_EDMA_UNROLL). The remote eDMA
> + * backend currently only supports this layout.
> + */
> +#define DMA_WRITE_INT_STATUS_OFF 0x4c
> +#define DMA_WRITE_INT_MASK_OFF 0x54
> +#define DMA_WRITE_INT_CLEAR_OFF 0x58
> +#define DMA_READ_INT_STATUS_OFF 0xa0
> +#define DMA_READ_INT_MASK_OFF 0xa8
> +#define DMA_READ_INT_CLEAR_OFF 0xac
Not sure why need access EDMA register because EMDA driver already export
as dmaengine driver.
> +
> +#define NTB_EDMA_NOTIFY_MAX_QP 64
> +
> +static unsigned int edma_spi = 417; /* 0x1a1 */
> +module_param(edma_spi, uint, 0644);
> +MODULE_PARM_DESC(edma_spi, "SPI number used by remote eDMA interrupt (EP local)");
> +
> +static u64 edma_regs_phys = 0xe65d5000;
> +module_param(edma_regs_phys, ullong, 0644);
> +MODULE_PARM_DESC(edma_regs_phys, "Physical base address of local eDMA registers (EP)");
> +
> +static unsigned long edma_regs_size = 0x1200;
> +module_param(edma_regs_size, ulong, 0644);
> +MODULE_PARM_DESC(edma_regs_size, "Size of the local eDMA register space (EP)");
> +
> +struct ntb_edma_intr {
> + u32 db[NTB_EDMA_NOTIFY_MAX_QP];
> +};
> +
> +struct ntb_edma_ctx {
> + void *ll_wr_virt[EDMA_WR_CH_NUM];
> + dma_addr_t ll_wr_phys[EDMA_WR_CH_NUM];
> + void *ll_rd_virt[EDMA_RD_CH_NUM + 1];
> + dma_addr_t ll_rd_phys[EDMA_RD_CH_NUM + 1];
> +
> + struct ntb_edma_intr *intr_ep_virt;
> + dma_addr_t intr_ep_phys;
> + struct ntb_edma_intr *intr_rc_virt;
> + dma_addr_t intr_rc_phys;
> + u32 notify_qp_max;
> +
> + bool initialized;
> +};
> +
> +static struct ntb_edma_ctx edma_ctx;
> +
> +typedef void (*ntb_edma_interrupt_cb_t)(void *data, int qp_num);
> +
> +struct ntb_edma_interrupt {
> + int virq;
> + void __iomem *base;
> + ntb_edma_interrupt_cb_t cb;
> + void *data;
> +};
> +
> +static struct ntb_edma_interrupt ntb_edma_intr;
> +
> +static int ntb_edma_map_spi_to_virq(struct device *dev, unsigned int spi)
> +{
> + struct device_node *np = dev_of_node(dev);
> + struct device_node *parent;
> + struct irq_fwspec fwspec = { 0 };
> + int virq;
> +
> + parent = of_irq_find_parent(np);
> + if (!parent)
> + return -ENODEV;
> +
> + fwspec.fwnode = of_fwnode_handle(parent);
> + fwspec.param_count = 3;
> + fwspec.param[0] = GIC_SPI;
> + fwspec.param[1] = spi;
> + fwspec.param[2] = IRQ_TYPE_LEVEL_HIGH;
> +
> + virq = irq_create_fwspec_mapping(&fwspec);
> + of_node_put(parent);
> + return (virq > 0) ? virq : -EINVAL;
> +}
> +
> +static irqreturn_t ntb_edma_isr(int irq, void *data)
> +{
Not sue why dw_edma_interrupt_write/read() does work for your case. Suppose
just register callback for dmeengine.
> + struct ntb_edma_interrupt *v = data;
> + u32 mask = BIT(EDMA_RD_CH_NUM);
> + u32 i, val;
> +
> + /*
> + * We do not ack interrupts here but instead we mask all local interrupt
> + * sources except the read channel used for notification. This reduces
> + * needless ISR invocations.
> + *
> + * In theory we could configure LIE=1/RIE=0 only for the notification
> + * transfer (keeping all other channels at LIE=1/RIE=1), but that would
> + * require intrusive changes to the dw-edma core.
> + *
> + * Note: The host side may have already cleared the read interrupt used
> + * for notification, so reading DMA_READ_INT_CLEAR_OFF is not a reliable
> + * way to detect it. As a result, we cannot reliably tell which specific
> + * channel triggered this interrupt. intr_ep_virt->db[i] teaches us
> + * instead.
> + */
> + iowrite32(~0x0, v->base + DMA_WRITE_INT_MASK_OFF);
> + iowrite32(~mask, v->base + DMA_READ_INT_MASK_OFF);
> +
> + if (!v->cb || !edma_ctx.intr_ep_virt)
> + return IRQ_HANDLED;
> +
> + for (i = 0; i < edma_ctx.notify_qp_max; i++) {
> + val = READ_ONCE(edma_ctx.intr_ep_virt->db[i]);
> + if (!val)
> + continue;
> +
> + WRITE_ONCE(edma_ctx.intr_ep_virt->db[i], 0);
> + v->cb(v->data, i);
> + }
> +
> + return IRQ_HANDLED;
> +}
> +
...
> +
> +int ntb_edma_setup_peer(struct ntb_dev *ndev)
> +{
> + struct ntb_edma_info *info;
> + unsigned int wr_cnt, rd_cnt;
> + struct dw_edma_chip *chip;
> + void __iomem *edma_virt;
> + phys_addr_t edma_phys;
> + resource_size_t mw_size;
> + u64 off = EDMA_REG_SIZE;
> + int peer_mw, mw_index;
> + unsigned int i;
> + int ret;
> +
> + peer_mw = ntb_peer_mw_count(ndev);
> + if (peer_mw <= 0)
> + return -ENODEV;
> +
> + mw_index = peer_mw - 1; /* last MW */
> +
> + ret = ntb_peer_mw_get_addr(ndev, mw_index, &edma_phys,
> + &mw_size);
> + if (ret)
> + return -1;
> +
> + edma_virt = ioremap(edma_phys, mw_size);
> +
> + chip = devm_kzalloc(&ndev->dev, sizeof(*chip), GFP_KERNEL);
> + if (!chip) {
> + ret = -ENOMEM;
> + return ret;
> + }
> +
> + chip->dev = &ndev->pdev->dev;
> + chip->nr_irqs = 4;
> + chip->ops = &ntb_edma_ops;
> + chip->flags = 0;
> + chip->reg_base = edma_virt;
> + chip->mf = EDMA_MF_EDMA_UNROLL;
> +
> + info = edma_virt + off;
> + if (info->magic != NTB_EDMA_INFO_MAGIC)
> + return -EINVAL;
> + wr_cnt = info->wr_cnt;
> + rd_cnt = info->rd_cnt;
> + chip->ll_wr_cnt = wr_cnt;
> + chip->ll_rd_cnt = rd_cnt;
> + off += PAGE_SIZE;
> +
> + edma_ctx.notify_qp_max = NTB_EDMA_NOTIFY_MAX_QP;
> + edma_ctx.intr_ep_phys = info->intr_dar_base;
> + if (edma_ctx.intr_ep_phys) {
> + edma_ctx.intr_rc_virt =
> + dma_alloc_coherent(&ndev->pdev->dev,
> + sizeof(struct ntb_edma_intr),
> + &edma_ctx.intr_rc_phys,
> + GFP_KERNEL);
> + if (!edma_ctx.intr_rc_virt)
> + return -ENOMEM;
> + memset(edma_ctx.intr_rc_virt, 0,
> + sizeof(struct ntb_edma_intr));
> + }
> +
> + for (i = 0; i < wr_cnt; i++) {
> + chip->ll_region_wr[i].vaddr.io = edma_virt + off;
> + chip->ll_region_wr[i].paddr = info->ll_wr_phys[i];
> + chip->ll_region_wr[i].sz = DMA_LLP_MEM_SIZE;
> + off += DMA_LLP_MEM_SIZE;
> + }
> + for (i = 0; i < rd_cnt; i++) {
> + chip->ll_region_rd[i].vaddr.io = edma_virt + off;
> + chip->ll_region_rd[i].paddr = info->ll_rd_phys[i];
> + chip->ll_region_rd[i].sz = DMA_LLP_MEM_SIZE;
> + off += DMA_LLP_MEM_SIZE;
> + }
> +
> + if (!pci_dev_msi_enabled(ndev->pdev))
> + return -ENXIO;
> +
> + ret = dw_edma_probe(chip);
I think dw_edma_probe() should be in ntb_hw_epf.c, which provide DMA
dma engine support.
EP side, suppose default dwc controller driver already setup edma engine,
so use correct filter function, you should get dma chan.
Frank
> + if (ret) {
> + dev_err(&ndev->dev, "dw_edma_probe failed: %d\n", ret);
> + return ret;
> + }
> +
> + return 0;
> +}
> +
> +struct ntb_edma_filter {
> + struct device *dma_dev;
> + u32 direction;
> +};
> +
> +static bool ntb_edma_filter_fn(struct dma_chan *chan, void *arg)
> +{
> + struct ntb_edma_filter *filter = arg;
> + u32 dir = filter->direction;
> + struct dma_slave_caps caps;
> + int ret;
> +
> + if (chan->device->dev != filter->dma_dev)
> + return false;
> +
> + ret = dma_get_slave_caps(chan, &caps);
> + if (ret < 0)
> + return false;
> +
> + return !!(caps.directions & dir);
> +}
> +
> +void ntb_edma_teardown_chans(struct ntb_edma_chans *edma)
> +{
> + unsigned int i;
> +
> + for (i = 0; i < edma->num_wr_chan; i++)
> + dma_release_channel(edma->wr_chan[i]);
> +
> + for (i = 0; i < edma->num_rd_chan; i++)
> + dma_release_channel(edma->rd_chan[i]);
> +
> + if (edma->intr_chan)
> + dma_release_channel(edma->intr_chan);
> +}
> +
> +int ntb_edma_setup_chans(struct device *dma_dev, struct ntb_edma_chans *edma)
> +{
> + struct ntb_edma_filter filter;
> + dma_cap_mask_t dma_mask;
> + unsigned int i;
> +
> + dma_cap_zero(dma_mask);
> + dma_cap_set(DMA_SLAVE, dma_mask);
> +
> + memset(edma, 0, sizeof(*edma));
> + edma->dev = dma_dev;
> +
> + filter.dma_dev = dma_dev;
> + filter.direction = BIT(DMA_DEV_TO_MEM);
> + for (i = 0; i < EDMA_WR_CH_NUM; i++) {
> + edma->wr_chan[i] = dma_request_channel(dma_mask,
> + ntb_edma_filter_fn,
> + &filter);
> + if (!edma->wr_chan[i])
> + break;
> + edma->num_wr_chan++;
> + }
> +
> + filter.direction = BIT(DMA_MEM_TO_DEV);
> + for (i = 0; i < EDMA_RD_CH_NUM; i++) {
> + edma->rd_chan[i] = dma_request_channel(dma_mask,
> + ntb_edma_filter_fn,
> + &filter);
> + if (!edma->rd_chan[i])
> + break;
> + edma->num_rd_chan++;
> + }
> +
> + edma->intr_chan = dma_request_channel(dma_mask, ntb_edma_filter_fn,
> + &filter);
> + if (!edma->intr_chan)
> + dev_warn(dma_dev,
> + "Remote eDMA notify channel could not be allocated\n");
> +
> + if (!edma->num_wr_chan || !edma->num_rd_chan) {
> + dev_warn(dma_dev, "Remote eDMA channels failed to initialize\n");
> + ntb_edma_teardown_chans(edma);
> + return -ENODEV;
> + }
> + return 0;
> +}
> +
> +struct dma_chan *ntb_edma_pick_chan(struct ntb_edma_chans *edma,
> + remote_edma_dir_t dir)
> +{
> + unsigned int n, cur, idx;
> + struct dma_chan **chans;
> + atomic_t *cur_chan;
> +
> + if (dir == REMOTE_EDMA_WRITE) {
> + n = edma->num_wr_chan;
> + chans = edma->wr_chan;
> + cur_chan = &edma->cur_wr_chan;
> + } else {
> + n = edma->num_rd_chan;
> + chans = edma->rd_chan;
> + cur_chan = &edma->cur_rd_chan;
> + }
> + if (WARN_ON_ONCE(!n))
> + return NULL;
> +
> + /* Simple round-robin */
> + cur = (unsigned int)atomic_inc_return(cur_chan) - 1;
> + idx = cur % n;
> + return chans[idx];
> +}
> +
> +int ntb_edma_notify_peer(struct ntb_edma_chans *edma, int qp_num)
> +{
> + struct dma_async_tx_descriptor *txd;
> + struct dma_slave_config cfg;
> + struct scatterlist sgl;
> + dma_cookie_t cookie;
> + struct device *dev;
> +
> + if (!edma || !edma->intr_chan)
> + return -ENXIO;
> +
> + if (qp_num < 0 || qp_num >= edma_ctx.notify_qp_max)
> + return -EINVAL;
> +
> + if (!edma_ctx.intr_rc_virt || !edma_ctx.intr_ep_phys)
> + return -EINVAL;
> +
> + dev = edma->dev;
> + if (!dev)
> + return -ENODEV;
> +
> + WRITE_ONCE(edma_ctx.intr_rc_virt->db[qp_num], 1);
> +
> + /* Ensure store is visible before kicking the DMA transfer */
> + wmb();
> +
> + sg_init_table(&sgl, 1);
> + sg_dma_address(&sgl) = edma_ctx.intr_rc_phys + qp_num * sizeof(u32);
> + sg_dma_len(&sgl) = sizeof(u32);
> +
> + memset(&cfg, 0, sizeof(cfg));
> + cfg.dst_addr = edma_ctx.intr_ep_phys + qp_num * sizeof(u32);
> + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> + cfg.direction = DMA_MEM_TO_DEV;
> +
> + if (dmaengine_slave_config(edma->intr_chan, &cfg))
> + return -EINVAL;
> +
> + txd = dmaengine_prep_slave_sg(edma->intr_chan, &sgl, 1,
> + DMA_MEM_TO_DEV,
> + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> + if (!txd)
> + return -ENOSPC;
> +
> + cookie = dmaengine_submit(txd);
> + if (dma_submit_error(cookie))
> + return -ENOSPC;
> +
> + dma_async_issue_pending(edma->intr_chan);
> + return 0;
> +}
> diff --git a/drivers/ntb/ntb_edma.h b/drivers/ntb/ntb_edma.h
> new file mode 100644
> index 000000000000..da0451827edb
> --- /dev/null
> +++ b/drivers/ntb/ntb_edma.h
> @@ -0,0 +1,128 @@
> +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
> +#ifndef _NTB_EDMA_H_
> +#define _NTB_EDMA_H_
> +
> +#include <linux/completion.h>
> +#include <linux/device.h>
> +#include <linux/interrupt.h>
> +
> +#define EDMA_REG_SIZE SZ_64K
> +#define DMA_LLP_MEM_SIZE SZ_4K
> +#define EDMA_WR_CH_NUM 4
> +#define EDMA_RD_CH_NUM 4
> +#define NTB_EDMA_MAX_CH 8
> +
> +#define NTB_EDMA_INFO_MAGIC 0x45444D41 /* "EDMA" */
> +#define NTB_EDMA_INFO_OFF EDMA_REG_SIZE
> +
> +#define NTB_EDMA_RING_ORDER 7
> +#define NTB_EDMA_RING_ENTRIES (1U << NTB_EDMA_RING_ORDER)
> +#define NTB_EDMA_RING_MASK (NTB_EDMA_RING_ENTRIES - 1)
> +
> +typedef void (*ntb_edma_interrupt_cb_t)(void *data, int qp_num);
> +
> +/*
> + * REMOTE_EDMA_EP:
> + * Endpoint owns the eDMA engine and pushes descriptors into a shared MW.
> + *
> + * REMOTE_EDMA_RC:
> + * Root Complex controls the endpoint eDMA through the shared MW and
> + * drives reads/writes on behalf of the host.
> + */
> +typedef enum {
> + REMOTE_EDMA_UNKNOWN,
> + REMOTE_EDMA_EP,
> + REMOTE_EDMA_RC,
> +} remote_edma_mode_t;
> +
> +typedef enum {
> + REMOTE_EDMA_WRITE,
> + REMOTE_EDMA_READ,
> +} remote_edma_dir_t;
> +
> +/*
> + * Layout of remote eDMA MW (EP local address space, RC sees via peer MW):
> + *
> + * 0 .. EDMA_REG_SIZE-1 : DesignWare eDMA registers
> + * EDMA_REG_SIZE .. +PAGE_SIZE : struct ntb_edma_info (EP writes, RC reads)
> + * +PAGE_SIZE .. : LL ring buffers (EP allocates phys addresses,
> + * RC configures via dw_edma)
> + *
> + * ntb_edma_setup_mws() on EP:
> + * - allocates ntb_edma_info and LLs in EP memory
> + * - programs inbound iATU so that RC peer MW[n] points at this block
> + *
> + * ntb_edma_setup_peer() on RC:
> + * - ioremaps peer MW[n]
> + * - reads ntb_edma_info
> + * - sets up dw_edma_chip ll_region_* from that info
> + */
> +struct ntb_edma_info {
> + u32 magic;
> + u16 wr_cnt;
> + u16 rd_cnt;
> + u64 regs_phys;
> + u32 ll_stride;
> + u32 rsvd;
> + u64 ll_wr_phys[NTB_EDMA_MAX_CH];
> + u64 ll_rd_phys[NTB_EDMA_MAX_CH];
> +
> + u64 intr_dar_base;
> +} __packed;
> +
> +struct ll_dma_addrs {
> + dma_addr_t wr[EDMA_WR_CH_NUM];
> + dma_addr_t rd[EDMA_RD_CH_NUM];
> +};
> +
> +struct ntb_edma_chans {
> + struct device *dev;
> +
> + struct dma_chan *wr_chan[EDMA_WR_CH_NUM];
> + struct dma_chan *rd_chan[EDMA_RD_CH_NUM];
> + struct dma_chan *intr_chan;
> +
> + unsigned int num_wr_chan;
> + unsigned int num_rd_chan;
> + atomic_t cur_wr_chan;
> + atomic_t cur_rd_chan;
> +};
> +
> +static __always_inline u32 ntb_edma_ring_idx(u32 v)
> +{
> + return v & NTB_EDMA_RING_MASK;
> +}
> +
> +static __always_inline u32 ntb_edma_ring_used_entry(u32 head, u32 tail)
> +{
> + if (head >= tail) {
> + WARN_ON_ONCE((head - tail) > (NTB_EDMA_RING_ENTRIES - 1));
> + return head - tail;
> + }
> +
> + WARN_ON_ONCE((U32_MAX - tail + head + 1) > (NTB_EDMA_RING_ENTRIES - 1));
> + return U32_MAX - tail + head + 1;
> +}
> +
> +static __always_inline u32 ntb_edma_ring_free_entry(u32 head, u32 tail)
> +{
> + return NTB_EDMA_RING_ENTRIES - ntb_edma_ring_used_entry(head, tail) - 1;
> +}
> +
> +static __always_inline bool ntb_edma_ring_full(u32 head, u32 tail)
> +{
> + return ntb_edma_ring_free_entry(head, tail) == 0;
> +}
> +
> +int ntb_edma_setup_isr(struct device *dev, struct device *epc_dev,
> + ntb_edma_interrupt_cb_t cb, void *data);
> +void ntb_edma_teardown_isr(struct device *dev);
> +int ntb_edma_setup_mws(struct ntb_dev *ndev);
> +int ntb_edma_setup_peer(struct ntb_dev *ndev);
> +int ntb_edma_setup_chans(struct device *dma_dev, struct ntb_edma_chans *edma);
> +struct dma_chan *ntb_edma_pick_chan(struct ntb_edma_chans *edma,
> + remote_edma_dir_t dir);
> +void ntb_edma_teardown_chans(struct ntb_edma_chans *edma);
> +int ntb_edma_notify_peer(struct ntb_edma_chans *edma, int qp_num);
> +
> +#endif
> diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport_core.c
> similarity index 65%
> rename from drivers/ntb/ntb_transport.c
> rename to drivers/ntb/ntb_transport_core.c
> index 907db6c93d4d..48d48921978d 100644
> --- a/drivers/ntb/ntb_transport.c
> +++ b/drivers/ntb/ntb_transport_core.c
> @@ -47,6 +47,9 @@
> * Contact Information:
> * Jon Mason <jon.mason@intel.com>
> */
> +#include <linux/atomic.h>
> +#include <linux/bug.h>
> +#include <linux/compiler.h>
> #include <linux/debugfs.h>
> #include <linux/delay.h>
> #include <linux/dmaengine.h>
> @@ -71,6 +74,8 @@
> #define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB"
> #define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2)
>
> +#define NTB_EDMA_MAX_POLL 32
> +
> MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
> MODULE_VERSION(NTB_TRANSPORT_VER);
> MODULE_LICENSE("Dual BSD/GPL");
> @@ -102,6 +107,13 @@ module_param(use_msi, bool, 0644);
> MODULE_PARM_DESC(use_msi, "Use MSI interrupts instead of doorbells");
> #endif
>
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> +#include "ntb_edma.h"
> +static bool use_remote_edma;
> +module_param(use_remote_edma, bool, 0644);
> +MODULE_PARM_DESC(use_remote_edma, "Use remote eDMA mode (when enabled, use_msi is ignored)");
> +#endif
> +
> static struct dentry *nt_debugfs_dir;
>
> /* Only two-ports NTB devices are supported */
> @@ -125,6 +137,14 @@ struct ntb_queue_entry {
> struct ntb_payload_header __iomem *tx_hdr;
> struct ntb_payload_header *rx_hdr;
> };
> +
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + dma_addr_t addr;
> +
> + /* Used by RC side only */
> + struct scatterlist sgl;
> + struct work_struct dma_work;
> +#endif
> };
>
> struct ntb_rx_info {
> @@ -202,6 +222,33 @@ struct ntb_transport_qp {
> int msi_irq;
> struct ntb_msi_desc msi_desc;
> struct ntb_msi_desc peer_msi_desc;
> +
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + /*
> + * For ensuring peer notification in non-atomic context.
> + * ntb_peer_db_set might sleep or schedule.
> + */
> + struct work_struct db_work;
> +
> + /*
> + * wr: remote eDMA write transfer (EP -> RC direction)
> + * rd: remote eDMA read transfer (RC -> EP direction)
> + */
> + u32 wr_cons;
> + u32 rd_cons;
> + u32 wr_prod;
> + u32 rd_prod;
> + u32 wr_issue;
> + u32 rd_issue;
> +
> + spinlock_t ep_tx_lock;
> + spinlock_t ep_rx_lock;
> + spinlock_t rc_lock;
> +
> + /* Completion work for read/write transfers. */
> + struct work_struct read_work;
> + struct work_struct write_work;
> +#endif
> };
>
> struct ntb_transport_mw {
> @@ -249,6 +296,13 @@ struct ntb_transport_ctx {
>
> /* Make sure workq of link event be executed serially */
> struct mutex link_event_lock;
> +
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + remote_edma_mode_t remote_edma_mode;
> + struct device *dma_dev;
> + struct workqueue_struct *wq;
> + struct ntb_edma_chans edma;
> +#endif
> };
>
> enum {
> @@ -262,6 +316,19 @@ struct ntb_payload_header {
> unsigned int flags;
> };
>
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> +static void ntb_transport_edma_uninit(struct ntb_transport_ctx *nt);
> +static int ntb_transport_edma_init(struct ntb_transport_ctx *nt,
> + unsigned int *mw_count);
> +static void ntb_transport_edma_init_queue(struct ntb_transport_ctx *nt,
> + unsigned int qp_num);
> +static void ntb_transport_edma_create_queue(struct ntb_transport_ctx *nt,
> + struct ntb_transport_qp *qp);
> +static int ntb_transport_edma_rc_init(struct ntb_transport_ctx *nt);
> +static int ntb_transport_edma_ep_init(struct ntb_transport_ctx *nt);
> +static void ntb_transport_edma_rc_dma_work(struct work_struct *work);
> +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> +
> /*
> * Return the device that should be used for DMA mapping.
> *
> @@ -298,7 +365,7 @@ enum {
> container_of((__drv), struct ntb_transport_client, driver)
>
> #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count)
> -#define NTB_QP_DEF_NUM_ENTRIES 100
> +#define NTB_QP_DEF_NUM_ENTRIES 128
> #define NTB_LINK_DOWN_TIMEOUT 10
>
> static void ntb_transport_rxc_db(unsigned long data);
> @@ -1015,6 +1082,10 @@ static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
> count = ntb_spad_count(nt->ndev);
> for (i = 0; i < count; i++)
> ntb_spad_write(nt->ndev, i, 0);
> +
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + ntb_edma_teardown_chans(&nt->edma);
> +#endif
> }
>
> static void ntb_transport_link_cleanup_work(struct work_struct *work)
> @@ -1051,6 +1122,14 @@ static void ntb_transport_link_work(struct work_struct *work)
>
> /* send the local info, in the opposite order of the way we read it */
>
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + rc = ntb_transport_edma_ep_init(nt);
> + if (rc) {
> + dev_err(&pdev->dev, "Failed to init EP: %d\n", rc);
> + return;
> + }
> +#endif
> +
> if (nt->use_msi) {
> rc = ntb_msi_setup_mws(ndev);
> if (rc) {
> @@ -1132,6 +1211,14 @@ static void ntb_transport_link_work(struct work_struct *work)
>
> nt->link_is_up = true;
>
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + rc = ntb_transport_edma_rc_init(nt);
> + if (rc) {
> + dev_err(&pdev->dev, "Failed to init RC: %d\n", rc);
> + goto out1;
> + }
> +#endif
> +
> for (i = 0; i < nt->qp_count; i++) {
> struct ntb_transport_qp *qp = &nt->qp_vec[i];
>
> @@ -1277,6 +1364,8 @@ static const struct ntb_transport_backend_ops default_backend_ops = {
> .debugfs_stats_show = ntb_transport_default_debugfs_stats_show,
> };
>
> +static const struct ntb_transport_backend_ops edma_backend_ops;
> +
> static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> {
> struct ntb_transport_ctx *nt;
> @@ -1311,7 +1400,23 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
>
> nt->ndev = ndev;
>
> - nt->backend_ops = default_backend_ops;
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + if (use_remote_edma) {
> + rc = ntb_transport_edma_init(nt, &mw_count);
> + if (rc) {
> + nt->mw_count = 0;
> + goto err;
> + }
> + nt->backend_ops = edma_backend_ops;
> +
> + /*
> + * On remote eDMA mode, we reserve a read channel for Host->EP
> + * interruption.
> + */
> + use_msi = false;
> + } else
> +#endif
> + nt->backend_ops = default_backend_ops;
>
> /*
> * If we are using MSI, and have at least one extra memory window,
> @@ -1402,6 +1507,10 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> rc = ntb_transport_init_queue(nt, i);
> if (rc)
> goto err2;
> +
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + ntb_transport_edma_init_queue(nt, i);
> +#endif
> }
>
> INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);
> @@ -1433,6 +1542,9 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> }
> kfree(nt->mw_vec);
> err:
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + ntb_transport_edma_uninit(nt);
> +#endif
> kfree(nt);
> return rc;
> }
> @@ -2055,11 +2167,16 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
>
> nt->qp_bitmap_free &= ~qp_bit;
>
> + qp->qp_bit = qp_bit;
> qp->cb_data = data;
> qp->rx_handler = handlers->rx_handler;
> qp->tx_handler = handlers->tx_handler;
> qp->event_handler = handlers->event_handler;
>
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + ntb_transport_edma_create_queue(nt, qp);
> +#endif
> +
> dma_cap_zero(dma_mask);
> dma_cap_set(DMA_MEMCPY, dma_mask);
>
> @@ -2105,6 +2222,9 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
> goto err1;
>
> entry->qp = qp;
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + INIT_WORK(&entry->dma_work, ntb_transport_edma_rc_dma_work);
> +#endif
> ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> &qp->rx_free_q);
> }
> @@ -2156,8 +2276,8 @@ EXPORT_SYMBOL_GPL(ntb_transport_create_queue);
> */
> void ntb_transport_free_queue(struct ntb_transport_qp *qp)
> {
> - struct pci_dev *pdev;
> struct ntb_queue_entry *entry;
> + struct pci_dev *pdev;
> u64 qp_bit;
>
> if (!qp)
> @@ -2208,6 +2328,10 @@ void ntb_transport_free_queue(struct ntb_transport_qp *qp)
> tasklet_kill(&qp->rxc_db_work);
>
> cancel_delayed_work_sync(&qp->link_work);
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + cancel_work_sync(&qp->read_work);
> + cancel_work_sync(&qp->write_work);
> +#endif
>
> qp->cb_data = NULL;
> qp->rx_handler = NULL;
> @@ -2346,6 +2470,1157 @@ int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
> }
> EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue);
>
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> +/*
> + * Remote eDMA mode implementation
> + */
> +struct ntb_edma_desc {
> + u32 len;
> + u32 flags;
> + u64 addr; /* DMA address */
> + u64 data;
> +};
> +
> +struct ntb_edma_ring {
> + struct ntb_edma_desc desc[NTB_EDMA_RING_ENTRIES];
> + u32 head;
> + u32 tail;
> +};
> +
> +#define NTB_EDMA_DESC_OFF(i) ((size_t)(i) * sizeof(struct ntb_edma_desc))
> +
> +#define __NTB_EDMA_CHECK_INDEX(_i) \
> +({ \
> + unsigned long __i = (unsigned long)(_i); \
> + WARN_ONCE(__i >= (unsigned long)NTB_EDMA_RING_ENTRIES, \
> + "ntb_edma: index i=%lu >= ring_entries=%lu\n", \
> + __i, (unsigned long)NTB_EDMA_RING_ENTRIES); \
> + __i; \
> +})
> +
> +#define NTB_EDMA_DESC_I(qp, i, n) \
> +({ \
> + typeof(qp) __qp = (qp); \
> + unsigned long __i = __NTB_EDMA_CHECK_INDEX(i); \
> + (struct ntb_edma_desc *) \
> + ((char *)(__qp)->rx_buff + \
> + (sizeof(struct ntb_edma_ring) * n) + \
> + NTB_EDMA_DESC_OFF(__i)); \
> +})
> +
> +#define NTB_EDMA_DESC_O(qp, i, n) \
> +({ \
> + typeof(qp) __qp = (qp); \
> + unsigned long __i = __NTB_EDMA_CHECK_INDEX(i); \
> + (struct ntb_edma_desc __iomem *) \
> + ((char __iomem *)(__qp)->tx_mw + \
> + (sizeof(struct ntb_edma_ring) * n) + \
> + NTB_EDMA_DESC_OFF(__i)); \
> +})
> +
> +#define NTB_EDMA_HEAD_I(qp, n) ((u32 *)((char *)qp->rx_buff + \
> + (sizeof(struct ntb_edma_ring) * n) + \
> + offsetof(struct ntb_edma_ring, head)))
> +#define NTB_EDMA_HEAD_O(qp, n) ((u32 *)((char __iomem *)qp->tx_mw + \
> + (sizeof(struct ntb_edma_ring) * n) + \
> + offsetof(struct ntb_edma_ring, head)))
> +#define NTB_EDMA_TAIL_I(qp, n) ((u32 *)((char *)qp->rx_buff + \
> + (sizeof(struct ntb_edma_ring) * n) + \
> + offsetof(struct ntb_edma_ring, tail)))
> +#define NTB_EDMA_TAIL_O(qp, n) ((u32 *)((char __iomem *)qp->tx_mw + \
> + (sizeof(struct ntb_edma_ring) * n) + \
> + offsetof(struct ntb_edma_ring, tail)))
> +
> +/*
> + * Macro naming rule:
> + * NTB_DESC_RD_EP_I (as an example)
> + * ^^ ^^ ^
> + * : : `-- I(n) or O(ut). In = Read, Out = Write.
> + * : `----- Who uses this macro.
> + * `-------- DESC / HEAD / TAIL
> + *
> + * Read transfers (RC->EP):
> + *
> + * EP view (outbound, written via NTB):
> + * - descs: NTB_DESC_RD_EP_O(qp, i) / NTB_DESC_RD_EP_I(qp, i)
> + * [ len ][ flags ][ addr ][ data ]
> + * [ len ][ flags ][ addr ][ data ]
> + * :
> + * [ len ][ flags ][ addr ][ data ]
> + * - head: NTB_HEAD_RD_EP_O(qp)
> + * - tail: NTB_TAIL_RD_EP_I(qp)
> + *
> + * RC view (inbound, local mapping):
> + * - descs: NTB_DESC_RD_RC_I(qp, i) / NTB_DESC_RD_RC_O(qp, i)
> + * [ len ][ flags ][ addr ][ data ]
> + * [ len ][ flags ][ addr ][ data ]
> + * :
> + * [ len ][ flags ][ addr ][ data ]
> + * - head: NTB_HEAD_RD_RC_I(qp)
> + * - tail: NTB_TAIL_RD_RC_O(qp)
> + *
> + * Write transfers (EP -> RC) are analogous but use
> + * NTB_DESC_WR_{EP_O,RC_I}(), NTB_HEAD_WR_{EP_O,RC_I}(),
> + * and NTB_TAIL_WR_{EP_I,RC_O}().
> + */
> +#define NTB_DESC_RD_EP_I(qp, i) NTB_EDMA_DESC_I(qp, i, 0)
> +#define NTB_DESC_RD_EP_O(qp, i) NTB_EDMA_DESC_O(qp, i, 0)
> +#define NTB_DESC_WR_EP_I(qp, i) NTB_EDMA_DESC_I(qp, i, 1)
> +#define NTB_DESC_WR_EP_O(qp, i) NTB_EDMA_DESC_O(qp, i, 1)
> +#define NTB_DESC_RD_RC_I(qp, i) NTB_EDMA_DESC_I(qp, i, 0)
> +#define NTB_DESC_RD_RC_O(qp, i) NTB_EDMA_DESC_O(qp, i, 0)
> +#define NTB_DESC_WR_RC_I(qp, i) NTB_EDMA_DESC_I(qp, i, 1)
> +#define NTB_DESC_WR_RC_O(qp, i) NTB_EDMA_DESC_O(qp, i, 1)
> +
> +#define NTB_HEAD_RD_EP_O(qp) NTB_EDMA_HEAD_O(qp, 0)
> +#define NTB_HEAD_WR_EP_O(qp) NTB_EDMA_HEAD_O(qp, 1)
> +#define NTB_HEAD_RD_RC_I(qp) NTB_EDMA_HEAD_I(qp, 0)
> +#define NTB_HEAD_WR_RC_I(qp) NTB_EDMA_HEAD_I(qp, 1)
> +
> +#define NTB_TAIL_RD_EP_I(qp) NTB_EDMA_TAIL_I(qp, 0)
> +#define NTB_TAIL_WR_EP_I(qp) NTB_EDMA_TAIL_I(qp, 1)
> +#define NTB_TAIL_RD_RC_O(qp) NTB_EDMA_TAIL_O(qp, 0)
> +#define NTB_TAIL_WR_RC_O(qp) NTB_EDMA_TAIL_O(qp, 1)
> +
> +static inline bool ntb_qp_edma_is_rc(struct ntb_transport_qp *qp)
> +{
> + return qp->transport->remote_edma_mode == REMOTE_EDMA_RC;
> +}
> +
> +static inline bool ntb_qp_edma_is_ep(struct ntb_transport_qp *qp)
> +{
> + return qp->transport->remote_edma_mode == REMOTE_EDMA_EP;
> +}
> +
> +static inline bool ntb_qp_edma_enabled(struct ntb_transport_qp *qp)
> +{
> + return ntb_qp_edma_is_rc(qp) || ntb_qp_edma_is_ep(qp);
> +}
> +
> +static unsigned int ntb_transport_edma_tx_free_entry(struct ntb_transport_qp *qp)
> +{
> + unsigned int head, tail;
> +
> + if (ntb_qp_edma_is_ep(qp)) {
> + scoped_guard(spinlock_irqsave, &qp->ep_tx_lock) {
> + /* In this scope, only 'head' might proceed */
> + tail = READ_ONCE(qp->wr_cons);
> + head = READ_ONCE(qp->wr_prod);
> + }
> + return ntb_edma_ring_free_entry(head, tail);
> + }
> +
> + scoped_guard(spinlock_irqsave, &qp->rc_lock) {
> + /* In this scope, only 'head' might proceed */
> + tail = READ_ONCE(qp->rd_issue);
> + head = READ_ONCE(*NTB_HEAD_RD_RC_I(qp));
> + }
> + /*
> + * On RC side, 'used' amount indicates how much EP side
> + * has refilled, which are available for us to use for TX.
> + */
> + return ntb_edma_ring_used_entry(head, tail);
> +}
> +
> +static void ntb_transport_edma_debugfs_stats_show(struct seq_file *s,
> + struct ntb_transport_qp *qp)
> +{
> + seq_printf(s, "rx_bytes - \t%llu\n", qp->rx_bytes);
> + seq_printf(s, "rx_pkts - \t%llu\n", qp->rx_pkts);
> + seq_printf(s, "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
> + seq_printf(s, "rx_buff - \t0x%p\n", qp->rx_buff);
> + seq_printf(s, "rx_max_entry - \t%u\n", qp->rx_max_entry);
> + seq_printf(s, "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry);
> +
> + seq_printf(s, "tx_bytes - \t%llu\n", qp->tx_bytes);
> + seq_printf(s, "tx_pkts - \t%llu\n", qp->tx_pkts);
> + seq_printf(s, "tx_ring_full - \t%llu\n", qp->tx_ring_full);
> + seq_printf(s, "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
> + seq_printf(s, "tx_mw - \t0x%p\n", qp->tx_mw);
> + seq_printf(s, "tx_max_entry - \t%u\n", qp->tx_max_entry);
> + seq_printf(s, "free tx - \t%u\n", ntb_transport_tx_free_entry(qp));
> + seq_putc(s, '\n');
> +
> + seq_puts(s, "Using Remote eDMA - Yes\n");
> + seq_printf(s, "QP Link - \t%s\n", qp->link_is_up ? "Up" : "Down");
> +}
> +
> +static void ntb_transport_edma_uninit(struct ntb_transport_ctx *nt)
> +{
> + struct ntb_dev *ndev = nt->ndev;
> +
> + if (nt->remote_edma_mode == REMOTE_EDMA_EP && ndev && ndev->pdev)
> + ntb_edma_teardown_isr(&ndev->pdev->dev);
> +
/pr> + if (nt->wq)
> + destroy_workqueue(nt->wq);
> + nt->wq = NULL;
> +}
> +
> +static int ntb_transport_edma_init(struct ntb_transport_ctx *nt,
> + unsigned int *mw_count)
> +{
> + struct ntb_dev *ndev = nt->ndev;
> +
> + /*
> + * We need at least one MW for the transport plus one MW reserved
> + * for the remote eDMA window (see ntb_edma_setup_mws/peer).
> + */
> + if (*mw_count <= 1) {
> + dev_err(&ndev->dev,
> + "remote eDMA requires at least two MWS (have %u)\n",
> + *mw_count);
> + return -ENODEV;
> + }
> +
> + nt->wq = alloc_workqueue("ntb-edma-wq", WQ_UNBOUND | WQ_SYSFS, 0);
> + if (!nt->wq) {
> + ntb_transport_edma_uninit(nt);
> + return -ENOMEM;
> + }
> +
> + /* Reserve the last peer MW exclusively for the eDMA window. */
> + *mw_count -= 1;
> +
> + return 0;
> +}
> +
> +static void ntb_transport_edma_db_work(struct work_struct *work)
> +{
> + struct ntb_transport_qp *qp =
> + container_of(work, struct ntb_transport_qp, db_work);
> +
> + ntb_peer_db_set(qp->ndev, qp->qp_bit);
> +}
> +
> +static void ntb_transport_edma_notify_peer(struct ntb_transport_qp *qp)
> +{
> + if (ntb_qp_edma_is_rc(qp))
> + if (!ntb_edma_notify_peer(&qp->transport->edma, qp->qp_num))
> + return;
> +
> + /*
> + * Called from contexts that may be atomic. Since ntb_peer_db_set()
> + * may sleep, delegate the actual doorbell write to a workqueue.
> + */
> + queue_work(system_highpri_wq, &qp->db_work);
> +}
> +
> +static void ntb_transport_edma_isr(void *data, int qp_num)
> +{
> + struct ntb_transport_ctx *nt = data;
> + struct ntb_transport_qp *qp;
> +
> + if (qp_num < 0 || qp_num >= nt->qp_count)
> + return;
> +
> + qp = &nt->qp_vec[qp_num];
> + if (WARN_ON(!qp))
> + return;
> +
> + queue_work(nt->wq, &qp->read_work);
> + queue_work(nt->wq, &qp->write_work);
> +}
> +
> +static int ntb_transport_edma_rc_init(struct ntb_transport_ctx *nt)
> +{
> + struct ntb_dev *ndev = nt->ndev;
> + struct pci_dev *pdev = ndev->pdev;
> + int rc;
> +
> + if (!use_remote_edma || nt->remote_edma_mode != REMOTE_EDMA_UNKNOWN)
> + return 0;
> +
> + rc = ntb_edma_setup_peer(ndev);
> + if (rc) {
> + dev_err(&pdev->dev, "Failed to enable remote eDMA: %d\n", rc);
> + return rc;
> + }
> +
> + rc = ntb_edma_setup_chans(get_dma_dev(ndev), &nt->edma);
> + if (rc) {
> + dev_err(&pdev->dev, "Failed to setup eDMA channels: %d\n", rc);
> + return rc;
> + }
> +
> + nt->remote_edma_mode = REMOTE_EDMA_RC;
> + return 0;
> +}
> +
> +static int ntb_transport_edma_ep_init(struct ntb_transport_ctx *nt)
> +{
> + struct ntb_dev *ndev = nt->ndev;
> + struct pci_dev *pdev = ndev->pdev;
> + struct pci_epc *epc;
> + int rc;
> +
> + if (!use_remote_edma || nt->remote_edma_mode == REMOTE_EDMA_EP)
> + return 0;
> +
> + /* Only EP side can return pci_epc */
> + epc = ntb_get_pci_epc(ndev);
> + if (!epc)
> + return 0;
> +
> + rc = ntb_edma_setup_mws(ndev);
> + if (rc) {
> + dev_err(&pdev->dev,
> + "Failed to set up memory window for eDMA: %d\n", rc);
> + return rc;
> + }
> +
> + rc = ntb_edma_setup_isr(&pdev->dev, &epc->dev, ntb_transport_edma_isr, nt);
> + if (rc) {
> + dev_err(&pdev->dev, "Failed to setup eDMA ISR (%d)\n", rc);
> + return rc;
> + }
> +
> + nt->remote_edma_mode = REMOTE_EDMA_EP;
> + return 0;
> +}
> +
> +static int ntb_transport_edma_setup_qp_mw(struct ntb_transport_ctx *nt,
> + unsigned int qp_num)
> +{
> + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
> + struct ntb_dev *ndev = nt->ndev;
> + struct ntb_queue_entry *entry;
> + struct ntb_transport_mw *mw;
> + unsigned int mw_num, mw_count, qp_count;
> + unsigned int qp_offset, rx_info_offset;
> + unsigned int mw_size, mw_size_per_qp;
> + unsigned int num_qps_mw;
> + size_t edma_total;
> + unsigned int i;
> + int node;
> +
> + mw_count = nt->mw_count;
> + qp_count = nt->qp_count;
> +
> + mw_num = QP_TO_MW(nt, qp_num);
> + mw = &nt->mw_vec[mw_num];
> +
> + if (!mw->virt_addr)
> + return -ENOMEM;
> +
> + if (mw_num < qp_count % mw_count)
> + num_qps_mw = qp_count / mw_count + 1;
> + else
> + num_qps_mw = qp_count / mw_count;
> +
> + mw_size = min(nt->mw_vec[mw_num].phys_size, mw->xlat_size);
> + if (max_mw_size && mw_size > max_mw_size)
> + mw_size = max_mw_size;
> +
> + mw_size_per_qp = round_down((unsigned int)mw_size / num_qps_mw, SZ_64);
> + qp_offset = mw_size_per_qp * (qp_num / mw_count);
> + rx_info_offset = mw_size_per_qp - sizeof(struct ntb_rx_info);
> +
> + qp->tx_mw_size = mw_size_per_qp;
> + qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;
> + if (!qp->tx_mw)
> + return -EINVAL;
> + qp->tx_mw_phys = nt->mw_vec[mw_num].phys_addr + qp_offset;
> + if (!qp->tx_mw_phys)
> + return -EINVAL;
> + qp->rx_info = qp->tx_mw + rx_info_offset;
> + qp->rx_buff = mw->virt_addr + qp_offset;
> + qp->remote_rx_info = qp->rx_buff + rx_info_offset;
> +
> + /* Due to housekeeping, there must be at least 2 buffs */
> + qp->tx_max_frame = min(transport_mtu, mw_size_per_qp / 2);
> + qp->rx_max_frame = min(transport_mtu, mw_size_per_qp / 2);
> +
> + /* In eDMA mode, decouple from MW sizing and force ring-sized entries */
> + edma_total = 2 * sizeof(struct ntb_edma_ring);
> + if (rx_info_offset < edma_total) {
> + dev_err(&ndev->dev, "Ring space requires %luB (>=%uB)\n",
> + edma_total, rx_info_offset);
> + return -EINVAL;
> + }
> + qp->tx_max_entry = NTB_EDMA_RING_ENTRIES;
> + qp->rx_max_entry = NTB_EDMA_RING_ENTRIES;
> +
> + /*
> + * Checking to see if we have more entries than the default.
> + * We should add additional entries if that is the case so we
> + * can be in sync with the transport frames.
> + */
> + node = dev_to_node(&ndev->dev);
> + for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) {
> + entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
> + if (!entry)
> + return -ENOMEM;
> +
> + entry->qp = qp;
> + INIT_WORK(&entry->dma_work, ntb_transport_edma_rc_dma_work);
> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> + &qp->rx_free_q);
> + qp->rx_alloc_entry++;
> + }
> +
> + memset(qp->rx_buff, 0, edma_total);
> +
> + qp->rx_pkts = 0;
> + qp->tx_pkts = 0;
> +
> + return 0;
> +}
> +
> +static int ntb_transport_edma_ep_read_complete(struct ntb_transport_qp *qp)
> +{
> + struct device *dma_dev = get_dma_dev(qp->ndev);
> + struct ntb_queue_entry *entry;
> + struct ntb_edma_desc *in;
> + unsigned int len;
> + u32 idx;
> +
> + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_TAIL_RD_EP_I(qp)),
> + qp->rd_cons) == 0)
> + return 0;
> +
> + idx = ntb_edma_ring_idx(qp->rd_cons);
> + in = NTB_DESC_RD_EP_I(qp, idx);
> + if (!(in->flags & DESC_DONE_FLAG))
> + return 0;
> +
> + in->flags = 0;
> + len = in->len; /* might be smaller than entry->len */
> +
> + entry = (struct ntb_queue_entry *)(in->data);
> + if (WARN_ON(!entry))
> + return 0;
> +
> + if (in->flags & LINK_DOWN_FLAG) {
> + ntb_qp_link_down(qp);
> + qp->rd_cons++;
> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> + return 1;
> + }
> +
> + dma_unmap_single(dma_dev, entry->addr, entry->len, DMA_FROM_DEVICE);
> +
> + qp->rx_bytes += len;
> + qp->rx_pkts++;
> + qp->rd_cons++;
> +
> + if (qp->rx_handler && qp->client_ready)
> + qp->rx_handler(qp, qp->cb_data, entry->cb_data, len);
> +
> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> + return 1;
> +}
> +
> +static int ntb_transport_edma_ep_write_complete(struct ntb_transport_qp *qp)
> +{
> + struct ntb_queue_entry *entry;
> + struct ntb_edma_desc *in;
> + u32 idx;
> +
> + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_TAIL_WR_EP_I(qp)),
> + qp->wr_cons) == 0)
> + return 0;
> +
> + idx = ntb_edma_ring_idx(qp->wr_cons);
> + in = NTB_DESC_WR_EP_I(qp, idx);
> +
> + entry = (struct ntb_queue_entry *)(in->data);
> + if (WARN_ON(!entry))
> + return 0;
> +
> + qp->wr_cons++;
> +
> + if (qp->tx_handler)
> + qp->tx_handler(qp, qp->cb_data, entry->cb_data, entry->len);
> +
> + ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q);
> + return 1;
> +}
> +
> +static void ntb_transport_edma_ep_read_work(struct work_struct *work)
> +{
> + struct ntb_transport_qp *qp = container_of(
> + work, struct ntb_transport_qp, read_work);
> + unsigned int i;
> +
> + for (i = 0; i < NTB_EDMA_MAX_POLL; i++) {
> + if (!ntb_transport_edma_ep_read_complete(qp))
> + break;
> + }
> +
> + if (ntb_transport_edma_ep_read_complete(qp))
> + queue_work(qp->transport->wq, &qp->read_work);
> +}
> +
> +static void ntb_transport_edma_ep_write_work(struct work_struct *work)
> +{
> + struct ntb_transport_qp *qp = container_of(
> + work, struct ntb_transport_qp, write_work);
> + unsigned int i;
> +
> + for (i = 0; i < NTB_EDMA_MAX_POLL; i++) {
> + if (!ntb_transport_edma_ep_write_complete(qp))
> + break;
> + }
> +
> + if (ntb_transport_edma_ep_write_complete(qp))
> + queue_work(qp->transport->wq, &qp->write_work);
> +}
> +
> +static void ntb_transport_edma_rc_write_complete_work(struct work_struct *work)
> +{
> + struct ntb_transport_qp *qp = container_of(
> + work, struct ntb_transport_qp, write_work);
> + struct ntb_queue_entry *entry;
> + struct ntb_edma_desc *in;
> + unsigned int len;
> + void *cb_data;
> + u32 idx;
> +
> + while (ntb_edma_ring_used_entry(READ_ONCE(qp->wr_issue),
> + qp->wr_cons) != 0) {
> + /* Paired with smp_wmb() in ntb_transport_edma_rc_poll() */
> + smp_rmb();
> +
> + idx = ntb_edma_ring_idx(qp->wr_cons);
> + in = NTB_DESC_WR_RC_I(qp, idx);
> + entry = (struct ntb_queue_entry *)READ_ONCE(in->data);
> + if (!entry || !(entry->flags & DESC_DONE_FLAG))
> + break;
> +
> + in->data = 0;
> +
> + cb_data = entry->cb_data;
> + len = entry->len;
> +
> + iowrite32(++qp->wr_cons, NTB_TAIL_WR_RC_O(qp));
> +
> + if (unlikely(entry->flags & LINK_DOWN_FLAG)) {
> + ntb_qp_link_down(qp);
> + continue;
> + }
> +
> + ntb_transport_edma_notify_peer(qp);
> +
> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> +
> + if (qp->rx_handler && qp->client_ready)
> + qp->rx_handler(qp, qp->cb_data, cb_data, len);
> +
> + /* stat updates */
> + qp->rx_bytes += len;
> + qp->rx_pkts++;
> + }
> +}
> +
> +static void ntb_transport_edma_rc_write_cb(void *data,
> + const struct dmaengine_result *res)
> +{
> + struct ntb_queue_entry *entry = data;
> + struct ntb_transport_qp *qp = entry->qp;
> + struct ntb_transport_ctx *nt = qp->transport;
> + enum dmaengine_tx_result dma_err = res->result;
> + struct device *dma_dev = get_dma_dev(qp->ndev);
> +
> + switch (dma_err) {
> + case DMA_TRANS_READ_FAILED:
> + case DMA_TRANS_WRITE_FAILED:
> + case DMA_TRANS_ABORTED:
> + entry->errors++;
> + entry->len = -EIO;
> + break;
> + case DMA_TRANS_NOERROR:
> + default:
> + break;
> + }
> + dma_unmap_sg(dma_dev, &entry->sgl, 1, DMA_FROM_DEVICE);
> + sg_dma_address(&entry->sgl) = 0;
> +
> + entry->flags |= DESC_DONE_FLAG;
> +
> + queue_work(nt->wq, &qp->write_work);
> +}
> +
> +static void ntb_transport_edma_rc_read_complete_work(struct work_struct *work)
> +{
> + struct ntb_transport_qp *qp = container_of(
> + work, struct ntb_transport_qp, read_work);
> + struct ntb_edma_desc *in, __iomem *out;
> + struct ntb_queue_entry *entry;
> + unsigned int len;
> + void *cb_data;
> + u32 idx;
> +
> + while (ntb_edma_ring_used_entry(READ_ONCE(qp->rd_issue),
> + qp->rd_cons) != 0) {
> + /* Paired with smp_wmb() in ntb_transport_edma_rc_tx_enqueue() */
> + smp_rmb();
> +
> + idx = ntb_edma_ring_idx(qp->rd_cons);
> + in = NTB_DESC_RD_RC_I(qp, idx);
> + entry = (struct ntb_queue_entry *)in->data;
> + if (!entry || !(entry->flags & DESC_DONE_FLAG))
> + break;
> +
> + in->data = 0;
> +
> + cb_data = entry->cb_data;
> + len = entry->len;
> +
> + out = NTB_DESC_RD_RC_O(qp, idx);
> +
> + WRITE_ONCE(qp->rd_cons, qp->rd_cons + 1);
> +
> + /*
> + * No need to add barrier in-between to enforce ordering here.
> + * The other side proceeds only after both flags and tail are
> + * updated.
> + */
> + iowrite32(entry->flags, &out->flags);
> + iowrite32(qp->rd_cons, NTB_TAIL_RD_RC_O(qp));
> +
> + ntb_transport_edma_notify_peer(qp);
> +
> + ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
> + &qp->tx_free_q);
> +
> + if (qp->tx_handler)
> + qp->tx_handler(qp, qp->cb_data, cb_data, len);
> +
> + /* stat updates */
> + qp->tx_bytes += len;
> + qp->tx_pkts++;
> + }
> +}
> +
> +static void ntb_transport_edma_rc_read_cb(void *data,
> + const struct dmaengine_result *res)
> +{
> + struct ntb_queue_entry *entry = data;
> + struct ntb_transport_qp *qp = entry->qp;
> + struct ntb_transport_ctx *nt = qp->transport;
> + struct device *dma_dev = get_dma_dev(qp->ndev);
> + enum dmaengine_tx_result dma_err = res->result;
> +
> + switch (dma_err) {
> + case DMA_TRANS_READ_FAILED:
> + case DMA_TRANS_WRITE_FAILED:
> + case DMA_TRANS_ABORTED:
> + entry->errors++;
> + entry->len = -EIO;
> + break;
> + case DMA_TRANS_NOERROR:
> + default:
> + break;
> + }
> + dma_unmap_sg(dma_dev, &entry->sgl, 1, DMA_TO_DEVICE);
> + sg_dma_address(&entry->sgl) = 0;
> +
> + entry->flags |= DESC_DONE_FLAG;
> +
> + queue_work(nt->wq, &qp->read_work);
> +}
> +
> +static int ntb_transport_edma_rc_write_start(struct device *d,
> + struct dma_chan *chan, size_t len,
> + dma_addr_t ep_src, void *rc_dst,
> + struct ntb_queue_entry *entry)
> +{
> + struct scatterlist *sgl = &entry->sgl;
> + struct dma_async_tx_descriptor *txd;
> + struct dma_slave_config cfg;
> + dma_cookie_t cookie;
> + int nents, rc;
> +
> + if (!d)
> + return -ENODEV;
> +
> + if (!chan)
> + return -ENXIO;
> +
> + if (WARN_ON(!ep_src || !rc_dst))
> + return -EINVAL;
> +
> + if (WARN_ON(sg_dma_address(sgl)))
> + return -EINVAL;
> +
> + sg_init_one(sgl, rc_dst, len);
> + nents = dma_map_sg(d, sgl, 1, DMA_FROM_DEVICE);
> + if (nents <= 0)
> + return -EIO;
> +
> + memset(&cfg, 0, sizeof(cfg));
> + cfg.src_addr = ep_src;
> + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> + cfg.direction = DMA_DEV_TO_MEM;
> + rc = dmaengine_slave_config(chan, &cfg);
> + if (rc)
> + goto out_unmap;
> +
> + txd = dmaengine_prep_slave_sg(chan, sgl, 1, DMA_DEV_TO_MEM,
> + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> + if (!txd) {
> + rc = -EIO;
> + goto out_unmap;
> + }
> +
> + txd->callback_result = ntb_transport_edma_rc_write_cb;
> + txd->callback_param = entry;
> +
> + cookie = dmaengine_submit(txd);
> + if (dma_submit_error(cookie)) {
> + rc = -EIO;
> + goto out_unmap;
> + }
> + dma_async_issue_pending(chan);
> + return 0;
> +out_unmap:
> + dma_unmap_sg(d, sgl, 1, DMA_FROM_DEVICE);
> + return rc;
> +}
> +
> +static int ntb_transport_edma_rc_read_start(struct device *d,
> + struct dma_chan *chan, size_t len,
> + void *rc_src, dma_addr_t ep_dst,
> + struct ntb_queue_entry *entry)
> +{
> + struct scatterlist *sgl = &entry->sgl;
> + struct dma_async_tx_descriptor *txd;
> + struct dma_slave_config cfg;
> + dma_cookie_t cookie;
> + int nents, rc;
> +
> + if (!d)
> + return -ENODEV;
> +
> + if (!chan)
> + return -ENXIO;
> +
> + if (WARN_ON(!rc_src || !ep_dst))
> + return -EINVAL;
> +
> + if (WARN_ON(sg_dma_address(sgl)))
> + return -EINVAL;
> +
> + sg_init_one(sgl, rc_src, len);
> + nents = dma_map_sg(d, sgl, 1, DMA_TO_DEVICE);
> + if (nents <= 0)
> + return -EIO;
> +
> + memset(&cfg, 0, sizeof(cfg));
> + cfg.dst_addr = ep_dst;
> + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> + cfg.direction = DMA_MEM_TO_DEV;
> + rc = dmaengine_slave_config(chan, &cfg);
> + if (rc)
> + goto out_unmap;
> +
> + txd = dmaengine_prep_slave_sg(chan, sgl, 1, DMA_MEM_TO_DEV,
> + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> + if (!txd) {
> + rc = -EIO;
> + goto out_unmap;
> + }
> +
> + txd->callback_result = ntb_transport_edma_rc_read_cb;
> + txd->callback_param = entry;
> +
> + cookie = dmaengine_submit(txd);
> + if (dma_submit_error(cookie)) {
> + rc = -EIO;
> + goto out_unmap;
> + }
> + dma_async_issue_pending(chan);
> + return 0;
> +out_unmap:
> + dma_unmap_sg(d, sgl, 1, DMA_TO_DEVICE);
> + return rc;
> +}
> +
> +static void ntb_transport_edma_rc_dma_work(struct work_struct *work)
> +{
> + struct ntb_queue_entry *entry = container_of(
> + work, struct ntb_queue_entry, dma_work);
> + struct ntb_transport_qp *qp = entry->qp;
> + struct ntb_transport_ctx *nt = qp->transport;
> + struct device *dma_dev = get_dma_dev(qp->ndev);
> + struct dma_chan *chan;
> + int rc;
> +
> + chan = ntb_edma_pick_chan(&nt->edma, REMOTE_EDMA_WRITE);
> + rc = ntb_transport_edma_rc_write_start(dma_dev, chan, entry->len,
> + entry->addr, entry->buf, entry);
> + if (rc) {
> + entry->errors++;
> + entry->len = -EIO;
> + entry->flags |= DESC_DONE_FLAG;
> + queue_work(nt->wq, &qp->write_work);
> + return;
> + }
> +}
> +
> +static void ntb_transport_edma_rc_poll(struct ntb_transport_qp *qp)
> +{
> + struct ntb_transport_ctx *nt = qp->transport;
> + unsigned int budget = NTB_EDMA_MAX_POLL;
> + struct ntb_queue_entry *entry;
> + struct ntb_edma_desc *in;
> + dma_addr_t ep_src;
> + u32 len, idx;
> +
> + while (budget--) {
> + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_HEAD_WR_RC_I(qp)),
> + qp->wr_issue) == 0)
> + break;
> +
> + idx = ntb_edma_ring_idx(qp->wr_issue);
> + in = NTB_DESC_WR_RC_I(qp, idx);
> +
> + len = READ_ONCE(in->len);
> + ep_src = (dma_addr_t)READ_ONCE(in->addr);
> +
> + /* Prepare 'entry' for write completion */
> + entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q);
> + if (!entry) {
> + qp->rx_err_no_buf++;
> + break;
> + }
> + if (WARN_ON(entry->flags & DESC_DONE_FLAG))
> + entry->flags &= ~DESC_DONE_FLAG;
> + entry->len = len; /* NB. entry->len can be <=0 */
> + entry->addr = ep_src;
> +
> + /*
> + * ntb_transport_edma_rc_write_complete_work() checks entry->flags
> + * so it needs to be set before wr_issue++.
> + */
> + in->data = (uintptr_t)entry;
> +
> + /* Ensure in->data visible before wr_issue++ */
> + smp_wmb();
> +
> + WRITE_ONCE(qp->wr_issue, qp->wr_issue + 1);
> +
> + if (!len) {
> + entry->flags |= DESC_DONE_FLAG;
> + queue_work(nt->wq, &qp->write_work);
> + continue;
> + }
> +
> + if (in->flags & LINK_DOWN_FLAG) {
> + dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n");
> + entry->flags |= DESC_DONE_FLAG | LINK_DOWN_FLAG;
> + queue_work(nt->wq, &qp->write_work);
> + continue;
> + }
> +
> + queue_work(nt->wq, &entry->dma_work);
> + }
> +
> + if (!budget)
> + tasklet_schedule(&qp->rxc_db_work);
> +}
> +
> +static int ntb_transport_edma_rc_tx_enqueue(struct ntb_transport_qp *qp,
> + struct ntb_queue_entry *entry)
> +{
> + struct device *dma_dev = get_dma_dev(qp->ndev);
> + struct ntb_transport_ctx *nt = qp->transport;
> + struct ntb_edma_desc *in, __iomem *out;
> + unsigned int len = entry->len;
> + struct dma_chan *chan;
> + u32 issue, idx, head;
> + dma_addr_t ep_dst;
> + int rc;
> +
> + WARN_ON_ONCE(entry->flags & DESC_DONE_FLAG);
> +
> + scoped_guard(spinlock_irqsave, &qp->rc_lock) {
> + head = READ_ONCE(*NTB_HEAD_RD_RC_I(qp));
> + issue = qp->rd_issue;
> + if (ntb_edma_ring_used_entry(head, issue) == 0) {
> + qp->tx_ring_full++;
> + return -ENOSPC;
> + }
> +
> + /*
> + * ntb_transport_edma_rc_read_complete_work() checks entry->flags
> + * so it needs to be set before rd_issue++.
> + */
> + idx = ntb_edma_ring_idx(issue);
> + in = NTB_DESC_RD_RC_I(qp, idx);
> + in->data = (uintptr_t)entry;
> +
> + /* Make in->data visible before rd_issue++ */
> + smp_wmb();
> +
> + WRITE_ONCE(qp->rd_issue, qp->rd_issue + 1);
> + }
> +
> + /* Publish the final transfer length to the EP side */
> + out = NTB_DESC_RD_RC_O(qp, idx);
> + iowrite32(len, &out->len);
> + ioread32(&out->len);
> +
> + if (unlikely(!len)) {
> + entry->flags |= DESC_DONE_FLAG;
> + queue_work(nt->wq, &qp->read_work);
> + return 0;
> + }
> +
> + /* Paired with dma_wmb() in ntb_transport_edma_ep_rx_enqueue() */
> + dma_rmb();
> +
> + /* kick remote eDMA read transfer */
> + ep_dst = (dma_addr_t)in->addr;
> + chan = ntb_edma_pick_chan(&nt->edma, REMOTE_EDMA_READ);
> + rc = ntb_transport_edma_rc_read_start(dma_dev, chan, len,
> + entry->buf, ep_dst, entry);
> + if (rc) {
> + entry->errors++;
> + entry->len = -EIO;
> + entry->flags |= DESC_DONE_FLAG;
> + queue_work(nt->wq, &qp->read_work);
> + }
> + return 0;
> +}
> +
> +static int ntb_transport_edma_ep_tx_enqueue(struct ntb_transport_qp *qp,
> + struct ntb_queue_entry *entry)
> +{
> + struct device *dma_dev = get_dma_dev(qp->ndev);
> + struct ntb_edma_desc *in, __iomem *out;
> + unsigned int len = entry->len;
> + dma_addr_t ep_src = 0;
> + u32 idx;
> + int rc;
> +
> + if (likely(len)) {
> + ep_src = dma_map_single(dma_dev, entry->buf, len,
> + DMA_TO_DEVICE);
> + rc = dma_mapping_error(dma_dev, ep_src);
> + if (rc)
> + return rc;
> + }
> +
> + scoped_guard(spinlock_irqsave, &qp->ep_tx_lock) {
> + if (ntb_edma_ring_full(qp->wr_prod, qp->wr_cons)) {
> + rc = -ENOSPC;
> + qp->tx_ring_full++;
> + goto out_unmap;
> + }
> +
> + idx = ntb_edma_ring_idx(qp->wr_prod);
> + in = NTB_DESC_WR_EP_I(qp, idx);
> + out = NTB_DESC_WR_EP_O(qp, idx);
> +
> + WARN_ON(in->flags & DESC_DONE_FLAG);
> + WARN_ON(entry->flags & DESC_DONE_FLAG);
> + in->flags = 0;
> + in->data = (uintptr_t)entry;
> + entry->addr = ep_src;
> +
> + iowrite32(len, &out->len);
> + iowrite32(entry->flags, &out->flags);
> + iowrite64(ep_src, &out->addr);
> + WRITE_ONCE(qp->wr_prod, qp->wr_prod + 1);
> +
> + dma_wmb();
> + iowrite32(qp->wr_prod, NTB_HEAD_WR_EP_O(qp));
> +
> + qp->tx_bytes += len;
> + qp->tx_pkts++;
> + }
> +
> + ntb_transport_edma_notify_peer(qp);
> +
> + return 0;
> +out_unmap:
> + if (likely(len))
> + dma_unmap_single(dma_dev, ep_src, len, DMA_TO_DEVICE);
> + return rc;
> +}
> +
> +static int ntb_transport_edma_tx_enqueue(struct ntb_transport_qp *qp,
> + struct ntb_queue_entry *entry,
> + void *cb, void *data, unsigned int len,
> + unsigned int flags)
> +{
> + struct device *dma_dev;
> +
> + if (entry->addr) {
> + /* Deferred unmap */
> + dma_dev = get_dma_dev(qp->ndev);
> + dma_unmap_single(dma_dev, entry->addr, entry->len, DMA_TO_DEVICE);
> + }
> +
> + entry->cb_data = cb;
> + entry->buf = data;
> + entry->len = len;
> + entry->flags = flags;
> + entry->errors = 0;
> + entry->addr = 0;
> +
> + WARN_ON_ONCE(!ntb_qp_edma_enabled(qp));
> +
> + if (ntb_qp_edma_is_ep(qp))
> + return ntb_transport_edma_ep_tx_enqueue(qp, entry);
> + else
> + return ntb_transport_edma_rc_tx_enqueue(qp, entry);
> +}
> +
> +static int ntb_transport_edma_ep_rx_enqueue(struct ntb_transport_qp *qp,
> + struct ntb_queue_entry *entry)
> +{
> + struct device *dma_dev = get_dma_dev(qp->ndev);
> + struct ntb_edma_desc *in, __iomem *out;
> + unsigned int len = entry->len;
> + void *data = entry->buf;
> + dma_addr_t ep_dst;
> + u32 idx;
> + int rc;
> +
> + ep_dst = dma_map_single(dma_dev, data, len, DMA_FROM_DEVICE);
> + rc = dma_mapping_error(dma_dev, ep_dst);
> + if (rc)
> + return rc;
> +
> + scoped_guard(spinlock_bh, &qp->ep_rx_lock) {
> + if (ntb_edma_ring_full(READ_ONCE(qp->rd_prod),
> + READ_ONCE(qp->rd_cons))) {
> + rc = -ENOSPC;
> + goto out_unmap;
> + }
> +
> + idx = ntb_edma_ring_idx(qp->rd_prod);
> + in = NTB_DESC_RD_EP_I(qp, idx);
> + out = NTB_DESC_RD_EP_O(qp, idx);
> +
> + iowrite32(len, &out->len);
> + iowrite64(ep_dst, &out->addr);
> +
> + WARN_ON(in->flags & DESC_DONE_FLAG);
> + in->data = (uintptr_t)entry;
> + entry->addr = ep_dst;
> +
> + /* Ensure len/addr are visible before the head update */
> + dma_wmb();
> +
> + WRITE_ONCE(qp->rd_prod, qp->rd_prod + 1);
> + iowrite32(qp->rd_prod, NTB_HEAD_RD_EP_O(qp));
> + }
> + return 0;
> +out_unmap:
> + dma_unmap_single(dma_dev, ep_dst, len, DMA_FROM_DEVICE);
> + return rc;
> +}
> +
> +static int ntb_transport_edma_rx_enqueue(struct ntb_transport_qp *qp,
> + struct ntb_queue_entry *entry)
> +{
> + int rc;
> +
> + /* The behaviour is the same as the default backend for RC side */
> + if (ntb_qp_edma_is_ep(qp)) {
> + rc = ntb_transport_edma_ep_rx_enqueue(qp, entry);
> + if (rc) {
> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> + &qp->rx_free_q);
> + return rc;
> + }
> + }
> +
> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q);
> +
> + if (qp->active)
> + tasklet_schedule(&qp->rxc_db_work);
> +
> + return 0;
> +}
> +
> +static void ntb_transport_edma_rx_poll(struct ntb_transport_qp *qp)
> +{
> + struct ntb_transport_ctx *nt = qp->transport;
> +
> + if (ntb_qp_edma_is_rc(qp))
> + ntb_transport_edma_rc_poll(qp);
> + else if (ntb_qp_edma_is_ep(qp)) {
> + /*
> + * Make sure we poll the rings even if an eDMA interrupt is
> + * cleared on the RC side earlier.
> + */
> + queue_work(nt->wq, &qp->read_work);
> + queue_work(nt->wq, &qp->write_work);
> + } else
> + /* Unreachable */
> + WARN_ON_ONCE(1);
> +}
> +
> +static void ntb_transport_edma_read_work(struct work_struct *work)
> +{
> + struct ntb_transport_qp *qp = container_of(
> + work, struct ntb_transport_qp, read_work);
> +
> + if (ntb_qp_edma_is_rc(qp))
> + ntb_transport_edma_rc_read_complete_work(work);
> + else if (ntb_qp_edma_is_ep(qp))
> + ntb_transport_edma_ep_read_work(work);
> + else
> + /* Unreachable */
> + WARN_ON_ONCE(1);
> +}
> +
> +static void ntb_transport_edma_write_work(struct work_struct *work)
> +{
> + struct ntb_transport_qp *qp = container_of(
> + work, struct ntb_transport_qp, write_work);
> +
> + if (ntb_qp_edma_is_rc(qp))
> + ntb_transport_edma_rc_write_complete_work(work);
> + else if (ntb_qp_edma_is_ep(qp))
> + ntb_transport_edma_ep_write_work(work);
> + else
> + /* Unreachable */
> + WARN_ON_ONCE(1);
> +}
> +
> +static void ntb_transport_edma_init_queue(struct ntb_transport_ctx *nt,
> + unsigned int qp_num)
> +{
> + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
> +
> + qp->wr_cons = 0;
> + qp->rd_cons = 0;
> + qp->wr_prod = 0;
> + qp->rd_prod = 0;
> + qp->wr_issue = 0;
> + qp->rd_issue = 0;
> +
> + INIT_WORK(&qp->db_work, ntb_transport_edma_db_work);
> + INIT_WORK(&qp->read_work, ntb_transport_edma_read_work);
> + INIT_WORK(&qp->write_work, ntb_transport_edma_write_work);
> +}
> +
> +static void ntb_transport_edma_create_queue(struct ntb_transport_ctx *nt,
> + struct ntb_transport_qp *qp)
> +{
> + spin_lock_init(&qp->ep_tx_lock);
> + spin_lock_init(&qp->ep_rx_lock);
> + spin_lock_init(&qp->rc_lock);
> +}
> +
> +static const struct ntb_transport_backend_ops edma_backend_ops = {
> + .setup_qp_mw = ntb_transport_edma_setup_qp_mw,
> + .tx_free_entry = ntb_transport_edma_tx_free_entry,
> + .tx_enqueue = ntb_transport_edma_tx_enqueue,
> + .rx_enqueue = ntb_transport_edma_rx_enqueue,
> + .rx_poll = ntb_transport_edma_rx_poll,
> + .debugfs_stats_show = ntb_transport_edma_debugfs_stats_show,
> +};
> +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> +
> /**
> * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
> * @qp: NTB transport layer queue to be enabled
> --
> 2.48.1
>
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-12-01 21:41 ` Frank Li
@ 2025-12-02 6:43 ` Koichiro Den
2025-12-02 15:42 ` Frank Li
0 siblings, 1 reply; 19+ messages in thread
From: Koichiro Den @ 2025-12-02 6:43 UTC (permalink / raw)
To: Frank Li
Cc: ntb, linux-pci, dmaengine, linux-kernel, mani, kwilczynski,
kishon, bhelgaas, corbet, vkoul, jdmason, dave.jiang, allenbh,
Basavaraj.Natikar, Shyam-sundar.S-k, kurt.schwemmer, logang,
jingoohan1, lpieralisi, robh, jbrunet, fancer.lancer, arnd,
pstanner, elfring
On Mon, Dec 01, 2025 at 04:41:05PM -0500, Frank Li wrote:
> On Sun, Nov 30, 2025 at 01:03:58AM +0900, Koichiro Den wrote:
> > Add a new transport backend that uses a remote DesignWare eDMA engine
> > located on the NTB endpoint to move data between host and endpoint.
> >
> > In this mode:
> >
> > - The endpoint exposes a dedicated memory window that contains the
> > eDMA register block followed by a small control structure (struct
> > ntb_edma_info) and per-channel linked-list (LL) rings.
> >
> > - On the endpoint side, ntb_edma_setup_mws() allocates the control
> > structure and LL rings in endpoint memory, then programs an inbound
> > iATU region so that the host can access them via a peer MW.
> >
> > - On the host side, ntb_edma_setup_peer() ioremaps the peer MW, reads
> > ntb_edma_info and configures a dw-edma DMA device to use the LL
> > rings provided by the endpoint.
> >
> > - ntb_transport is extended with a new backend_ops implementation that
> > routes TX and RX enqueue/poll operations through the remote eDMA
> > rings while keeping the existing shared-memory backend intact.
> >
> > - The host signals the endpoint via a dedicated DMA read channel.
> > 'use_msi' module option is ignored when 'use_remote_edma=1'.
> >
> > The new mode is guarded by a Kconfig option (NTB_TRANSPORT_EDMA) and a
> > module parameter (use_remote_edma). When disabled, the existing
> > ntb_transport behaviour is unchanged.
> >
> > Signed-off-by: Koichiro Den <den@valinux.co.jp>
> > ---
> > drivers/ntb/Kconfig | 11 +
> > drivers/ntb/Makefile | 3 +
> > drivers/ntb/ntb_edma.c | 628 ++++++++
> > drivers/ntb/ntb_edma.h | 128 ++
> > .../{ntb_transport.c => ntb_transport_core.c} | 1281 ++++++++++++++++-
> > 5 files changed, 2048 insertions(+), 3 deletions(-)
> > create mode 100644 drivers/ntb/ntb_edma.c
> > create mode 100644 drivers/ntb/ntb_edma.h
> > rename drivers/ntb/{ntb_transport.c => ntb_transport_core.c} (65%)
> >
> > diff --git a/drivers/ntb/Kconfig b/drivers/ntb/Kconfig
> > index df16c755b4da..db63f02bb116 100644
> > --- a/drivers/ntb/Kconfig
> > +++ b/drivers/ntb/Kconfig
> > @@ -37,4 +37,15 @@ config NTB_TRANSPORT
> >
> > If unsure, say N.
> >
> > +config NTB_TRANSPORT_EDMA
> > + bool "NTB Transport backed by remote eDMA"
> > + depends on NTB_TRANSPORT
> > + depends on PCI
> > + select DMA_ENGINE
> > + help
> > + Enable a transport backend that uses a remote DesignWare eDMA engine
> > + exposed through a dedicated NTB memory window. The host uses the
> > + endpoint's eDMA engine to move data in both directions.
> > + Say Y here if you intend to use the 'use_remote_edma' module parameter.
> > +
> > endif # NTB
> > diff --git a/drivers/ntb/Makefile b/drivers/ntb/Makefile
> > index 3a6fa181ff99..51f0e1e3aec7 100644
> > --- a/drivers/ntb/Makefile
> > +++ b/drivers/ntb/Makefile
> > @@ -4,3 +4,6 @@ obj-$(CONFIG_NTB_TRANSPORT) += ntb_transport.o
> >
> > ntb-y := core.o
> > ntb-$(CONFIG_NTB_MSI) += msi.o
> > +
> > +ntb_transport-y := ntb_transport_core.o
> > +ntb_transport-$(CONFIG_NTB_TRANSPORT_EDMA) += ntb_edma.o
> > diff --git a/drivers/ntb/ntb_edma.c b/drivers/ntb/ntb_edma.c
> > new file mode 100644
> > index 000000000000..cb35e0d56aa8
> > --- /dev/null
> > +++ b/drivers/ntb/ntb_edma.c
> > @@ -0,0 +1,628 @@
> > +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
> > +
> > +#include <linux/module.h>
> > +#include <linux/device.h>
> > +#include <linux/pci.h>
> > +#include <linux/ntb.h>
> > +#include <linux/io.h>
> > +#include <linux/iommu.h>
> > +#include <linux/dmaengine.h>
> > +#include <linux/pci-epc.h>
> > +#include <linux/dma/edma.h>
> > +#include <linux/irq.h>
> > +#include <linux/irqdomain.h>
> > +#include <linux/of.h>
> > +#include <linux/of_irq.h>
> > +#include <dt-bindings/interrupt-controller/arm-gic.h>
> > +
> > +#include "ntb_edma.h"
> > +
> > +/*
> > + * The interrupt register offsets below are taken from the DesignWare
> > + * eDMA "unrolled" register map (EDMA_MF_EDMA_UNROLL). The remote eDMA
> > + * backend currently only supports this layout.
> > + */
> > +#define DMA_WRITE_INT_STATUS_OFF 0x4c
> > +#define DMA_WRITE_INT_MASK_OFF 0x54
> > +#define DMA_WRITE_INT_CLEAR_OFF 0x58
> > +#define DMA_READ_INT_STATUS_OFF 0xa0
> > +#define DMA_READ_INT_MASK_OFF 0xa8
> > +#define DMA_READ_INT_CLEAR_OFF 0xac
>
> Not sure why need access EDMA register because EMDA driver already export
> as dmaengine driver.
These are intended for EP use. In my current design I intentionally don't
use the standard dw-edma dmaengine driver on the EP side.
>
> > +
> > +#define NTB_EDMA_NOTIFY_MAX_QP 64
> > +
> > +static unsigned int edma_spi = 417; /* 0x1a1 */
> > +module_param(edma_spi, uint, 0644);
> > +MODULE_PARM_DESC(edma_spi, "SPI number used by remote eDMA interrupt (EP local)");
> > +
> > +static u64 edma_regs_phys = 0xe65d5000;
> > +module_param(edma_regs_phys, ullong, 0644);
> > +MODULE_PARM_DESC(edma_regs_phys, "Physical base address of local eDMA registers (EP)");
> > +
> > +static unsigned long edma_regs_size = 0x1200;
> > +module_param(edma_regs_size, ulong, 0644);
> > +MODULE_PARM_DESC(edma_regs_size, "Size of the local eDMA register space (EP)");
> > +
> > +struct ntb_edma_intr {
> > + u32 db[NTB_EDMA_NOTIFY_MAX_QP];
> > +};
> > +
> > +struct ntb_edma_ctx {
> > + void *ll_wr_virt[EDMA_WR_CH_NUM];
> > + dma_addr_t ll_wr_phys[EDMA_WR_CH_NUM];
> > + void *ll_rd_virt[EDMA_RD_CH_NUM + 1];
> > + dma_addr_t ll_rd_phys[EDMA_RD_CH_NUM + 1];
> > +
> > + struct ntb_edma_intr *intr_ep_virt;
> > + dma_addr_t intr_ep_phys;
> > + struct ntb_edma_intr *intr_rc_virt;
> > + dma_addr_t intr_rc_phys;
> > + u32 notify_qp_max;
> > +
> > + bool initialized;
> > +};
> > +
> > +static struct ntb_edma_ctx edma_ctx;
> > +
> > +typedef void (*ntb_edma_interrupt_cb_t)(void *data, int qp_num);
> > +
> > +struct ntb_edma_interrupt {
> > + int virq;
> > + void __iomem *base;
> > + ntb_edma_interrupt_cb_t cb;
> > + void *data;
> > +};
> > +
> > +static struct ntb_edma_interrupt ntb_edma_intr;
> > +
> > +static int ntb_edma_map_spi_to_virq(struct device *dev, unsigned int spi)
> > +{
> > + struct device_node *np = dev_of_node(dev);
> > + struct device_node *parent;
> > + struct irq_fwspec fwspec = { 0 };
> > + int virq;
> > +
> > + parent = of_irq_find_parent(np);
> > + if (!parent)
> > + return -ENODEV;
> > +
> > + fwspec.fwnode = of_fwnode_handle(parent);
> > + fwspec.param_count = 3;
> > + fwspec.param[0] = GIC_SPI;
> > + fwspec.param[1] = spi;
> > + fwspec.param[2] = IRQ_TYPE_LEVEL_HIGH;
> > +
> > + virq = irq_create_fwspec_mapping(&fwspec);
> > + of_node_put(parent);
> > + return (virq > 0) ? virq : -EINVAL;
> > +}
> > +
> > +static irqreturn_t ntb_edma_isr(int irq, void *data)
> > +{
>
> Not sue why dw_edma_interrupt_write/read() does work for your case. Suppose
> just register callback for dmeengine.
If we ran dw_edma_probe() on both the EP and RC sides and let the dmaengine
callbacks handle int_status/int_clear, I think we could hit races. One side
might clear a status bit before the other side has a chance to see it and
invoke its callback. Please correct me if I'm missing something here.
To avoid that, in my current implementation, the RC side handles the
status/int_clear registers in the usual way, and the EP side only tries to
suppress needless edma_int as much as possible.
That said, I'm now wondering if it would be better to set LIE=0/RIE=1 for
the DMA transfer channels and LIE=1/RIE=0 for the notification channel.
That would require some changes on dw-edma core.
>
> > + struct ntb_edma_interrupt *v = data;
> > + u32 mask = BIT(EDMA_RD_CH_NUM);
> > + u32 i, val;
> > +
> > + /*
> > + * We do not ack interrupts here but instead we mask all local interrupt
> > + * sources except the read channel used for notification. This reduces
> > + * needless ISR invocations.
> > + *
> > + * In theory we could configure LIE=1/RIE=0 only for the notification
> > + * transfer (keeping all other channels at LIE=1/RIE=1), but that would
> > + * require intrusive changes to the dw-edma core.
> > + *
> > + * Note: The host side may have already cleared the read interrupt used
> > + * for notification, so reading DMA_READ_INT_CLEAR_OFF is not a reliable
> > + * way to detect it. As a result, we cannot reliably tell which specific
> > + * channel triggered this interrupt. intr_ep_virt->db[i] teaches us
> > + * instead.
> > + */
> > + iowrite32(~0x0, v->base + DMA_WRITE_INT_MASK_OFF);
> > + iowrite32(~mask, v->base + DMA_READ_INT_MASK_OFF);
> > +
> > + if (!v->cb || !edma_ctx.intr_ep_virt)
> > + return IRQ_HANDLED;
> > +
> > + for (i = 0; i < edma_ctx.notify_qp_max; i++) {
> > + val = READ_ONCE(edma_ctx.intr_ep_virt->db[i]);
> > + if (!val)
> > + continue;
> > +
> > + WRITE_ONCE(edma_ctx.intr_ep_virt->db[i], 0);
> > + v->cb(v->data, i);
> > + }
> > +
> > + return IRQ_HANDLED;
> > +}
> > +
> ...
> > +
> > +int ntb_edma_setup_peer(struct ntb_dev *ndev)
> > +{
> > + struct ntb_edma_info *info;
> > + unsigned int wr_cnt, rd_cnt;
> > + struct dw_edma_chip *chip;
> > + void __iomem *edma_virt;
> > + phys_addr_t edma_phys;
> > + resource_size_t mw_size;
> > + u64 off = EDMA_REG_SIZE;
> > + int peer_mw, mw_index;
> > + unsigned int i;
> > + int ret;
> > +
> > + peer_mw = ntb_peer_mw_count(ndev);
> > + if (peer_mw <= 0)
> > + return -ENODEV;
> > +
> > + mw_index = peer_mw - 1; /* last MW */
> > +
> > + ret = ntb_peer_mw_get_addr(ndev, mw_index, &edma_phys,
> > + &mw_size);
> > + if (ret)
> > + return -1;
> > +
> > + edma_virt = ioremap(edma_phys, mw_size);
> > +
> > + chip = devm_kzalloc(&ndev->dev, sizeof(*chip), GFP_KERNEL);
> > + if (!chip) {
> > + ret = -ENOMEM;
> > + return ret;
> > + }
> > +
> > + chip->dev = &ndev->pdev->dev;
> > + chip->nr_irqs = 4;
> > + chip->ops = &ntb_edma_ops;
> > + chip->flags = 0;
> > + chip->reg_base = edma_virt;
> > + chip->mf = EDMA_MF_EDMA_UNROLL;
> > +
> > + info = edma_virt + off;
> > + if (info->magic != NTB_EDMA_INFO_MAGIC)
> > + return -EINVAL;
> > + wr_cnt = info->wr_cnt;
> > + rd_cnt = info->rd_cnt;
> > + chip->ll_wr_cnt = wr_cnt;
> > + chip->ll_rd_cnt = rd_cnt;
> > + off += PAGE_SIZE;
> > +
> > + edma_ctx.notify_qp_max = NTB_EDMA_NOTIFY_MAX_QP;
> > + edma_ctx.intr_ep_phys = info->intr_dar_base;
> > + if (edma_ctx.intr_ep_phys) {
> > + edma_ctx.intr_rc_virt =
> > + dma_alloc_coherent(&ndev->pdev->dev,
> > + sizeof(struct ntb_edma_intr),
> > + &edma_ctx.intr_rc_phys,
> > + GFP_KERNEL);
> > + if (!edma_ctx.intr_rc_virt)
> > + return -ENOMEM;
> > + memset(edma_ctx.intr_rc_virt, 0,
> > + sizeof(struct ntb_edma_intr));
> > + }
> > +
> > + for (i = 0; i < wr_cnt; i++) {
> > + chip->ll_region_wr[i].vaddr.io = edma_virt + off;
> > + chip->ll_region_wr[i].paddr = info->ll_wr_phys[i];
> > + chip->ll_region_wr[i].sz = DMA_LLP_MEM_SIZE;
> > + off += DMA_LLP_MEM_SIZE;
> > + }
> > + for (i = 0; i < rd_cnt; i++) {
> > + chip->ll_region_rd[i].vaddr.io = edma_virt + off;
> > + chip->ll_region_rd[i].paddr = info->ll_rd_phys[i];
> > + chip->ll_region_rd[i].sz = DMA_LLP_MEM_SIZE;
> > + off += DMA_LLP_MEM_SIZE;
> > + }
> > +
> > + if (!pci_dev_msi_enabled(ndev->pdev))
> > + return -ENXIO;
> > +
> > + ret = dw_edma_probe(chip);
>
> I think dw_edma_probe() should be in ntb_hw_epf.c, which provide DMA
> dma engine support.
>
> EP side, suppose default dwc controller driver already setup edma engine,
> so use correct filter function, you should get dma chan.
I intentionally hid edma for EP side in .dts patch in [RFC PATCH v2 26/27]
so that RC side only manages eDMA remotely and avoids the potential race
condition I mentioned above.
Thanks for reviewing,
Koichiro
>
> Frank
>
> > + if (ret) {
> > + dev_err(&ndev->dev, "dw_edma_probe failed: %d\n", ret);
> > + return ret;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +struct ntb_edma_filter {
> > + struct device *dma_dev;
> > + u32 direction;
> > +};
> > +
> > +static bool ntb_edma_filter_fn(struct dma_chan *chan, void *arg)
> > +{
> > + struct ntb_edma_filter *filter = arg;
> > + u32 dir = filter->direction;
> > + struct dma_slave_caps caps;
> > + int ret;
> > +
> > + if (chan->device->dev != filter->dma_dev)
> > + return false;
> > +
> > + ret = dma_get_slave_caps(chan, &caps);
> > + if (ret < 0)
> > + return false;
> > +
> > + return !!(caps.directions & dir);
> > +}
> > +
> > +void ntb_edma_teardown_chans(struct ntb_edma_chans *edma)
> > +{
> > + unsigned int i;
> > +
> > + for (i = 0; i < edma->num_wr_chan; i++)
> > + dma_release_channel(edma->wr_chan[i]);
> > +
> > + for (i = 0; i < edma->num_rd_chan; i++)
> > + dma_release_channel(edma->rd_chan[i]);
> > +
> > + if (edma->intr_chan)
> > + dma_release_channel(edma->intr_chan);
> > +}
> > +
> > +int ntb_edma_setup_chans(struct device *dma_dev, struct ntb_edma_chans *edma)
> > +{
> > + struct ntb_edma_filter filter;
> > + dma_cap_mask_t dma_mask;
> > + unsigned int i;
> > +
> > + dma_cap_zero(dma_mask);
> > + dma_cap_set(DMA_SLAVE, dma_mask);
> > +
> > + memset(edma, 0, sizeof(*edma));
> > + edma->dev = dma_dev;
> > +
> > + filter.dma_dev = dma_dev;
> > + filter.direction = BIT(DMA_DEV_TO_MEM);
> > + for (i = 0; i < EDMA_WR_CH_NUM; i++) {
> > + edma->wr_chan[i] = dma_request_channel(dma_mask,
> > + ntb_edma_filter_fn,
> > + &filter);
> > + if (!edma->wr_chan[i])
> > + break;
> > + edma->num_wr_chan++;
> > + }
> > +
> > + filter.direction = BIT(DMA_MEM_TO_DEV);
> > + for (i = 0; i < EDMA_RD_CH_NUM; i++) {
> > + edma->rd_chan[i] = dma_request_channel(dma_mask,
> > + ntb_edma_filter_fn,
> > + &filter);
> > + if (!edma->rd_chan[i])
> > + break;
> > + edma->num_rd_chan++;
> > + }
> > +
> > + edma->intr_chan = dma_request_channel(dma_mask, ntb_edma_filter_fn,
> > + &filter);
> > + if (!edma->intr_chan)
> > + dev_warn(dma_dev,
> > + "Remote eDMA notify channel could not be allocated\n");
> > +
> > + if (!edma->num_wr_chan || !edma->num_rd_chan) {
> > + dev_warn(dma_dev, "Remote eDMA channels failed to initialize\n");
> > + ntb_edma_teardown_chans(edma);
> > + return -ENODEV;
> > + }
> > + return 0;
> > +}
> > +
> > +struct dma_chan *ntb_edma_pick_chan(struct ntb_edma_chans *edma,
> > + remote_edma_dir_t dir)
> > +{
> > + unsigned int n, cur, idx;
> > + struct dma_chan **chans;
> > + atomic_t *cur_chan;
> > +
> > + if (dir == REMOTE_EDMA_WRITE) {
> > + n = edma->num_wr_chan;
> > + chans = edma->wr_chan;
> > + cur_chan = &edma->cur_wr_chan;
> > + } else {
> > + n = edma->num_rd_chan;
> > + chans = edma->rd_chan;
> > + cur_chan = &edma->cur_rd_chan;
> > + }
> > + if (WARN_ON_ONCE(!n))
> > + return NULL;
> > +
> > + /* Simple round-robin */
> > + cur = (unsigned int)atomic_inc_return(cur_chan) - 1;
> > + idx = cur % n;
> > + return chans[idx];
> > +}
> > +
> > +int ntb_edma_notify_peer(struct ntb_edma_chans *edma, int qp_num)
> > +{
> > + struct dma_async_tx_descriptor *txd;
> > + struct dma_slave_config cfg;
> > + struct scatterlist sgl;
> > + dma_cookie_t cookie;
> > + struct device *dev;
> > +
> > + if (!edma || !edma->intr_chan)
> > + return -ENXIO;
> > +
> > + if (qp_num < 0 || qp_num >= edma_ctx.notify_qp_max)
> > + return -EINVAL;
> > +
> > + if (!edma_ctx.intr_rc_virt || !edma_ctx.intr_ep_phys)
> > + return -EINVAL;
> > +
> > + dev = edma->dev;
> > + if (!dev)
> > + return -ENODEV;
> > +
> > + WRITE_ONCE(edma_ctx.intr_rc_virt->db[qp_num], 1);
> > +
> > + /* Ensure store is visible before kicking the DMA transfer */
> > + wmb();
> > +
> > + sg_init_table(&sgl, 1);
> > + sg_dma_address(&sgl) = edma_ctx.intr_rc_phys + qp_num * sizeof(u32);
> > + sg_dma_len(&sgl) = sizeof(u32);
> > +
> > + memset(&cfg, 0, sizeof(cfg));
> > + cfg.dst_addr = edma_ctx.intr_ep_phys + qp_num * sizeof(u32);
> > + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> > + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> > + cfg.direction = DMA_MEM_TO_DEV;
> > +
> > + if (dmaengine_slave_config(edma->intr_chan, &cfg))
> > + return -EINVAL;
> > +
> > + txd = dmaengine_prep_slave_sg(edma->intr_chan, &sgl, 1,
> > + DMA_MEM_TO_DEV,
> > + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> > + if (!txd)
> > + return -ENOSPC;
> > +
> > + cookie = dmaengine_submit(txd);
> > + if (dma_submit_error(cookie))
> > + return -ENOSPC;
> > +
> > + dma_async_issue_pending(edma->intr_chan);
> > + return 0;
> > +}
> > diff --git a/drivers/ntb/ntb_edma.h b/drivers/ntb/ntb_edma.h
> > new file mode 100644
> > index 000000000000..da0451827edb
> > --- /dev/null
> > +++ b/drivers/ntb/ntb_edma.h
> > @@ -0,0 +1,128 @@
> > +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
> > +#ifndef _NTB_EDMA_H_
> > +#define _NTB_EDMA_H_
> > +
> > +#include <linux/completion.h>
> > +#include <linux/device.h>
> > +#include <linux/interrupt.h>
> > +
> > +#define EDMA_REG_SIZE SZ_64K
> > +#define DMA_LLP_MEM_SIZE SZ_4K
> > +#define EDMA_WR_CH_NUM 4
> > +#define EDMA_RD_CH_NUM 4
> > +#define NTB_EDMA_MAX_CH 8
> > +
> > +#define NTB_EDMA_INFO_MAGIC 0x45444D41 /* "EDMA" */
> > +#define NTB_EDMA_INFO_OFF EDMA_REG_SIZE
> > +
> > +#define NTB_EDMA_RING_ORDER 7
> > +#define NTB_EDMA_RING_ENTRIES (1U << NTB_EDMA_RING_ORDER)
> > +#define NTB_EDMA_RING_MASK (NTB_EDMA_RING_ENTRIES - 1)
> > +
> > +typedef void (*ntb_edma_interrupt_cb_t)(void *data, int qp_num);
> > +
> > +/*
> > + * REMOTE_EDMA_EP:
> > + * Endpoint owns the eDMA engine and pushes descriptors into a shared MW.
> > + *
> > + * REMOTE_EDMA_RC:
> > + * Root Complex controls the endpoint eDMA through the shared MW and
> > + * drives reads/writes on behalf of the host.
> > + */
> > +typedef enum {
> > + REMOTE_EDMA_UNKNOWN,
> > + REMOTE_EDMA_EP,
> > + REMOTE_EDMA_RC,
> > +} remote_edma_mode_t;
> > +
> > +typedef enum {
> > + REMOTE_EDMA_WRITE,
> > + REMOTE_EDMA_READ,
> > +} remote_edma_dir_t;
> > +
> > +/*
> > + * Layout of remote eDMA MW (EP local address space, RC sees via peer MW):
> > + *
> > + * 0 .. EDMA_REG_SIZE-1 : DesignWare eDMA registers
> > + * EDMA_REG_SIZE .. +PAGE_SIZE : struct ntb_edma_info (EP writes, RC reads)
> > + * +PAGE_SIZE .. : LL ring buffers (EP allocates phys addresses,
> > + * RC configures via dw_edma)
> > + *
> > + * ntb_edma_setup_mws() on EP:
> > + * - allocates ntb_edma_info and LLs in EP memory
> > + * - programs inbound iATU so that RC peer MW[n] points at this block
> > + *
> > + * ntb_edma_setup_peer() on RC:
> > + * - ioremaps peer MW[n]
> > + * - reads ntb_edma_info
> > + * - sets up dw_edma_chip ll_region_* from that info
> > + */
> > +struct ntb_edma_info {
> > + u32 magic;
> > + u16 wr_cnt;
> > + u16 rd_cnt;
> > + u64 regs_phys;
> > + u32 ll_stride;
> > + u32 rsvd;
> > + u64 ll_wr_phys[NTB_EDMA_MAX_CH];
> > + u64 ll_rd_phys[NTB_EDMA_MAX_CH];
> > +
> > + u64 intr_dar_base;
> > +} __packed;
> > +
> > +struct ll_dma_addrs {
> > + dma_addr_t wr[EDMA_WR_CH_NUM];
> > + dma_addr_t rd[EDMA_RD_CH_NUM];
> > +};
> > +
> > +struct ntb_edma_chans {
> > + struct device *dev;
> > +
> > + struct dma_chan *wr_chan[EDMA_WR_CH_NUM];
> > + struct dma_chan *rd_chan[EDMA_RD_CH_NUM];
> > + struct dma_chan *intr_chan;
> > +
> > + unsigned int num_wr_chan;
> > + unsigned int num_rd_chan;
> > + atomic_t cur_wr_chan;
> > + atomic_t cur_rd_chan;
> > +};
> > +
> > +static __always_inline u32 ntb_edma_ring_idx(u32 v)
> > +{
> > + return v & NTB_EDMA_RING_MASK;
> > +}
> > +
> > +static __always_inline u32 ntb_edma_ring_used_entry(u32 head, u32 tail)
> > +{
> > + if (head >= tail) {
> > + WARN_ON_ONCE((head - tail) > (NTB_EDMA_RING_ENTRIES - 1));
> > + return head - tail;
> > + }
> > +
> > + WARN_ON_ONCE((U32_MAX - tail + head + 1) > (NTB_EDMA_RING_ENTRIES - 1));
> > + return U32_MAX - tail + head + 1;
> > +}
> > +
> > +static __always_inline u32 ntb_edma_ring_free_entry(u32 head, u32 tail)
> > +{
> > + return NTB_EDMA_RING_ENTRIES - ntb_edma_ring_used_entry(head, tail) - 1;
> > +}
> > +
> > +static __always_inline bool ntb_edma_ring_full(u32 head, u32 tail)
> > +{
> > + return ntb_edma_ring_free_entry(head, tail) == 0;
> > +}
> > +
> > +int ntb_edma_setup_isr(struct device *dev, struct device *epc_dev,
> > + ntb_edma_interrupt_cb_t cb, void *data);
> > +void ntb_edma_teardown_isr(struct device *dev);
> > +int ntb_edma_setup_mws(struct ntb_dev *ndev);
> > +int ntb_edma_setup_peer(struct ntb_dev *ndev);
> > +int ntb_edma_setup_chans(struct device *dma_dev, struct ntb_edma_chans *edma);
> > +struct dma_chan *ntb_edma_pick_chan(struct ntb_edma_chans *edma,
> > + remote_edma_dir_t dir);
> > +void ntb_edma_teardown_chans(struct ntb_edma_chans *edma);
> > +int ntb_edma_notify_peer(struct ntb_edma_chans *edma, int qp_num);
> > +
> > +#endif
> > diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport_core.c
> > similarity index 65%
> > rename from drivers/ntb/ntb_transport.c
> > rename to drivers/ntb/ntb_transport_core.c
> > index 907db6c93d4d..48d48921978d 100644
> > --- a/drivers/ntb/ntb_transport.c
> > +++ b/drivers/ntb/ntb_transport_core.c
> > @@ -47,6 +47,9 @@
> > * Contact Information:
> > * Jon Mason <jon.mason@intel.com>
> > */
> > +#include <linux/atomic.h>
> > +#include <linux/bug.h>
> > +#include <linux/compiler.h>
> > #include <linux/debugfs.h>
> > #include <linux/delay.h>
> > #include <linux/dmaengine.h>
> > @@ -71,6 +74,8 @@
> > #define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB"
> > #define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2)
> >
> > +#define NTB_EDMA_MAX_POLL 32
> > +
> > MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
> > MODULE_VERSION(NTB_TRANSPORT_VER);
> > MODULE_LICENSE("Dual BSD/GPL");
> > @@ -102,6 +107,13 @@ module_param(use_msi, bool, 0644);
> > MODULE_PARM_DESC(use_msi, "Use MSI interrupts instead of doorbells");
> > #endif
> >
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > +#include "ntb_edma.h"
> > +static bool use_remote_edma;
> > +module_param(use_remote_edma, bool, 0644);
> > +MODULE_PARM_DESC(use_remote_edma, "Use remote eDMA mode (when enabled, use_msi is ignored)");
> > +#endif
> > +
> > static struct dentry *nt_debugfs_dir;
> >
> > /* Only two-ports NTB devices are supported */
> > @@ -125,6 +137,14 @@ struct ntb_queue_entry {
> > struct ntb_payload_header __iomem *tx_hdr;
> > struct ntb_payload_header *rx_hdr;
> > };
> > +
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + dma_addr_t addr;
> > +
> > + /* Used by RC side only */
> > + struct scatterlist sgl;
> > + struct work_struct dma_work;
> > +#endif
> > };
> >
> > struct ntb_rx_info {
> > @@ -202,6 +222,33 @@ struct ntb_transport_qp {
> > int msi_irq;
> > struct ntb_msi_desc msi_desc;
> > struct ntb_msi_desc peer_msi_desc;
> > +
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + /*
> > + * For ensuring peer notification in non-atomic context.
> > + * ntb_peer_db_set might sleep or schedule.
> > + */
> > + struct work_struct db_work;
> > +
> > + /*
> > + * wr: remote eDMA write transfer (EP -> RC direction)
> > + * rd: remote eDMA read transfer (RC -> EP direction)
> > + */
> > + u32 wr_cons;
> > + u32 rd_cons;
> > + u32 wr_prod;
> > + u32 rd_prod;
> > + u32 wr_issue;
> > + u32 rd_issue;
> > +
> > + spinlock_t ep_tx_lock;
> > + spinlock_t ep_rx_lock;
> > + spinlock_t rc_lock;
> > +
> > + /* Completion work for read/write transfers. */
> > + struct work_struct read_work;
> > + struct work_struct write_work;
> > +#endif
> > };
> >
> > struct ntb_transport_mw {
> > @@ -249,6 +296,13 @@ struct ntb_transport_ctx {
> >
> > /* Make sure workq of link event be executed serially */
> > struct mutex link_event_lock;
> > +
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + remote_edma_mode_t remote_edma_mode;
> > + struct device *dma_dev;
> > + struct workqueue_struct *wq;
> > + struct ntb_edma_chans edma;
> > +#endif
> > };
> >
> > enum {
> > @@ -262,6 +316,19 @@ struct ntb_payload_header {
> > unsigned int flags;
> > };
> >
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > +static void ntb_transport_edma_uninit(struct ntb_transport_ctx *nt);
> > +static int ntb_transport_edma_init(struct ntb_transport_ctx *nt,
> > + unsigned int *mw_count);
> > +static void ntb_transport_edma_init_queue(struct ntb_transport_ctx *nt,
> > + unsigned int qp_num);
> > +static void ntb_transport_edma_create_queue(struct ntb_transport_ctx *nt,
> > + struct ntb_transport_qp *qp);
> > +static int ntb_transport_edma_rc_init(struct ntb_transport_ctx *nt);
> > +static int ntb_transport_edma_ep_init(struct ntb_transport_ctx *nt);
> > +static void ntb_transport_edma_rc_dma_work(struct work_struct *work);
> > +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> > +
> > /*
> > * Return the device that should be used for DMA mapping.
> > *
> > @@ -298,7 +365,7 @@ enum {
> > container_of((__drv), struct ntb_transport_client, driver)
> >
> > #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count)
> > -#define NTB_QP_DEF_NUM_ENTRIES 100
> > +#define NTB_QP_DEF_NUM_ENTRIES 128
> > #define NTB_LINK_DOWN_TIMEOUT 10
> >
> > static void ntb_transport_rxc_db(unsigned long data);
> > @@ -1015,6 +1082,10 @@ static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
> > count = ntb_spad_count(nt->ndev);
> > for (i = 0; i < count; i++)
> > ntb_spad_write(nt->ndev, i, 0);
> > +
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + ntb_edma_teardown_chans(&nt->edma);
> > +#endif
> > }
> >
> > static void ntb_transport_link_cleanup_work(struct work_struct *work)
> > @@ -1051,6 +1122,14 @@ static void ntb_transport_link_work(struct work_struct *work)
> >
> > /* send the local info, in the opposite order of the way we read it */
> >
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + rc = ntb_transport_edma_ep_init(nt);
> > + if (rc) {
> > + dev_err(&pdev->dev, "Failed to init EP: %d\n", rc);
> > + return;
> > + }
> > +#endif
> > +
> > if (nt->use_msi) {
> > rc = ntb_msi_setup_mws(ndev);
> > if (rc) {
> > @@ -1132,6 +1211,14 @@ static void ntb_transport_link_work(struct work_struct *work)
> >
> > nt->link_is_up = true;
> >
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + rc = ntb_transport_edma_rc_init(nt);
> > + if (rc) {
> > + dev_err(&pdev->dev, "Failed to init RC: %d\n", rc);
> > + goto out1;
> > + }
> > +#endif
> > +
> > for (i = 0; i < nt->qp_count; i++) {
> > struct ntb_transport_qp *qp = &nt->qp_vec[i];
> >
> > @@ -1277,6 +1364,8 @@ static const struct ntb_transport_backend_ops default_backend_ops = {
> > .debugfs_stats_show = ntb_transport_default_debugfs_stats_show,
> > };
> >
> > +static const struct ntb_transport_backend_ops edma_backend_ops;
> > +
> > static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> > {
> > struct ntb_transport_ctx *nt;
> > @@ -1311,7 +1400,23 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> >
> > nt->ndev = ndev;
> >
> > - nt->backend_ops = default_backend_ops;
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + if (use_remote_edma) {
> > + rc = ntb_transport_edma_init(nt, &mw_count);
> > + if (rc) {
> > + nt->mw_count = 0;
> > + goto err;
> > + }
> > + nt->backend_ops = edma_backend_ops;
> > +
> > + /*
> > + * On remote eDMA mode, we reserve a read channel for Host->EP
> > + * interruption.
> > + */
> > + use_msi = false;
> > + } else
> > +#endif
> > + nt->backend_ops = default_backend_ops;
> >
> > /*
> > * If we are using MSI, and have at least one extra memory window,
> > @@ -1402,6 +1507,10 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> > rc = ntb_transport_init_queue(nt, i);
> > if (rc)
> > goto err2;
> > +
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + ntb_transport_edma_init_queue(nt, i);
> > +#endif
> > }
> >
> > INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);
> > @@ -1433,6 +1542,9 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> > }
> > kfree(nt->mw_vec);
> > err:
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + ntb_transport_edma_uninit(nt);
> > +#endif
> > kfree(nt);
> > return rc;
> > }
> > @@ -2055,11 +2167,16 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
> >
> > nt->qp_bitmap_free &= ~qp_bit;
> >
> > + qp->qp_bit = qp_bit;
> > qp->cb_data = data;
> > qp->rx_handler = handlers->rx_handler;
> > qp->tx_handler = handlers->tx_handler;
> > qp->event_handler = handlers->event_handler;
> >
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + ntb_transport_edma_create_queue(nt, qp);
> > +#endif
> > +
> > dma_cap_zero(dma_mask);
> > dma_cap_set(DMA_MEMCPY, dma_mask);
> >
> > @@ -2105,6 +2222,9 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
> > goto err1;
> >
> > entry->qp = qp;
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + INIT_WORK(&entry->dma_work, ntb_transport_edma_rc_dma_work);
> > +#endif
> > ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> > &qp->rx_free_q);
> > }
> > @@ -2156,8 +2276,8 @@ EXPORT_SYMBOL_GPL(ntb_transport_create_queue);
> > */
> > void ntb_transport_free_queue(struct ntb_transport_qp *qp)
> > {
> > - struct pci_dev *pdev;
> > struct ntb_queue_entry *entry;
> > + struct pci_dev *pdev;
> > u64 qp_bit;
> >
> > if (!qp)
> > @@ -2208,6 +2328,10 @@ void ntb_transport_free_queue(struct ntb_transport_qp *qp)
> > tasklet_kill(&qp->rxc_db_work);
> >
> > cancel_delayed_work_sync(&qp->link_work);
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + cancel_work_sync(&qp->read_work);
> > + cancel_work_sync(&qp->write_work);
> > +#endif
> >
> > qp->cb_data = NULL;
> > qp->rx_handler = NULL;
> > @@ -2346,6 +2470,1157 @@ int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
> > }
> > EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue);
> >
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > +/*
> > + * Remote eDMA mode implementation
> > + */
> > +struct ntb_edma_desc {
> > + u32 len;
> > + u32 flags;
> > + u64 addr; /* DMA address */
> > + u64 data;
> > +};
> > +
> > +struct ntb_edma_ring {
> > + struct ntb_edma_desc desc[NTB_EDMA_RING_ENTRIES];
> > + u32 head;
> > + u32 tail;
> > +};
> > +
> > +#define NTB_EDMA_DESC_OFF(i) ((size_t)(i) * sizeof(struct ntb_edma_desc))
> > +
> > +#define __NTB_EDMA_CHECK_INDEX(_i) \
> > +({ \
> > + unsigned long __i = (unsigned long)(_i); \
> > + WARN_ONCE(__i >= (unsigned long)NTB_EDMA_RING_ENTRIES, \
> > + "ntb_edma: index i=%lu >= ring_entries=%lu\n", \
> > + __i, (unsigned long)NTB_EDMA_RING_ENTRIES); \
> > + __i; \
> > +})
> > +
> > +#define NTB_EDMA_DESC_I(qp, i, n) \
> > +({ \
> > + typeof(qp) __qp = (qp); \
> > + unsigned long __i = __NTB_EDMA_CHECK_INDEX(i); \
> > + (struct ntb_edma_desc *) \
> > + ((char *)(__qp)->rx_buff + \
> > + (sizeof(struct ntb_edma_ring) * n) + \
> > + NTB_EDMA_DESC_OFF(__i)); \
> > +})
> > +
> > +#define NTB_EDMA_DESC_O(qp, i, n) \
> > +({ \
> > + typeof(qp) __qp = (qp); \
> > + unsigned long __i = __NTB_EDMA_CHECK_INDEX(i); \
> > + (struct ntb_edma_desc __iomem *) \
> > + ((char __iomem *)(__qp)->tx_mw + \
> > + (sizeof(struct ntb_edma_ring) * n) + \
> > + NTB_EDMA_DESC_OFF(__i)); \
> > +})
> > +
> > +#define NTB_EDMA_HEAD_I(qp, n) ((u32 *)((char *)qp->rx_buff + \
> > + (sizeof(struct ntb_edma_ring) * n) + \
> > + offsetof(struct ntb_edma_ring, head)))
> > +#define NTB_EDMA_HEAD_O(qp, n) ((u32 *)((char __iomem *)qp->tx_mw + \
> > + (sizeof(struct ntb_edma_ring) * n) + \
> > + offsetof(struct ntb_edma_ring, head)))
> > +#define NTB_EDMA_TAIL_I(qp, n) ((u32 *)((char *)qp->rx_buff + \
> > + (sizeof(struct ntb_edma_ring) * n) + \
> > + offsetof(struct ntb_edma_ring, tail)))
> > +#define NTB_EDMA_TAIL_O(qp, n) ((u32 *)((char __iomem *)qp->tx_mw + \
> > + (sizeof(struct ntb_edma_ring) * n) + \
> > + offsetof(struct ntb_edma_ring, tail)))
> > +
> > +/*
> > + * Macro naming rule:
> > + * NTB_DESC_RD_EP_I (as an example)
> > + * ^^ ^^ ^
> > + * : : `-- I(n) or O(ut). In = Read, Out = Write.
> > + * : `----- Who uses this macro.
> > + * `-------- DESC / HEAD / TAIL
> > + *
> > + * Read transfers (RC->EP):
> > + *
> > + * EP view (outbound, written via NTB):
> > + * - descs: NTB_DESC_RD_EP_O(qp, i) / NTB_DESC_RD_EP_I(qp, i)
> > + * [ len ][ flags ][ addr ][ data ]
> > + * [ len ][ flags ][ addr ][ data ]
> > + * :
> > + * [ len ][ flags ][ addr ][ data ]
> > + * - head: NTB_HEAD_RD_EP_O(qp)
> > + * - tail: NTB_TAIL_RD_EP_I(qp)
> > + *
> > + * RC view (inbound, local mapping):
> > + * - descs: NTB_DESC_RD_RC_I(qp, i) / NTB_DESC_RD_RC_O(qp, i)
> > + * [ len ][ flags ][ addr ][ data ]
> > + * [ len ][ flags ][ addr ][ data ]
> > + * :
> > + * [ len ][ flags ][ addr ][ data ]
> > + * - head: NTB_HEAD_RD_RC_I(qp)
> > + * - tail: NTB_TAIL_RD_RC_O(qp)
> > + *
> > + * Write transfers (EP -> RC) are analogous but use
> > + * NTB_DESC_WR_{EP_O,RC_I}(), NTB_HEAD_WR_{EP_O,RC_I}(),
> > + * and NTB_TAIL_WR_{EP_I,RC_O}().
> > + */
> > +#define NTB_DESC_RD_EP_I(qp, i) NTB_EDMA_DESC_I(qp, i, 0)
> > +#define NTB_DESC_RD_EP_O(qp, i) NTB_EDMA_DESC_O(qp, i, 0)
> > +#define NTB_DESC_WR_EP_I(qp, i) NTB_EDMA_DESC_I(qp, i, 1)
> > +#define NTB_DESC_WR_EP_O(qp, i) NTB_EDMA_DESC_O(qp, i, 1)
> > +#define NTB_DESC_RD_RC_I(qp, i) NTB_EDMA_DESC_I(qp, i, 0)
> > +#define NTB_DESC_RD_RC_O(qp, i) NTB_EDMA_DESC_O(qp, i, 0)
> > +#define NTB_DESC_WR_RC_I(qp, i) NTB_EDMA_DESC_I(qp, i, 1)
> > +#define NTB_DESC_WR_RC_O(qp, i) NTB_EDMA_DESC_O(qp, i, 1)
> > +
> > +#define NTB_HEAD_RD_EP_O(qp) NTB_EDMA_HEAD_O(qp, 0)
> > +#define NTB_HEAD_WR_EP_O(qp) NTB_EDMA_HEAD_O(qp, 1)
> > +#define NTB_HEAD_RD_RC_I(qp) NTB_EDMA_HEAD_I(qp, 0)
> > +#define NTB_HEAD_WR_RC_I(qp) NTB_EDMA_HEAD_I(qp, 1)
> > +
> > +#define NTB_TAIL_RD_EP_I(qp) NTB_EDMA_TAIL_I(qp, 0)
> > +#define NTB_TAIL_WR_EP_I(qp) NTB_EDMA_TAIL_I(qp, 1)
> > +#define NTB_TAIL_RD_RC_O(qp) NTB_EDMA_TAIL_O(qp, 0)
> > +#define NTB_TAIL_WR_RC_O(qp) NTB_EDMA_TAIL_O(qp, 1)
> > +
> > +static inline bool ntb_qp_edma_is_rc(struct ntb_transport_qp *qp)
> > +{
> > + return qp->transport->remote_edma_mode == REMOTE_EDMA_RC;
> > +}
> > +
> > +static inline bool ntb_qp_edma_is_ep(struct ntb_transport_qp *qp)
> > +{
> > + return qp->transport->remote_edma_mode == REMOTE_EDMA_EP;
> > +}
> > +
> > +static inline bool ntb_qp_edma_enabled(struct ntb_transport_qp *qp)
> > +{
> > + return ntb_qp_edma_is_rc(qp) || ntb_qp_edma_is_ep(qp);
> > +}
> > +
> > +static unsigned int ntb_transport_edma_tx_free_entry(struct ntb_transport_qp *qp)
> > +{
> > + unsigned int head, tail;
> > +
> > + if (ntb_qp_edma_is_ep(qp)) {
> > + scoped_guard(spinlock_irqsave, &qp->ep_tx_lock) {
> > + /* In this scope, only 'head' might proceed */
> > + tail = READ_ONCE(qp->wr_cons);
> > + head = READ_ONCE(qp->wr_prod);
> > + }
> > + return ntb_edma_ring_free_entry(head, tail);
> > + }
> > +
> > + scoped_guard(spinlock_irqsave, &qp->rc_lock) {
> > + /* In this scope, only 'head' might proceed */
> > + tail = READ_ONCE(qp->rd_issue);
> > + head = READ_ONCE(*NTB_HEAD_RD_RC_I(qp));
> > + }
> > + /*
> > + * On RC side, 'used' amount indicates how much EP side
> > + * has refilled, which are available for us to use for TX.
> > + */
> > + return ntb_edma_ring_used_entry(head, tail);
> > +}
> > +
> > +static void ntb_transport_edma_debugfs_stats_show(struct seq_file *s,
> > + struct ntb_transport_qp *qp)
> > +{
> > + seq_printf(s, "rx_bytes - \t%llu\n", qp->rx_bytes);
> > + seq_printf(s, "rx_pkts - \t%llu\n", qp->rx_pkts);
> > + seq_printf(s, "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
> > + seq_printf(s, "rx_buff - \t0x%p\n", qp->rx_buff);
> > + seq_printf(s, "rx_max_entry - \t%u\n", qp->rx_max_entry);
> > + seq_printf(s, "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry);
> > +
> > + seq_printf(s, "tx_bytes - \t%llu\n", qp->tx_bytes);
> > + seq_printf(s, "tx_pkts - \t%llu\n", qp->tx_pkts);
> > + seq_printf(s, "tx_ring_full - \t%llu\n", qp->tx_ring_full);
> > + seq_printf(s, "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
> > + seq_printf(s, "tx_mw - \t0x%p\n", qp->tx_mw);
> > + seq_printf(s, "tx_max_entry - \t%u\n", qp->tx_max_entry);
> > + seq_printf(s, "free tx - \t%u\n", ntb_transport_tx_free_entry(qp));
> > + seq_putc(s, '\n');
> > +
> > + seq_puts(s, "Using Remote eDMA - Yes\n");
> > + seq_printf(s, "QP Link - \t%s\n", qp->link_is_up ? "Up" : "Down");
> > +}
> > +
> > +static void ntb_transport_edma_uninit(struct ntb_transport_ctx *nt)
> > +{
> > + struct ntb_dev *ndev = nt->ndev;
> > +
> > + if (nt->remote_edma_mode == REMOTE_EDMA_EP && ndev && ndev->pdev)
> > + ntb_edma_teardown_isr(&ndev->pdev->dev);
> > +
> /pr> + if (nt->wq)
> > + destroy_workqueue(nt->wq);
> > + nt->wq = NULL;
> > +}
> > +
> > +static int ntb_transport_edma_init(struct ntb_transport_ctx *nt,
> > + unsigned int *mw_count)
> > +{
> > + struct ntb_dev *ndev = nt->ndev;
> > +
> > + /*
> > + * We need at least one MW for the transport plus one MW reserved
> > + * for the remote eDMA window (see ntb_edma_setup_mws/peer).
> > + */
> > + if (*mw_count <= 1) {
> > + dev_err(&ndev->dev,
> > + "remote eDMA requires at least two MWS (have %u)\n",
> > + *mw_count);
> > + return -ENODEV;
> > + }
> > +
> > + nt->wq = alloc_workqueue("ntb-edma-wq", WQ_UNBOUND | WQ_SYSFS, 0);
> > + if (!nt->wq) {
> > + ntb_transport_edma_uninit(nt);
> > + return -ENOMEM;
> > + }
> > +
> > + /* Reserve the last peer MW exclusively for the eDMA window. */
> > + *mw_count -= 1;
> > +
> > + return 0;
> > +}
> > +
> > +static void ntb_transport_edma_db_work(struct work_struct *work)
> > +{
> > + struct ntb_transport_qp *qp =
> > + container_of(work, struct ntb_transport_qp, db_work);
> > +
> > + ntb_peer_db_set(qp->ndev, qp->qp_bit);
> > +}
> > +
> > +static void ntb_transport_edma_notify_peer(struct ntb_transport_qp *qp)
> > +{
> > + if (ntb_qp_edma_is_rc(qp))
> > + if (!ntb_edma_notify_peer(&qp->transport->edma, qp->qp_num))
> > + return;
> > +
> > + /*
> > + * Called from contexts that may be atomic. Since ntb_peer_db_set()
> > + * may sleep, delegate the actual doorbell write to a workqueue.
> > + */
> > + queue_work(system_highpri_wq, &qp->db_work);
> > +}
> > +
> > +static void ntb_transport_edma_isr(void *data, int qp_num)
> > +{
> > + struct ntb_transport_ctx *nt = data;
> > + struct ntb_transport_qp *qp;
> > +
> > + if (qp_num < 0 || qp_num >= nt->qp_count)
> > + return;
> > +
> > + qp = &nt->qp_vec[qp_num];
> > + if (WARN_ON(!qp))
> > + return;
> > +
> > + queue_work(nt->wq, &qp->read_work);
> > + queue_work(nt->wq, &qp->write_work);
> > +}
> > +
> > +static int ntb_transport_edma_rc_init(struct ntb_transport_ctx *nt)
> > +{
> > + struct ntb_dev *ndev = nt->ndev;
> > + struct pci_dev *pdev = ndev->pdev;
> > + int rc;
> > +
> > + if (!use_remote_edma || nt->remote_edma_mode != REMOTE_EDMA_UNKNOWN)
> > + return 0;
> > +
> > + rc = ntb_edma_setup_peer(ndev);
> > + if (rc) {
> > + dev_err(&pdev->dev, "Failed to enable remote eDMA: %d\n", rc);
> > + return rc;
> > + }
> > +
> > + rc = ntb_edma_setup_chans(get_dma_dev(ndev), &nt->edma);
> > + if (rc) {
> > + dev_err(&pdev->dev, "Failed to setup eDMA channels: %d\n", rc);
> > + return rc;
> > + }
> > +
> > + nt->remote_edma_mode = REMOTE_EDMA_RC;
> > + return 0;
> > +}
> > +
> > +static int ntb_transport_edma_ep_init(struct ntb_transport_ctx *nt)
> > +{
> > + struct ntb_dev *ndev = nt->ndev;
> > + struct pci_dev *pdev = ndev->pdev;
> > + struct pci_epc *epc;
> > + int rc;
> > +
> > + if (!use_remote_edma || nt->remote_edma_mode == REMOTE_EDMA_EP)
> > + return 0;
> > +
> > + /* Only EP side can return pci_epc */
> > + epc = ntb_get_pci_epc(ndev);
> > + if (!epc)
> > + return 0;
> > +
> > + rc = ntb_edma_setup_mws(ndev);
> > + if (rc) {
> > + dev_err(&pdev->dev,
> > + "Failed to set up memory window for eDMA: %d\n", rc);
> > + return rc;
> > + }
> > +
> > + rc = ntb_edma_setup_isr(&pdev->dev, &epc->dev, ntb_transport_edma_isr, nt);
> > + if (rc) {
> > + dev_err(&pdev->dev, "Failed to setup eDMA ISR (%d)\n", rc);
> > + return rc;
> > + }
> > +
> > + nt->remote_edma_mode = REMOTE_EDMA_EP;
> > + return 0;
> > +}
> > +
> > +static int ntb_transport_edma_setup_qp_mw(struct ntb_transport_ctx *nt,
> > + unsigned int qp_num)
> > +{
> > + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
> > + struct ntb_dev *ndev = nt->ndev;
> > + struct ntb_queue_entry *entry;
> > + struct ntb_transport_mw *mw;
> > + unsigned int mw_num, mw_count, qp_count;
> > + unsigned int qp_offset, rx_info_offset;
> > + unsigned int mw_size, mw_size_per_qp;
> > + unsigned int num_qps_mw;
> > + size_t edma_total;
> > + unsigned int i;
> > + int node;
> > +
> > + mw_count = nt->mw_count;
> > + qp_count = nt->qp_count;
> > +
> > + mw_num = QP_TO_MW(nt, qp_num);
> > + mw = &nt->mw_vec[mw_num];
> > +
> > + if (!mw->virt_addr)
> > + return -ENOMEM;
> > +
> > + if (mw_num < qp_count % mw_count)
> > + num_qps_mw = qp_count / mw_count + 1;
> > + else
> > + num_qps_mw = qp_count / mw_count;
> > +
> > + mw_size = min(nt->mw_vec[mw_num].phys_size, mw->xlat_size);
> > + if (max_mw_size && mw_size > max_mw_size)
> > + mw_size = max_mw_size;
> > +
> > + mw_size_per_qp = round_down((unsigned int)mw_size / num_qps_mw, SZ_64);
> > + qp_offset = mw_size_per_qp * (qp_num / mw_count);
> > + rx_info_offset = mw_size_per_qp - sizeof(struct ntb_rx_info);
> > +
> > + qp->tx_mw_size = mw_size_per_qp;
> > + qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;
> > + if (!qp->tx_mw)
> > + return -EINVAL;
> > + qp->tx_mw_phys = nt->mw_vec[mw_num].phys_addr + qp_offset;
> > + if (!qp->tx_mw_phys)
> > + return -EINVAL;
> > + qp->rx_info = qp->tx_mw + rx_info_offset;
> > + qp->rx_buff = mw->virt_addr + qp_offset;
> > + qp->remote_rx_info = qp->rx_buff + rx_info_offset;
> > +
> > + /* Due to housekeeping, there must be at least 2 buffs */
> > + qp->tx_max_frame = min(transport_mtu, mw_size_per_qp / 2);
> > + qp->rx_max_frame = min(transport_mtu, mw_size_per_qp / 2);
> > +
> > + /* In eDMA mode, decouple from MW sizing and force ring-sized entries */
> > + edma_total = 2 * sizeof(struct ntb_edma_ring);
> > + if (rx_info_offset < edma_total) {
> > + dev_err(&ndev->dev, "Ring space requires %luB (>=%uB)\n",
> > + edma_total, rx_info_offset);
> > + return -EINVAL;
> > + }
> > + qp->tx_max_entry = NTB_EDMA_RING_ENTRIES;
> > + qp->rx_max_entry = NTB_EDMA_RING_ENTRIES;
> > +
> > + /*
> > + * Checking to see if we have more entries than the default.
> > + * We should add additional entries if that is the case so we
> > + * can be in sync with the transport frames.
> > + */
> > + node = dev_to_node(&ndev->dev);
> > + for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) {
> > + entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
> > + if (!entry)
> > + return -ENOMEM;
> > +
> > + entry->qp = qp;
> > + INIT_WORK(&entry->dma_work, ntb_transport_edma_rc_dma_work);
> > + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> > + &qp->rx_free_q);
> > + qp->rx_alloc_entry++;
> > + }
> > +
> > + memset(qp->rx_buff, 0, edma_total);
> > +
> > + qp->rx_pkts = 0;
> > + qp->tx_pkts = 0;
> > +
> > + return 0;
> > +}
> > +
> > +static int ntb_transport_edma_ep_read_complete(struct ntb_transport_qp *qp)
> > +{
> > + struct device *dma_dev = get_dma_dev(qp->ndev);
> > + struct ntb_queue_entry *entry;
> > + struct ntb_edma_desc *in;
> > + unsigned int len;
> > + u32 idx;
> > +
> > + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_TAIL_RD_EP_I(qp)),
> > + qp->rd_cons) == 0)
> > + return 0;
> > +
> > + idx = ntb_edma_ring_idx(qp->rd_cons);
> > + in = NTB_DESC_RD_EP_I(qp, idx);
> > + if (!(in->flags & DESC_DONE_FLAG))
> > + return 0;
> > +
> > + in->flags = 0;
> > + len = in->len; /* might be smaller than entry->len */
> > +
> > + entry = (struct ntb_queue_entry *)(in->data);
> > + if (WARN_ON(!entry))
> > + return 0;
> > +
> > + if (in->flags & LINK_DOWN_FLAG) {
> > + ntb_qp_link_down(qp);
> > + qp->rd_cons++;
> > + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> > + return 1;
> > + }
> > +
> > + dma_unmap_single(dma_dev, entry->addr, entry->len, DMA_FROM_DEVICE);
> > +
> > + qp->rx_bytes += len;
> > + qp->rx_pkts++;
> > + qp->rd_cons++;
> > +
> > + if (qp->rx_handler && qp->client_ready)
> > + qp->rx_handler(qp, qp->cb_data, entry->cb_data, len);
> > +
> > + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> > + return 1;
> > +}
> > +
> > +static int ntb_transport_edma_ep_write_complete(struct ntb_transport_qp *qp)
> > +{
> > + struct ntb_queue_entry *entry;
> > + struct ntb_edma_desc *in;
> > + u32 idx;
> > +
> > + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_TAIL_WR_EP_I(qp)),
> > + qp->wr_cons) == 0)
> > + return 0;
> > +
> > + idx = ntb_edma_ring_idx(qp->wr_cons);
> > + in = NTB_DESC_WR_EP_I(qp, idx);
> > +
> > + entry = (struct ntb_queue_entry *)(in->data);
> > + if (WARN_ON(!entry))
> > + return 0;
> > +
> > + qp->wr_cons++;
> > +
> > + if (qp->tx_handler)
> > + qp->tx_handler(qp, qp->cb_data, entry->cb_data, entry->len);
> > +
> > + ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q);
> > + return 1;
> > +}
> > +
> > +static void ntb_transport_edma_ep_read_work(struct work_struct *work)
> > +{
> > + struct ntb_transport_qp *qp = container_of(
> > + work, struct ntb_transport_qp, read_work);
> > + unsigned int i;
> > +
> > + for (i = 0; i < NTB_EDMA_MAX_POLL; i++) {
> > + if (!ntb_transport_edma_ep_read_complete(qp))
> > + break;
> > + }
> > +
> > + if (ntb_transport_edma_ep_read_complete(qp))
> > + queue_work(qp->transport->wq, &qp->read_work);
> > +}
> > +
> > +static void ntb_transport_edma_ep_write_work(struct work_struct *work)
> > +{
> > + struct ntb_transport_qp *qp = container_of(
> > + work, struct ntb_transport_qp, write_work);
> > + unsigned int i;
> > +
> > + for (i = 0; i < NTB_EDMA_MAX_POLL; i++) {
> > + if (!ntb_transport_edma_ep_write_complete(qp))
> > + break;
> > + }
> > +
> > + if (ntb_transport_edma_ep_write_complete(qp))
> > + queue_work(qp->transport->wq, &qp->write_work);
> > +}
> > +
> > +static void ntb_transport_edma_rc_write_complete_work(struct work_struct *work)
> > +{
> > + struct ntb_transport_qp *qp = container_of(
> > + work, struct ntb_transport_qp, write_work);
> > + struct ntb_queue_entry *entry;
> > + struct ntb_edma_desc *in;
> > + unsigned int len;
> > + void *cb_data;
> > + u32 idx;
> > +
> > + while (ntb_edma_ring_used_entry(READ_ONCE(qp->wr_issue),
> > + qp->wr_cons) != 0) {
> > + /* Paired with smp_wmb() in ntb_transport_edma_rc_poll() */
> > + smp_rmb();
> > +
> > + idx = ntb_edma_ring_idx(qp->wr_cons);
> > + in = NTB_DESC_WR_RC_I(qp, idx);
> > + entry = (struct ntb_queue_entry *)READ_ONCE(in->data);
> > + if (!entry || !(entry->flags & DESC_DONE_FLAG))
> > + break;
> > +
> > + in->data = 0;
> > +
> > + cb_data = entry->cb_data;
> > + len = entry->len;
> > +
> > + iowrite32(++qp->wr_cons, NTB_TAIL_WR_RC_O(qp));
> > +
> > + if (unlikely(entry->flags & LINK_DOWN_FLAG)) {
> > + ntb_qp_link_down(qp);
> > + continue;
> > + }
> > +
> > + ntb_transport_edma_notify_peer(qp);
> > +
> > + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> > +
> > + if (qp->rx_handler && qp->client_ready)
> > + qp->rx_handler(qp, qp->cb_data, cb_data, len);
> > +
> > + /* stat updates */
> > + qp->rx_bytes += len;
> > + qp->rx_pkts++;
> > + }
> > +}
> > +
> > +static void ntb_transport_edma_rc_write_cb(void *data,
> > + const struct dmaengine_result *res)
> > +{
> > + struct ntb_queue_entry *entry = data;
> > + struct ntb_transport_qp *qp = entry->qp;
> > + struct ntb_transport_ctx *nt = qp->transport;
> > + enum dmaengine_tx_result dma_err = res->result;
> > + struct device *dma_dev = get_dma_dev(qp->ndev);
> > +
> > + switch (dma_err) {
> > + case DMA_TRANS_READ_FAILED:
> > + case DMA_TRANS_WRITE_FAILED:
> > + case DMA_TRANS_ABORTED:
> > + entry->errors++;
> > + entry->len = -EIO;
> > + break;
> > + case DMA_TRANS_NOERROR:
> > + default:
> > + break;
> > + }
> > + dma_unmap_sg(dma_dev, &entry->sgl, 1, DMA_FROM_DEVICE);
> > + sg_dma_address(&entry->sgl) = 0;
> > +
> > + entry->flags |= DESC_DONE_FLAG;
> > +
> > + queue_work(nt->wq, &qp->write_work);
> > +}
> > +
> > +static void ntb_transport_edma_rc_read_complete_work(struct work_struct *work)
> > +{
> > + struct ntb_transport_qp *qp = container_of(
> > + work, struct ntb_transport_qp, read_work);
> > + struct ntb_edma_desc *in, __iomem *out;
> > + struct ntb_queue_entry *entry;
> > + unsigned int len;
> > + void *cb_data;
> > + u32 idx;
> > +
> > + while (ntb_edma_ring_used_entry(READ_ONCE(qp->rd_issue),
> > + qp->rd_cons) != 0) {
> > + /* Paired with smp_wmb() in ntb_transport_edma_rc_tx_enqueue() */
> > + smp_rmb();
> > +
> > + idx = ntb_edma_ring_idx(qp->rd_cons);
> > + in = NTB_DESC_RD_RC_I(qp, idx);
> > + entry = (struct ntb_queue_entry *)in->data;
> > + if (!entry || !(entry->flags & DESC_DONE_FLAG))
> > + break;
> > +
> > + in->data = 0;
> > +
> > + cb_data = entry->cb_data;
> > + len = entry->len;
> > +
> > + out = NTB_DESC_RD_RC_O(qp, idx);
> > +
> > + WRITE_ONCE(qp->rd_cons, qp->rd_cons + 1);
> > +
> > + /*
> > + * No need to add barrier in-between to enforce ordering here.
> > + * The other side proceeds only after both flags and tail are
> > + * updated.
> > + */
> > + iowrite32(entry->flags, &out->flags);
> > + iowrite32(qp->rd_cons, NTB_TAIL_RD_RC_O(qp));
> > +
> > + ntb_transport_edma_notify_peer(qp);
> > +
> > + ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
> > + &qp->tx_free_q);
> > +
> > + if (qp->tx_handler)
> > + qp->tx_handler(qp, qp->cb_data, cb_data, len);
> > +
> > + /* stat updates */
> > + qp->tx_bytes += len;
> > + qp->tx_pkts++;
> > + }
> > +}
> > +
> > +static void ntb_transport_edma_rc_read_cb(void *data,
> > + const struct dmaengine_result *res)
> > +{
> > + struct ntb_queue_entry *entry = data;
> > + struct ntb_transport_qp *qp = entry->qp;
> > + struct ntb_transport_ctx *nt = qp->transport;
> > + struct device *dma_dev = get_dma_dev(qp->ndev);
> > + enum dmaengine_tx_result dma_err = res->result;
> > +
> > + switch (dma_err) {
> > + case DMA_TRANS_READ_FAILED:
> > + case DMA_TRANS_WRITE_FAILED:
> > + case DMA_TRANS_ABORTED:
> > + entry->errors++;
> > + entry->len = -EIO;
> > + break;
> > + case DMA_TRANS_NOERROR:
> > + default:
> > + break;
> > + }
> > + dma_unmap_sg(dma_dev, &entry->sgl, 1, DMA_TO_DEVICE);
> > + sg_dma_address(&entry->sgl) = 0;
> > +
> > + entry->flags |= DESC_DONE_FLAG;
> > +
> > + queue_work(nt->wq, &qp->read_work);
> > +}
> > +
> > +static int ntb_transport_edma_rc_write_start(struct device *d,
> > + struct dma_chan *chan, size_t len,
> > + dma_addr_t ep_src, void *rc_dst,
> > + struct ntb_queue_entry *entry)
> > +{
> > + struct scatterlist *sgl = &entry->sgl;
> > + struct dma_async_tx_descriptor *txd;
> > + struct dma_slave_config cfg;
> > + dma_cookie_t cookie;
> > + int nents, rc;
> > +
> > + if (!d)
> > + return -ENODEV;
> > +
> > + if (!chan)
> > + return -ENXIO;
> > +
> > + if (WARN_ON(!ep_src || !rc_dst))
> > + return -EINVAL;
> > +
> > + if (WARN_ON(sg_dma_address(sgl)))
> > + return -EINVAL;
> > +
> > + sg_init_one(sgl, rc_dst, len);
> > + nents = dma_map_sg(d, sgl, 1, DMA_FROM_DEVICE);
> > + if (nents <= 0)
> > + return -EIO;
> > +
> > + memset(&cfg, 0, sizeof(cfg));
> > + cfg.src_addr = ep_src;
> > + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> > + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> > + cfg.direction = DMA_DEV_TO_MEM;
> > + rc = dmaengine_slave_config(chan, &cfg);
> > + if (rc)
> > + goto out_unmap;
> > +
> > + txd = dmaengine_prep_slave_sg(chan, sgl, 1, DMA_DEV_TO_MEM,
> > + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> > + if (!txd) {
> > + rc = -EIO;
> > + goto out_unmap;
> > + }
> > +
> > + txd->callback_result = ntb_transport_edma_rc_write_cb;
> > + txd->callback_param = entry;
> > +
> > + cookie = dmaengine_submit(txd);
> > + if (dma_submit_error(cookie)) {
> > + rc = -EIO;
> > + goto out_unmap;
> > + }
> > + dma_async_issue_pending(chan);
> > + return 0;
> > +out_unmap:
> > + dma_unmap_sg(d, sgl, 1, DMA_FROM_DEVICE);
> > + return rc;
> > +}
> > +
> > +static int ntb_transport_edma_rc_read_start(struct device *d,
> > + struct dma_chan *chan, size_t len,
> > + void *rc_src, dma_addr_t ep_dst,
> > + struct ntb_queue_entry *entry)
> > +{
> > + struct scatterlist *sgl = &entry->sgl;
> > + struct dma_async_tx_descriptor *txd;
> > + struct dma_slave_config cfg;
> > + dma_cookie_t cookie;
> > + int nents, rc;
> > +
> > + if (!d)
> > + return -ENODEV;
> > +
> > + if (!chan)
> > + return -ENXIO;
> > +
> > + if (WARN_ON(!rc_src || !ep_dst))
> > + return -EINVAL;
> > +
> > + if (WARN_ON(sg_dma_address(sgl)))
> > + return -EINVAL;
> > +
> > + sg_init_one(sgl, rc_src, len);
> > + nents = dma_map_sg(d, sgl, 1, DMA_TO_DEVICE);
> > + if (nents <= 0)
> > + return -EIO;
> > +
> > + memset(&cfg, 0, sizeof(cfg));
> > + cfg.dst_addr = ep_dst;
> > + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> > + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> > + cfg.direction = DMA_MEM_TO_DEV;
> > + rc = dmaengine_slave_config(chan, &cfg);
> > + if (rc)
> > + goto out_unmap;
> > +
> > + txd = dmaengine_prep_slave_sg(chan, sgl, 1, DMA_MEM_TO_DEV,
> > + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> > + if (!txd) {
> > + rc = -EIO;
> > + goto out_unmap;
> > + }
> > +
> > + txd->callback_result = ntb_transport_edma_rc_read_cb;
> > + txd->callback_param = entry;
> > +
> > + cookie = dmaengine_submit(txd);
> > + if (dma_submit_error(cookie)) {
> > + rc = -EIO;
> > + goto out_unmap;
> > + }
> > + dma_async_issue_pending(chan);
> > + return 0;
> > +out_unmap:
> > + dma_unmap_sg(d, sgl, 1, DMA_TO_DEVICE);
> > + return rc;
> > +}
> > +
> > +static void ntb_transport_edma_rc_dma_work(struct work_struct *work)
> > +{
> > + struct ntb_queue_entry *entry = container_of(
> > + work, struct ntb_queue_entry, dma_work);
> > + struct ntb_transport_qp *qp = entry->qp;
> > + struct ntb_transport_ctx *nt = qp->transport;
> > + struct device *dma_dev = get_dma_dev(qp->ndev);
> > + struct dma_chan *chan;
> > + int rc;
> > +
> > + chan = ntb_edma_pick_chan(&nt->edma, REMOTE_EDMA_WRITE);
> > + rc = ntb_transport_edma_rc_write_start(dma_dev, chan, entry->len,
> > + entry->addr, entry->buf, entry);
> > + if (rc) {
> > + entry->errors++;
> > + entry->len = -EIO;
> > + entry->flags |= DESC_DONE_FLAG;
> > + queue_work(nt->wq, &qp->write_work);
> > + return;
> > + }
> > +}
> > +
> > +static void ntb_transport_edma_rc_poll(struct ntb_transport_qp *qp)
> > +{
> > + struct ntb_transport_ctx *nt = qp->transport;
> > + unsigned int budget = NTB_EDMA_MAX_POLL;
> > + struct ntb_queue_entry *entry;
> > + struct ntb_edma_desc *in;
> > + dma_addr_t ep_src;
> > + u32 len, idx;
> > +
> > + while (budget--) {
> > + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_HEAD_WR_RC_I(qp)),
> > + qp->wr_issue) == 0)
> > + break;
> > +
> > + idx = ntb_edma_ring_idx(qp->wr_issue);
> > + in = NTB_DESC_WR_RC_I(qp, idx);
> > +
> > + len = READ_ONCE(in->len);
> > + ep_src = (dma_addr_t)READ_ONCE(in->addr);
> > +
> > + /* Prepare 'entry' for write completion */
> > + entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q);
> > + if (!entry) {
> > + qp->rx_err_no_buf++;
> > + break;
> > + }
> > + if (WARN_ON(entry->flags & DESC_DONE_FLAG))
> > + entry->flags &= ~DESC_DONE_FLAG;
> > + entry->len = len; /* NB. entry->len can be <=0 */
> > + entry->addr = ep_src;
> > +
> > + /*
> > + * ntb_transport_edma_rc_write_complete_work() checks entry->flags
> > + * so it needs to be set before wr_issue++.
> > + */
> > + in->data = (uintptr_t)entry;
> > +
> > + /* Ensure in->data visible before wr_issue++ */
> > + smp_wmb();
> > +
> > + WRITE_ONCE(qp->wr_issue, qp->wr_issue + 1);
> > +
> > + if (!len) {
> > + entry->flags |= DESC_DONE_FLAG;
> > + queue_work(nt->wq, &qp->write_work);
> > + continue;
> > + }
> > +
> > + if (in->flags & LINK_DOWN_FLAG) {
> > + dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n");
> > + entry->flags |= DESC_DONE_FLAG | LINK_DOWN_FLAG;
> > + queue_work(nt->wq, &qp->write_work);
> > + continue;
> > + }
> > +
> > + queue_work(nt->wq, &entry->dma_work);
> > + }
> > +
> > + if (!budget)
> > + tasklet_schedule(&qp->rxc_db_work);
> > +}
> > +
> > +static int ntb_transport_edma_rc_tx_enqueue(struct ntb_transport_qp *qp,
> > + struct ntb_queue_entry *entry)
> > +{
> > + struct device *dma_dev = get_dma_dev(qp->ndev);
> > + struct ntb_transport_ctx *nt = qp->transport;
> > + struct ntb_edma_desc *in, __iomem *out;
> > + unsigned int len = entry->len;
> > + struct dma_chan *chan;
> > + u32 issue, idx, head;
> > + dma_addr_t ep_dst;
> > + int rc;
> > +
> > + WARN_ON_ONCE(entry->flags & DESC_DONE_FLAG);
> > +
> > + scoped_guard(spinlock_irqsave, &qp->rc_lock) {
> > + head = READ_ONCE(*NTB_HEAD_RD_RC_I(qp));
> > + issue = qp->rd_issue;
> > + if (ntb_edma_ring_used_entry(head, issue) == 0) {
> > + qp->tx_ring_full++;
> > + return -ENOSPC;
> > + }
> > +
> > + /*
> > + * ntb_transport_edma_rc_read_complete_work() checks entry->flags
> > + * so it needs to be set before rd_issue++.
> > + */
> > + idx = ntb_edma_ring_idx(issue);
> > + in = NTB_DESC_RD_RC_I(qp, idx);
> > + in->data = (uintptr_t)entry;
> > +
> > + /* Make in->data visible before rd_issue++ */
> > + smp_wmb();
> > +
> > + WRITE_ONCE(qp->rd_issue, qp->rd_issue + 1);
> > + }
> > +
> > + /* Publish the final transfer length to the EP side */
> > + out = NTB_DESC_RD_RC_O(qp, idx);
> > + iowrite32(len, &out->len);
> > + ioread32(&out->len);
> > +
> > + if (unlikely(!len)) {
> > + entry->flags |= DESC_DONE_FLAG;
> > + queue_work(nt->wq, &qp->read_work);
> > + return 0;
> > + }
> > +
> > + /* Paired with dma_wmb() in ntb_transport_edma_ep_rx_enqueue() */
> > + dma_rmb();
> > +
> > + /* kick remote eDMA read transfer */
> > + ep_dst = (dma_addr_t)in->addr;
> > + chan = ntb_edma_pick_chan(&nt->edma, REMOTE_EDMA_READ);
> > + rc = ntb_transport_edma_rc_read_start(dma_dev, chan, len,
> > + entry->buf, ep_dst, entry);
> > + if (rc) {
> > + entry->errors++;
> > + entry->len = -EIO;
> > + entry->flags |= DESC_DONE_FLAG;
> > + queue_work(nt->wq, &qp->read_work);
> > + }
> > + return 0;
> > +}
> > +
> > +static int ntb_transport_edma_ep_tx_enqueue(struct ntb_transport_qp *qp,
> > + struct ntb_queue_entry *entry)
> > +{
> > + struct device *dma_dev = get_dma_dev(qp->ndev);
> > + struct ntb_edma_desc *in, __iomem *out;
> > + unsigned int len = entry->len;
> > + dma_addr_t ep_src = 0;
> > + u32 idx;
> > + int rc;
> > +
> > + if (likely(len)) {
> > + ep_src = dma_map_single(dma_dev, entry->buf, len,
> > + DMA_TO_DEVICE);
> > + rc = dma_mapping_error(dma_dev, ep_src);
> > + if (rc)
> > + return rc;
> > + }
> > +
> > + scoped_guard(spinlock_irqsave, &qp->ep_tx_lock) {
> > + if (ntb_edma_ring_full(qp->wr_prod, qp->wr_cons)) {
> > + rc = -ENOSPC;
> > + qp->tx_ring_full++;
> > + goto out_unmap;
> > + }
> > +
> > + idx = ntb_edma_ring_idx(qp->wr_prod);
> > + in = NTB_DESC_WR_EP_I(qp, idx);
> > + out = NTB_DESC_WR_EP_O(qp, idx);
> > +
> > + WARN_ON(in->flags & DESC_DONE_FLAG);
> > + WARN_ON(entry->flags & DESC_DONE_FLAG);
> > + in->flags = 0;
> > + in->data = (uintptr_t)entry;
> > + entry->addr = ep_src;
> > +
> > + iowrite32(len, &out->len);
> > + iowrite32(entry->flags, &out->flags);
> > + iowrite64(ep_src, &out->addr);
> > + WRITE_ONCE(qp->wr_prod, qp->wr_prod + 1);
> > +
> > + dma_wmb();
> > + iowrite32(qp->wr_prod, NTB_HEAD_WR_EP_O(qp));
> > +
> > + qp->tx_bytes += len;
> > + qp->tx_pkts++;
> > + }
> > +
> > + ntb_transport_edma_notify_peer(qp);
> > +
> > + return 0;
> > +out_unmap:
> > + if (likely(len))
> > + dma_unmap_single(dma_dev, ep_src, len, DMA_TO_DEVICE);
> > + return rc;
> > +}
> > +
> > +static int ntb_transport_edma_tx_enqueue(struct ntb_transport_qp *qp,
> > + struct ntb_queue_entry *entry,
> > + void *cb, void *data, unsigned int len,
> > + unsigned int flags)
> > +{
> > + struct device *dma_dev;
> > +
> > + if (entry->addr) {
> > + /* Deferred unmap */
> > + dma_dev = get_dma_dev(qp->ndev);
> > + dma_unmap_single(dma_dev, entry->addr, entry->len, DMA_TO_DEVICE);
> > + }
> > +
> > + entry->cb_data = cb;
> > + entry->buf = data;
> > + entry->len = len;
> > + entry->flags = flags;
> > + entry->errors = 0;
> > + entry->addr = 0;
> > +
> > + WARN_ON_ONCE(!ntb_qp_edma_enabled(qp));
> > +
> > + if (ntb_qp_edma_is_ep(qp))
> > + return ntb_transport_edma_ep_tx_enqueue(qp, entry);
> > + else
> > + return ntb_transport_edma_rc_tx_enqueue(qp, entry);
> > +}
> > +
> > +static int ntb_transport_edma_ep_rx_enqueue(struct ntb_transport_qp *qp,
> > + struct ntb_queue_entry *entry)
> > +{
> > + struct device *dma_dev = get_dma_dev(qp->ndev);
> > + struct ntb_edma_desc *in, __iomem *out;
> > + unsigned int len = entry->len;
> > + void *data = entry->buf;
> > + dma_addr_t ep_dst;
> > + u32 idx;
> > + int rc;
> > +
> > + ep_dst = dma_map_single(dma_dev, data, len, DMA_FROM_DEVICE);
> > + rc = dma_mapping_error(dma_dev, ep_dst);
> > + if (rc)
> > + return rc;
> > +
> > + scoped_guard(spinlock_bh, &qp->ep_rx_lock) {
> > + if (ntb_edma_ring_full(READ_ONCE(qp->rd_prod),
> > + READ_ONCE(qp->rd_cons))) {
> > + rc = -ENOSPC;
> > + goto out_unmap;
> > + }
> > +
> > + idx = ntb_edma_ring_idx(qp->rd_prod);
> > + in = NTB_DESC_RD_EP_I(qp, idx);
> > + out = NTB_DESC_RD_EP_O(qp, idx);
> > +
> > + iowrite32(len, &out->len);
> > + iowrite64(ep_dst, &out->addr);
> > +
> > + WARN_ON(in->flags & DESC_DONE_FLAG);
> > + in->data = (uintptr_t)entry;
> > + entry->addr = ep_dst;
> > +
> > + /* Ensure len/addr are visible before the head update */
> > + dma_wmb();
> > +
> > + WRITE_ONCE(qp->rd_prod, qp->rd_prod + 1);
> > + iowrite32(qp->rd_prod, NTB_HEAD_RD_EP_O(qp));
> > + }
> > + return 0;
> > +out_unmap:
> > + dma_unmap_single(dma_dev, ep_dst, len, DMA_FROM_DEVICE);
> > + return rc;
> > +}
> > +
> > +static int ntb_transport_edma_rx_enqueue(struct ntb_transport_qp *qp,
> > + struct ntb_queue_entry *entry)
> > +{
> > + int rc;
> > +
> > + /* The behaviour is the same as the default backend for RC side */
> > + if (ntb_qp_edma_is_ep(qp)) {
> > + rc = ntb_transport_edma_ep_rx_enqueue(qp, entry);
> > + if (rc) {
> > + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> > + &qp->rx_free_q);
> > + return rc;
> > + }
> > + }
> > +
> > + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q);
> > +
> > + if (qp->active)
> > + tasklet_schedule(&qp->rxc_db_work);
> > +
> > + return 0;
> > +}
> > +
> > +static void ntb_transport_edma_rx_poll(struct ntb_transport_qp *qp)
> > +{
> > + struct ntb_transport_ctx *nt = qp->transport;
> > +
> > + if (ntb_qp_edma_is_rc(qp))
> > + ntb_transport_edma_rc_poll(qp);
> > + else if (ntb_qp_edma_is_ep(qp)) {
> > + /*
> > + * Make sure we poll the rings even if an eDMA interrupt is
> > + * cleared on the RC side earlier.
> > + */
> > + queue_work(nt->wq, &qp->read_work);
> > + queue_work(nt->wq, &qp->write_work);
> > + } else
> > + /* Unreachable */
> > + WARN_ON_ONCE(1);
> > +}
> > +
> > +static void ntb_transport_edma_read_work(struct work_struct *work)
> > +{
> > + struct ntb_transport_qp *qp = container_of(
> > + work, struct ntb_transport_qp, read_work);
> > +
> > + if (ntb_qp_edma_is_rc(qp))
> > + ntb_transport_edma_rc_read_complete_work(work);
> > + else if (ntb_qp_edma_is_ep(qp))
> > + ntb_transport_edma_ep_read_work(work);
> > + else
> > + /* Unreachable */
> > + WARN_ON_ONCE(1);
> > +}
> > +
> > +static void ntb_transport_edma_write_work(struct work_struct *work)
> > +{
> > + struct ntb_transport_qp *qp = container_of(
> > + work, struct ntb_transport_qp, write_work);
> > +
> > + if (ntb_qp_edma_is_rc(qp))
> > + ntb_transport_edma_rc_write_complete_work(work);
> > + else if (ntb_qp_edma_is_ep(qp))
> > + ntb_transport_edma_ep_write_work(work);
> > + else
> > + /* Unreachable */
> > + WARN_ON_ONCE(1);
> > +}
> > +
> > +static void ntb_transport_edma_init_queue(struct ntb_transport_ctx *nt,
> > + unsigned int qp_num)
> > +{
> > + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
> > +
> > + qp->wr_cons = 0;
> > + qp->rd_cons = 0;
> > + qp->wr_prod = 0;
> > + qp->rd_prod = 0;
> > + qp->wr_issue = 0;
> > + qp->rd_issue = 0;
> > +
> > + INIT_WORK(&qp->db_work, ntb_transport_edma_db_work);
> > + INIT_WORK(&qp->read_work, ntb_transport_edma_read_work);
> > + INIT_WORK(&qp->write_work, ntb_transport_edma_write_work);
> > +}
> > +
> > +static void ntb_transport_edma_create_queue(struct ntb_transport_ctx *nt,
> > + struct ntb_transport_qp *qp)
> > +{
> > + spin_lock_init(&qp->ep_tx_lock);
> > + spin_lock_init(&qp->ep_rx_lock);
> > + spin_lock_init(&qp->rc_lock);
> > +}
> > +
> > +static const struct ntb_transport_backend_ops edma_backend_ops = {
> > + .setup_qp_mw = ntb_transport_edma_setup_qp_mw,
> > + .tx_free_entry = ntb_transport_edma_tx_free_entry,
> > + .tx_enqueue = ntb_transport_edma_tx_enqueue,
> > + .rx_enqueue = ntb_transport_edma_rx_enqueue,
> > + .rx_poll = ntb_transport_edma_rx_poll,
> > + .debugfs_stats_show = ntb_transport_edma_debugfs_stats_show,
> > +};
> > +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> > +
> > /**
> > * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
> > * @qp: NTB transport layer queue to be enabled
> > --
> > 2.48.1
> >
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-12-02 6:43 ` Koichiro Den
@ 2025-12-02 15:42 ` Frank Li
2025-12-03 8:53 ` Koichiro Den
0 siblings, 1 reply; 19+ messages in thread
From: Frank Li @ 2025-12-02 15:42 UTC (permalink / raw)
To: Koichiro Den
Cc: ntb, linux-pci, dmaengine, linux-kernel, mani, kwilczynski,
kishon, bhelgaas, corbet, vkoul, jdmason, dave.jiang, allenbh,
Basavaraj.Natikar, Shyam-sundar.S-k, kurt.schwemmer, logang,
jingoohan1, lpieralisi, robh, jbrunet, fancer.lancer, arnd,
pstanner, elfring
On Tue, Dec 02, 2025 at 03:43:10PM +0900, Koichiro Den wrote:
> On Mon, Dec 01, 2025 at 04:41:05PM -0500, Frank Li wrote:
> > On Sun, Nov 30, 2025 at 01:03:58AM +0900, Koichiro Den wrote:
> > > Add a new transport backend that uses a remote DesignWare eDMA engine
> > > located on the NTB endpoint to move data between host and endpoint.
> > >
...
> > > +#include "ntb_edma.h"
> > > +
> > > +/*
> > > + * The interrupt register offsets below are taken from the DesignWare
> > > + * eDMA "unrolled" register map (EDMA_MF_EDMA_UNROLL). The remote eDMA
> > > + * backend currently only supports this layout.
> > > + */
> > > +#define DMA_WRITE_INT_STATUS_OFF 0x4c
> > > +#define DMA_WRITE_INT_MASK_OFF 0x54
> > > +#define DMA_WRITE_INT_CLEAR_OFF 0x58
> > > +#define DMA_READ_INT_STATUS_OFF 0xa0
> > > +#define DMA_READ_INT_MASK_OFF 0xa8
> > > +#define DMA_READ_INT_CLEAR_OFF 0xac
> >
> > Not sure why need access EDMA register because EMDA driver already export
> > as dmaengine driver.
>
> These are intended for EP use. In my current design I intentionally don't
> use the standard dw-edma dmaengine driver on the EP side.
why not?
>
> >
> > > +
> > > +#define NTB_EDMA_NOTIFY_MAX_QP 64
> > > +
...
> > > +
> > > + virq = irq_create_fwspec_mapping(&fwspec);
> > > + of_node_put(parent);
> > > + return (virq > 0) ? virq : -EINVAL;
> > > +}
> > > +
> > > +static irqreturn_t ntb_edma_isr(int irq, void *data)
> > > +{
> >
> > Not sue why dw_edma_interrupt_write/read() does work for your case. Suppose
> > just register callback for dmeengine.
>
> If we ran dw_edma_probe() on both the EP and RC sides and let the dmaengine
> callbacks handle int_status/int_clear, I think we could hit races. One side
> might clear a status bit before the other side has a chance to see it and
> invoke its callback. Please correct me if I'm missing something here.
You should use difference channel?
>
> To avoid that, in my current implementation, the RC side handles the
> status/int_clear registers in the usual way, and the EP side only tries to
> suppress needless edma_int as much as possible.
>
> That said, I'm now wondering if it would be better to set LIE=0/RIE=1 for
> the DMA transfer channels and LIE=1/RIE=0 for the notification channel.
> That would require some changes on dw-edma core.
If dw-edma work as remote DMA, which should enable RIE. like
dw-edma-pcie.c, but not one actually use it recently.
Use EDMA as doorbell should be new case and I think it is quite useful.
> >
> > > + struct ntb_edma_interrupt *v = data;
> > > + u32 mask = BIT(EDMA_RD_CH_NUM);
> > > + u32 i, val;
> > > +
...
> > > + ret = dw_edma_probe(chip);
> >
> > I think dw_edma_probe() should be in ntb_hw_epf.c, which provide DMA
> > dma engine support.
> >
> > EP side, suppose default dwc controller driver already setup edma engine,
> > so use correct filter function, you should get dma chan.
>
> I intentionally hid edma for EP side in .dts patch in [RFC PATCH v2 26/27]
> so that RC side only manages eDMA remotely and avoids the potential race
> condition I mentioned above.
Improve eDMA core to suppport some dma channel work at local, some for
remote.
Frank
>
> Thanks for reviewing,
> Koichiro
>
> >
> > Frank
> >
> > > + if (ret) {
> > > + dev_err(&ndev->dev, "dw_edma_probe failed: %d\n", ret);
> > > + return ret;
> > > + }
> > > +
> > > + return 0;
> > > +}
> > > +
...
> > > +{
> > > + spin_lock_init(&qp->ep_tx_lock);
> > > + spin_lock_init(&qp->ep_rx_lock);
> > > + spin_lock_init(&qp->rc_lock);
> > > +}
> > > +
> > > +static const struct ntb_transport_backend_ops edma_backend_ops = {
> > > + .setup_qp_mw = ntb_transport_edma_setup_qp_mw,
> > > + .tx_free_entry = ntb_transport_edma_tx_free_entry,
> > > + .tx_enqueue = ntb_transport_edma_tx_enqueue,
> > > + .rx_enqueue = ntb_transport_edma_rx_enqueue,
> > > + .rx_poll = ntb_transport_edma_rx_poll,
> > > + .debugfs_stats_show = ntb_transport_edma_debugfs_stats_show,
> > > +};
> > > +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> > > +
> > > /**
> > > * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
> > > * @qp: NTB transport layer queue to be enabled
> > > --
> > > 2.48.1
> > >
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-12-02 15:42 ` Frank Li
@ 2025-12-03 8:53 ` Koichiro Den
2025-12-03 16:14 ` Frank Li
0 siblings, 1 reply; 19+ messages in thread
From: Koichiro Den @ 2025-12-03 8:53 UTC (permalink / raw)
To: Frank Li
Cc: ntb, linux-pci, dmaengine, linux-kernel, mani, kwilczynski,
kishon, bhelgaas, corbet, vkoul, jdmason, dave.jiang, allenbh,
Basavaraj.Natikar, Shyam-sundar.S-k, kurt.schwemmer, logang,
jingoohan1, lpieralisi, robh, jbrunet, fancer.lancer, arnd,
pstanner, elfring
On Tue, Dec 02, 2025 at 10:42:29AM -0500, Frank Li wrote:
> On Tue, Dec 02, 2025 at 03:43:10PM +0900, Koichiro Den wrote:
> > On Mon, Dec 01, 2025 at 04:41:05PM -0500, Frank Li wrote:
> > > On Sun, Nov 30, 2025 at 01:03:58AM +0900, Koichiro Den wrote:
> > > > Add a new transport backend that uses a remote DesignWare eDMA engine
> > > > located on the NTB endpoint to move data between host and endpoint.
> > > >
> ...
> > > > +#include "ntb_edma.h"
> > > > +
> > > > +/*
> > > > + * The interrupt register offsets below are taken from the DesignWare
> > > > + * eDMA "unrolled" register map (EDMA_MF_EDMA_UNROLL). The remote eDMA
> > > > + * backend currently only supports this layout.
> > > > + */
> > > > +#define DMA_WRITE_INT_STATUS_OFF 0x4c
> > > > +#define DMA_WRITE_INT_MASK_OFF 0x54
> > > > +#define DMA_WRITE_INT_CLEAR_OFF 0x58
> > > > +#define DMA_READ_INT_STATUS_OFF 0xa0
> > > > +#define DMA_READ_INT_MASK_OFF 0xa8
> > > > +#define DMA_READ_INT_CLEAR_OFF 0xac
> > >
> > > Not sure why need access EDMA register because EMDA driver already export
> > > as dmaengine driver.
> >
> > These are intended for EP use. In my current design I intentionally don't
> > use the standard dw-edma dmaengine driver on the EP side.
>
> why not?
Conceptually I agree that using the standard dw-edma driver on both sides
would be attractive for future extensibility and maintainability. However,
there are a couple of concerns for me, some of which might be alleviated by
your suggestion below, and some which are more generic safety concerns that
I tried to outline in my replies to your other comments.
>
> >
> > >
> > > > +
> > > > +#define NTB_EDMA_NOTIFY_MAX_QP 64
> > > > +
> ...
> > > > +
> > > > + virq = irq_create_fwspec_mapping(&fwspec);
> > > > + of_node_put(parent);
> > > > + return (virq > 0) ? virq : -EINVAL;
> > > > +}
> > > > +
> > > > +static irqreturn_t ntb_edma_isr(int irq, void *data)
> > > > +{
> > >
> > > Not sue why dw_edma_interrupt_write/read() does work for your case. Suppose
> > > just register callback for dmeengine.
> >
> > If we ran dw_edma_probe() on both the EP and RC sides and let the dmaengine
> > callbacks handle int_status/int_clear, I think we could hit races. One side
> > might clear a status bit before the other side has a chance to see it and
> > invoke its callback. Please correct me if I'm missing something here.
>
> You should use difference channel?
Do you mean something like this:
- on EP side, dw_edma_probe() only set up a dedicated channel for notification,
- on RC side, do not set up that particular channel via dw_edma_channel_setup(),
but do other remaining channels for DMA transfers.
Also, is it generically safe to have dw_edma_probe() executed from both ends on
the same eDMA instance, as long as the channels are carefully partitioned
between them?
>
> >
> > To avoid that, in my current implementation, the RC side handles the
> > status/int_clear registers in the usual way, and the EP side only tries to
> > suppress needless edma_int as much as possible.
> >
> > That said, I'm now wondering if it would be better to set LIE=0/RIE=1 for
> > the DMA transfer channels and LIE=1/RIE=0 for the notification channel.
> > That would require some changes on dw-edma core.
>
> If dw-edma work as remote DMA, which should enable RIE. like
> dw-edma-pcie.c, but not one actually use it recently.
>
> Use EDMA as doorbell should be new case and I think it is quite useful.
>
> > >
> > > > + struct ntb_edma_interrupt *v = data;
> > > > + u32 mask = BIT(EDMA_RD_CH_NUM);
> > > > + u32 i, val;
> > > > +
> ...
> > > > + ret = dw_edma_probe(chip);
> > >
> > > I think dw_edma_probe() should be in ntb_hw_epf.c, which provide DMA
> > > dma engine support.
> > >
> > > EP side, suppose default dwc controller driver already setup edma engine,
> > > so use correct filter function, you should get dma chan.
> >
> > I intentionally hid edma for EP side in .dts patch in [RFC PATCH v2 26/27]
> > so that RC side only manages eDMA remotely and avoids the potential race
> > condition I mentioned above.
>
> Improve eDMA core to suppport some dma channel work at local, some for
> remote.
Right, Firstly I experimented a bit more with different LIE/RIE settings and
ended up with the following observations:
* LIE=0/RIE=1 does not seem to work at the hardware level. When I tried this for
DMA transfer channels, the RC side never received any interrupt. The databook
(5.40a, 8.2.2 "Interrupts and Error Handling") has a hint that says
"If you want a remote interrupt and not a local interrupt then: Set LIE and
RIE [...]", so I think this behaviour is expected.
* LIE=1/RIE=0 does work at the hardware level, but is problematic for my current
design, where the RC issues the DMA transfer for the notification via
ntb_edma_notify_peer(). With RIE=0, the RC never calls
dw_edma_core_handle_int() for that channel, which means that internal state
such as dw_edma_chan.status is never managed correctly.
>
> Frank
> >
> > Thanks for reviewing,
> > Koichiro
> >
> > >
> > > Frank
> > >
> > > > + if (ret) {
> > > > + dev_err(&ndev->dev, "dw_edma_probe failed: %d\n", ret);
> > > > + return ret;
> > > > + }
> > > > +
> > > > + return 0;
> > > > +}
> > > > +
> ...
>
> > > > +{
> > > > + spin_lock_init(&qp->ep_tx_lock);
> > > > + spin_lock_init(&qp->ep_rx_lock);
> > > > + spin_lock_init(&qp->rc_lock);
> > > > +}
> > > > +
> > > > +static const struct ntb_transport_backend_ops edma_backend_ops = {
> > > > + .setup_qp_mw = ntb_transport_edma_setup_qp_mw,
> > > > + .tx_free_entry = ntb_transport_edma_tx_free_entry,
> > > > + .tx_enqueue = ntb_transport_edma_tx_enqueue,
> > > > + .rx_enqueue = ntb_transport_edma_rx_enqueue,
> > > > + .rx_poll = ntb_transport_edma_rx_poll,
> > > > + .debugfs_stats_show = ntb_transport_edma_debugfs_stats_show,
> > > > +};
> > > > +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> > > > +
> > > > /**
> > > > * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
> > > > * @qp: NTB transport layer queue to be enabled
> > > > --
> > > > 2.48.1
> > > >
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-12-03 8:53 ` Koichiro Den
@ 2025-12-03 16:14 ` Frank Li
2025-12-04 15:42 ` Koichiro Den
0 siblings, 1 reply; 19+ messages in thread
From: Frank Li @ 2025-12-03 16:14 UTC (permalink / raw)
To: Koichiro Den
Cc: ntb, linux-pci, dmaengine, linux-kernel, mani, kwilczynski,
kishon, bhelgaas, corbet, vkoul, jdmason, dave.jiang, allenbh,
Basavaraj.Natikar, Shyam-sundar.S-k, kurt.schwemmer, logang,
jingoohan1, lpieralisi, robh, jbrunet, fancer.lancer, arnd,
pstanner, elfring
On Wed, Dec 03, 2025 at 05:53:03PM +0900, Koichiro Den wrote:
> On Tue, Dec 02, 2025 at 10:42:29AM -0500, Frank Li wrote:
> > On Tue, Dec 02, 2025 at 03:43:10PM +0900, Koichiro Den wrote:
> > > On Mon, Dec 01, 2025 at 04:41:05PM -0500, Frank Li wrote:
> > > > On Sun, Nov 30, 2025 at 01:03:58AM +0900, Koichiro Den wrote:
> > > > > Add a new transport backend that uses a remote DesignWare eDMA engine
> > > > > located on the NTB endpoint to move data between host and endpoint.
> > > > >
> > ...
> > > > > +#include "ntb_edma.h"
> > > > > +
> > > > > +/*
> > > > > + * The interrupt register offsets below are taken from the DesignWare
> > > > > + * eDMA "unrolled" register map (EDMA_MF_EDMA_UNROLL). The remote eDMA
> > > > > + * backend currently only supports this layout.
> > > > > + */
> > > > > +#define DMA_WRITE_INT_STATUS_OFF 0x4c
> > > > > +#define DMA_WRITE_INT_MASK_OFF 0x54
> > > > > +#define DMA_WRITE_INT_CLEAR_OFF 0x58
> > > > > +#define DMA_READ_INT_STATUS_OFF 0xa0
> > > > > +#define DMA_READ_INT_MASK_OFF 0xa8
> > > > > +#define DMA_READ_INT_CLEAR_OFF 0xac
> > > >
> > > > Not sure why need access EDMA register because EMDA driver already export
> > > > as dmaengine driver.
> > >
> > > These are intended for EP use. In my current design I intentionally don't
> > > use the standard dw-edma dmaengine driver on the EP side.
> >
> > why not?
>
> Conceptually I agree that using the standard dw-edma driver on both sides
> would be attractive for future extensibility and maintainability. However,
> there are a couple of concerns for me, some of which might be alleviated by
> your suggestion below, and some which are more generic safety concerns that
> I tried to outline in my replies to your other comments.
>
> >
> > >
> > > >
> > > > > +
> > > > > +#define NTB_EDMA_NOTIFY_MAX_QP 64
> > > > > +
> > ...
> > > > > +
> > > > > + virq = irq_create_fwspec_mapping(&fwspec);
> > > > > + of_node_put(parent);
> > > > > + return (virq > 0) ? virq : -EINVAL;
> > > > > +}
> > > > > +
> > > > > +static irqreturn_t ntb_edma_isr(int irq, void *data)
> > > > > +{
> > > >
> > > > Not sue why dw_edma_interrupt_write/read() does work for your case. Suppose
> > > > just register callback for dmeengine.
> > >
> > > If we ran dw_edma_probe() on both the EP and RC sides and let the dmaengine
> > > callbacks handle int_status/int_clear, I think we could hit races. One side
> > > might clear a status bit before the other side has a chance to see it and
> > > invoke its callback. Please correct me if I'm missing something here.
> >
> > You should use difference channel?
>
> Do you mean something like this:
> - on EP side, dw_edma_probe() only set up a dedicated channel for notification,
> - on RC side, do not set up that particular channel via dw_edma_channel_setup(),
> but do other remaining channels for DMA transfers.
Yes, it may be simple overall. Of course this will waste a channel.
>
> Also, is it generically safe to have dw_edma_probe() executed from both ends on
> the same eDMA instance, as long as the channels are carefully partitioned
> between them?
Channel register MMIO space is sperated. Some channel register shared
into one 32bit register.
But the critical one, interrupt status is w1c. So only write BIT(channel)
is safe.
Need careful handle irq enable/disable.
Or you can defer all actual DMA transfer to EP side, you can append
MSI write at last item of link to notify RC side about DMA done. (actually
RIE should do the same thing)
>
> >
> > >
> > > To avoid that, in my current implementation, the RC side handles the
> > > status/int_clear registers in the usual way, and the EP side only tries to
> > > suppress needless edma_int as much as possible.
> > >
> > > That said, I'm now wondering if it would be better to set LIE=0/RIE=1 for
> > > the DMA transfer channels and LIE=1/RIE=0 for the notification channel.
> > > That would require some changes on dw-edma core.
> >
> > If dw-edma work as remote DMA, which should enable RIE. like
> > dw-edma-pcie.c, but not one actually use it recently.
> >
> > Use EDMA as doorbell should be new case and I think it is quite useful.
> >
> > > >
> > > > > + struct ntb_edma_interrupt *v = data;
> > > > > + u32 mask = BIT(EDMA_RD_CH_NUM);
> > > > > + u32 i, val;
> > > > > +
> > ...
> > > > > + ret = dw_edma_probe(chip);
> > > >
> > > > I think dw_edma_probe() should be in ntb_hw_epf.c, which provide DMA
> > > > dma engine support.
> > > >
> > > > EP side, suppose default dwc controller driver already setup edma engine,
> > > > so use correct filter function, you should get dma chan.
> > >
> > > I intentionally hid edma for EP side in .dts patch in [RFC PATCH v2 26/27]
> > > so that RC side only manages eDMA remotely and avoids the potential race
> > > condition I mentioned above.
> >
> > Improve eDMA core to suppport some dma channel work at local, some for
> > remote.
>
> Right, Firstly I experimented a bit more with different LIE/RIE settings and
> ended up with the following observations:
>
> * LIE=0/RIE=1 does not seem to work at the hardware level. When I tried this for
> DMA transfer channels, the RC side never received any interrupt. The databook
> (5.40a, 8.2.2 "Interrupts and Error Handling") has a hint that says
> "If you want a remote interrupt and not a local interrupt then: Set LIE and
> RIE [...]", so I think this behaviour is expected.
Actually, you can append MSI write at last one of DMA descriptor link. So
it will not depend on eDMA's IRQ at all.
> * LIE=1/RIE=0 does work at the hardware level, but is problematic for my current
> design, where the RC issues the DMA transfer for the notification via
> ntb_edma_notify_peer(). With RIE=0, the RC never calls
> dw_edma_core_handle_int() for that channel, which means that internal state
> such as dw_edma_chan.status is never managed correctly.
If you append on MSI write at DMA link, you needn't check status register,
just check current LL pos to know which descrptor already done.
Or you also enable LIE and disable related IRQ line(without register
irq handler), so Local IRQ will be ignore by GIC, you can safe handle at
RC side.
Frank
>
> >
> > Frank
> > >
> > > Thanks for reviewing,
> > > Koichiro
> > >
> > > >
> > > > Frank
> > > >
> > > > > + if (ret) {
> > > > > + dev_err(&ndev->dev, "dw_edma_probe failed: %d\n", ret);
> > > > > + return ret;
> > > > > + }
> > > > > +
> > > > > + return 0;
> > > > > +}
> > > > > +
> > ...
> >
> > > > > +{
> > > > > + spin_lock_init(&qp->ep_tx_lock);
> > > > > + spin_lock_init(&qp->ep_rx_lock);
> > > > > + spin_lock_init(&qp->rc_lock);
> > > > > +}
> > > > > +
> > > > > +static const struct ntb_transport_backend_ops edma_backend_ops = {
> > > > > + .setup_qp_mw = ntb_transport_edma_setup_qp_mw,
> > > > > + .tx_free_entry = ntb_transport_edma_tx_free_entry,
> > > > > + .tx_enqueue = ntb_transport_edma_tx_enqueue,
> > > > > + .rx_enqueue = ntb_transport_edma_rx_enqueue,
> > > > > + .rx_poll = ntb_transport_edma_rx_poll,
> > > > > + .debugfs_stats_show = ntb_transport_edma_debugfs_stats_show,
> > > > > +};
> > > > > +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> > > > > +
> > > > > /**
> > > > > * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
> > > > > * @qp: NTB transport layer queue to be enabled
> > > > > --
> > > > > 2.48.1
> > > > >
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-12-03 16:14 ` Frank Li
@ 2025-12-04 15:42 ` Koichiro Den
2025-12-04 20:16 ` Frank Li
0 siblings, 1 reply; 19+ messages in thread
From: Koichiro Den @ 2025-12-04 15:42 UTC (permalink / raw)
To: Frank Li
Cc: ntb, linux-pci, dmaengine, linux-kernel, mani, kwilczynski,
kishon, bhelgaas, corbet, vkoul, jdmason, dave.jiang, allenbh,
Basavaraj.Natikar, Shyam-sundar.S-k, kurt.schwemmer, logang,
jingoohan1, lpieralisi, robh, jbrunet, fancer.lancer, arnd,
pstanner, elfring
On Wed, Dec 03, 2025 at 11:14:43AM -0500, Frank Li wrote:
> On Wed, Dec 03, 2025 at 05:53:03PM +0900, Koichiro Den wrote:
> > On Tue, Dec 02, 2025 at 10:42:29AM -0500, Frank Li wrote:
> > > On Tue, Dec 02, 2025 at 03:43:10PM +0900, Koichiro Den wrote:
> > > > On Mon, Dec 01, 2025 at 04:41:05PM -0500, Frank Li wrote:
> > > > > On Sun, Nov 30, 2025 at 01:03:58AM +0900, Koichiro Den wrote:
> > > > > > Add a new transport backend that uses a remote DesignWare eDMA engine
> > > > > > located on the NTB endpoint to move data between host and endpoint.
> > > > > >
> > > ...
> > > > > > +#include "ntb_edma.h"
> > > > > > +
> > > > > > +/*
> > > > > > + * The interrupt register offsets below are taken from the DesignWare
> > > > > > + * eDMA "unrolled" register map (EDMA_MF_EDMA_UNROLL). The remote eDMA
> > > > > > + * backend currently only supports this layout.
> > > > > > + */
> > > > > > +#define DMA_WRITE_INT_STATUS_OFF 0x4c
> > > > > > +#define DMA_WRITE_INT_MASK_OFF 0x54
> > > > > > +#define DMA_WRITE_INT_CLEAR_OFF 0x58
> > > > > > +#define DMA_READ_INT_STATUS_OFF 0xa0
> > > > > > +#define DMA_READ_INT_MASK_OFF 0xa8
> > > > > > +#define DMA_READ_INT_CLEAR_OFF 0xac
> > > > >
> > > > > Not sure why need access EDMA register because EMDA driver already export
> > > > > as dmaengine driver.
> > > >
> > > > These are intended for EP use. In my current design I intentionally don't
> > > > use the standard dw-edma dmaengine driver on the EP side.
> > >
> > > why not?
> >
> > Conceptually I agree that using the standard dw-edma driver on both sides
> > would be attractive for future extensibility and maintainability. However,
> > there are a couple of concerns for me, some of which might be alleviated by
> > your suggestion below, and some which are more generic safety concerns that
> > I tried to outline in my replies to your other comments.
> >
> > >
> > > >
> > > > >
> > > > > > +
> > > > > > +#define NTB_EDMA_NOTIFY_MAX_QP 64
> > > > > > +
> > > ...
> > > > > > +
> > > > > > + virq = irq_create_fwspec_mapping(&fwspec);
> > > > > > + of_node_put(parent);
> > > > > > + return (virq > 0) ? virq : -EINVAL;
> > > > > > +}
> > > > > > +
> > > > > > +static irqreturn_t ntb_edma_isr(int irq, void *data)
> > > > > > +{
> > > > >
> > > > > Not sue why dw_edma_interrupt_write/read() does work for your case. Suppose
> > > > > just register callback for dmeengine.
> > > >
> > > > If we ran dw_edma_probe() on both the EP and RC sides and let the dmaengine
> > > > callbacks handle int_status/int_clear, I think we could hit races. One side
> > > > might clear a status bit before the other side has a chance to see it and
> > > > invoke its callback. Please correct me if I'm missing something here.
> > >
> > > You should use difference channel?
> >
> > Do you mean something like this:
> > - on EP side, dw_edma_probe() only set up a dedicated channel for notification,
> > - on RC side, do not set up that particular channel via dw_edma_channel_setup(),
> > but do other remaining channels for DMA transfers.
>
> Yes, it may be simple overall. Of course this will waste a channel.
So, on the EP side I see two possible approaches:
(a) Hide "dma" [1] as in [RFC PATCH v2 26/27] and call dw_edma_probe() with
hand-crafted settings (chip->ll_rd_cnt = 1, chip->ll_wr_cnt = 0).
(b) Or, teach this special-purpose policy (i.e. configuring only a single
notification channel) to the SoC glue driver's dw_pcie_ep_init_registers(),
for example via Kconfig. I don't think DT is a good place to describe
such a policy.
There is also another option, which do not necessarily run dw_edma_probe()
by ourselves:
(c) Leave the default initialization by the SoC glue as-is, and override the
per-channel role via some new dw-edma interface, with the guarantee
that all channels except the notification channel remain unused on its
side afterwards. In this model, the EP side builds the LL locations
for data transfers and the RC configures all channels, but it sets up
the notification channel in a special manner.
[1] https://github.com/jonmason/ntb/blob/68113d260674/Documentation/devicetree/bindings/pci/snps%2Cdw-pcie-ep.yaml#L83
>
> >
> > Also, is it generically safe to have dw_edma_probe() executed from both ends on
> > the same eDMA instance, as long as the channels are carefully partitioned
> > between them?
>
> Channel register MMIO space is sperated. Some channel register shared
> into one 32bit register.
>
> But the critical one, interrupt status is w1c. So only write BIT(channel)
> is safe.
>
> Need careful handle irq enable/disable.
Yeah, I agree it is unavoidable in this model.
>
> Or you can defer all actual DMA transfer to EP side, you can append
> MSI write at last item of link to notify RC side about DMA done. (actually
> RIE should do the same thing)
>
> >
> > >
> > > >
> > > > To avoid that, in my current implementation, the RC side handles the
> > > > status/int_clear registers in the usual way, and the EP side only tries to
> > > > suppress needless edma_int as much as possible.
> > > >
> > > > That said, I'm now wondering if it would be better to set LIE=0/RIE=1 for
> > > > the DMA transfer channels and LIE=1/RIE=0 for the notification channel.
> > > > That would require some changes on dw-edma core.
> > >
> > > If dw-edma work as remote DMA, which should enable RIE. like
> > > dw-edma-pcie.c, but not one actually use it recently.
> > >
> > > Use EDMA as doorbell should be new case and I think it is quite useful.
> > >
> > > > >
> > > > > > + struct ntb_edma_interrupt *v = data;
> > > > > > + u32 mask = BIT(EDMA_RD_CH_NUM);
> > > > > > + u32 i, val;
> > > > > > +
> > > ...
> > > > > > + ret = dw_edma_probe(chip);
> > > > >
> > > > > I think dw_edma_probe() should be in ntb_hw_epf.c, which provide DMA
> > > > > dma engine support.
> > > > >
> > > > > EP side, suppose default dwc controller driver already setup edma engine,
> > > > > so use correct filter function, you should get dma chan.
> > > >
> > > > I intentionally hid edma for EP side in .dts patch in [RFC PATCH v2 26/27]
> > > > so that RC side only manages eDMA remotely and avoids the potential race
> > > > condition I mentioned above.
> > >
> > > Improve eDMA core to suppport some dma channel work at local, some for
> > > remote.
> >
> > Right, Firstly I experimented a bit more with different LIE/RIE settings and
> > ended up with the following observations:
> >
> > * LIE=0/RIE=1 does not seem to work at the hardware level. When I tried this for
> > DMA transfer channels, the RC side never received any interrupt. The databook
> > (5.40a, 8.2.2 "Interrupts and Error Handling") has a hint that says
> > "If you want a remote interrupt and not a local interrupt then: Set LIE and
> > RIE [...]", so I think this behaviour is expected.
>
> Actually, you can append MSI write at last one of DMA descriptor link. So
> it will not depend on eDMA's IRQ at all.
For RC->EP interrupts on R-Car S4 in EP mode, using ITS_TRANSLATER as the
IB iATU target did not appear to work in practice. Indeed that was the
motivation for the RFC v1 series [2]. I have not tried using ITS_TRANSLATER
as the eDMA read transfer DAR.
But in any case, simply masking the local interrupt is sufficient here. I
mainly wanted to point out that my naive idea of LIE=0/RIE=1 is not
implementable with this hardware. This whole LIE/RIE topic is a bit
off-track, sorry for the noise.
[2] For the record, RFC v2 is conceptually orthogonal and introduces a
broader concept ie. remote eDMA model, but I reused many of the
preparatory commits from v1, which is why this is RFC v2 rather than a
separate series.
>
> > * LIE=1/RIE=0 does work at the hardware level, but is problematic for my current
> > design, where the RC issues the DMA transfer for the notification via
> > ntb_edma_notify_peer(). With RIE=0, the RC never calls
> > dw_edma_core_handle_int() for that channel, which means that internal state
> > such as dw_edma_chan.status is never managed correctly.
>
> If you append on MSI write at DMA link, you needn't check status register,
> just check current LL pos to know which descrptor already done.
>
> Or you also enable LIE and disable related IRQ line(without register
> irq handler), so Local IRQ will be ignore by GIC, you can safe handle at
> RC side.
What I was worried about here is that, with RIE=0 the current dw-edma
handling of struct dw_edma_chan::status field (not status register) would
not run for that channel, which could affect subsequent tx submissions. But
your suggestion also makes sense, thank you.
--8<--
So anyway the key point seems that we should avoid such hard-coded register
handling in [RFC PATCH v2 20/27] and rely only on the standard dw-edma
interfaces (possibly with some extensions to the dw-edma core). From your
feedback, I feel this is the essential direction.
From that perspective, I'm leaning toward (b) (which I wrote above in a
reply comment) with a Kconfig guard, i.e. in dw_pcie_ep_init_registers(),
if IS_ENABLED(CONFIG_DW_REMOTE_EDMA) we only configure the notification
channel. In practice, a DT-based variant of (b) (for example a new property
such as "dma-notification-channel = <N>;" and making
dw_pcie_ep_init_registers() honour it) would be very handy for users, but I
suspect putting this kind of policy into DT is not acceptable.
Assuming careful handling, (c) might actually be the simplest approach. I
may need to add a small hook for the notification channel in
dw_edma_done_interrupt(), via a new API such as
dw_edma_chan_register_notify().
Thank you for your time and review,
Koichiro
>
> Frank
> >
> > >
> > > Frank
> > > >
> > > > Thanks for reviewing,
> > > > Koichiro
> > > >
> > > > >
> > > > > Frank
> > > > >
> > > > > > + if (ret) {
> > > > > > + dev_err(&ndev->dev, "dw_edma_probe failed: %d\n", ret);
> > > > > > + return ret;
> > > > > > + }
> > > > > > +
> > > > > > + return 0;
> > > > > > +}
> > > > > > +
> > > ...
> > >
> > > > > > +{
> > > > > > + spin_lock_init(&qp->ep_tx_lock);
> > > > > > + spin_lock_init(&qp->ep_rx_lock);
> > > > > > + spin_lock_init(&qp->rc_lock);
> > > > > > +}
> > > > > > +
> > > > > > +static const struct ntb_transport_backend_ops edma_backend_ops = {
> > > > > > + .setup_qp_mw = ntb_transport_edma_setup_qp_mw,
> > > > > > + .tx_free_entry = ntb_transport_edma_tx_free_entry,
> > > > > > + .tx_enqueue = ntb_transport_edma_tx_enqueue,
> > > > > > + .rx_enqueue = ntb_transport_edma_rx_enqueue,
> > > > > > + .rx_poll = ntb_transport_edma_rx_poll,
> > > > > > + .debugfs_stats_show = ntb_transport_edma_debugfs_stats_show,
> > > > > > +};
> > > > > > +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> > > > > > +
> > > > > > /**
> > > > > > * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
> > > > > > * @qp: NTB transport layer queue to be enabled
> > > > > > --
> > > > > > 2.48.1
> > > > > >
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-12-04 15:42 ` Koichiro Den
@ 2025-12-04 20:16 ` Frank Li
2025-12-05 3:04 ` Koichiro Den
0 siblings, 1 reply; 19+ messages in thread
From: Frank Li @ 2025-12-04 20:16 UTC (permalink / raw)
To: Koichiro Den
Cc: ntb, linux-pci, dmaengine, linux-kernel, mani, kwilczynski,
kishon, bhelgaas, corbet, vkoul, jdmason, dave.jiang, allenbh,
Basavaraj.Natikar, Shyam-sundar.S-k, kurt.schwemmer, logang,
jingoohan1, lpieralisi, robh, jbrunet, fancer.lancer, arnd,
pstanner, elfring
On Fri, Dec 05, 2025 at 12:42:03AM +0900, Koichiro Den wrote:
> On Wed, Dec 03, 2025 at 11:14:43AM -0500, Frank Li wrote:
> > On Wed, Dec 03, 2025 at 05:53:03PM +0900, Koichiro Den wrote:
> > > On Tue, Dec 02, 2025 at 10:42:29AM -0500, Frank Li wrote:
> > > > On Tue, Dec 02, 2025 at 03:43:10PM +0900, Koichiro Den wrote:
> > > > > On Mon, Dec 01, 2025 at 04:41:05PM -0500, Frank Li wrote:
> > > > > > On Sun, Nov 30, 2025 at 01:03:58AM +0900, Koichiro Den wrote:
> > > > > > > Add a new transport backend that uses a remote DesignWare eDMA engine
> > > > > > > located on the NTB endpoint to move data between host and endpoint.
> > > > > > >
> > > > ...
> > > > > > > +#include "ntb_edma.h"
> > > > > > > +
> > > > > > > +/*
> > > > > > > + * The interrupt register offsets below are taken from the DesignWare
> > > > > > > + * eDMA "unrolled" register map (EDMA_MF_EDMA_UNROLL). The remote eDMA
> > > > > > > + * backend currently only supports this layout.
> > > > > > > + */
> > > > > > > +#define DMA_WRITE_INT_STATUS_OFF 0x4c
> > > > > > > +#define DMA_WRITE_INT_MASK_OFF 0x54
> > > > > > > +#define DMA_WRITE_INT_CLEAR_OFF 0x58
> > > > > > > +#define DMA_READ_INT_STATUS_OFF 0xa0
> > > > > > > +#define DMA_READ_INT_MASK_OFF 0xa8
> > > > > > > +#define DMA_READ_INT_CLEAR_OFF 0xac
> > > > > >
> > > > > > Not sure why need access EDMA register because EMDA driver already export
> > > > > > as dmaengine driver.
> > > > >
> > > > > These are intended for EP use. In my current design I intentionally don't
> > > > > use the standard dw-edma dmaengine driver on the EP side.
> > > >
> > > > why not?
> > >
> > > Conceptually I agree that using the standard dw-edma driver on both sides
> > > would be attractive for future extensibility and maintainability. However,
> > > there are a couple of concerns for me, some of which might be alleviated by
> > > your suggestion below, and some which are more generic safety concerns that
> > > I tried to outline in my replies to your other comments.
> > >
> > > >
> > > > >
> > > > > >
> > > > > > > +
> > > > > > > +#define NTB_EDMA_NOTIFY_MAX_QP 64
> > > > > > > +
> > > > ...
> > > > > > > +
> > > > > > > + virq = irq_create_fwspec_mapping(&fwspec);
> > > > > > > + of_node_put(parent);
> > > > > > > + return (virq > 0) ? virq : -EINVAL;
> > > > > > > +}
> > > > > > > +
> > > > > > > +static irqreturn_t ntb_edma_isr(int irq, void *data)
> > > > > > > +{
> > > > > >
> > > > > > Not sue why dw_edma_interrupt_write/read() does work for your case. Suppose
> > > > > > just register callback for dmeengine.
> > > > >
> > > > > If we ran dw_edma_probe() on both the EP and RC sides and let the dmaengine
> > > > > callbacks handle int_status/int_clear, I think we could hit races. One side
> > > > > might clear a status bit before the other side has a chance to see it and
> > > > > invoke its callback. Please correct me if I'm missing something here.
> > > >
> > > > You should use difference channel?
> > >
> > > Do you mean something like this:
> > > - on EP side, dw_edma_probe() only set up a dedicated channel for notification,
> > > - on RC side, do not set up that particular channel via dw_edma_channel_setup(),
> > > but do other remaining channels for DMA transfers.
> >
> > Yes, it may be simple overall. Of course this will waste a channel.
>
> So, on the EP side I see two possible approaches:
>
> (a) Hide "dma" [1] as in [RFC PATCH v2 26/27] and call dw_edma_probe() with
> hand-crafted settings (chip->ll_rd_cnt = 1, chip->ll_wr_cnt = 0).
> (b) Or, teach this special-purpose policy (i.e. configuring only a single
> notification channel) to the SoC glue driver's dw_pcie_ep_init_registers(),
> for example via Kconfig. I don't think DT is a good place to describe
> such a policy.
>
> There is also another option, which do not necessarily run dw_edma_probe()
> by ourselves:
>
> (c) Leave the default initialization by the SoC glue as-is, and override the
> per-channel role via some new dw-edma interface, with the guarantee
> that all channels except the notification channel remain unused on its
> side afterwards. In this model, the EP side builds the LL locations
> for data transfers and the RC configures all channels, but it sets up
> the notification channel in a special manner.
>
> [1] https://github.com/jonmason/ntb/blob/68113d260674/Documentation/devicetree/bindings/pci/snps%2Cdw-pcie-ep.yaml#L83
>
> >
> > >
> > > Also, is it generically safe to have dw_edma_probe() executed from both ends on
> > > the same eDMA instance, as long as the channels are carefully partitioned
> > > between them?
> >
> > Channel register MMIO space is sperated. Some channel register shared
> > into one 32bit register.
> >
> > But the critical one, interrupt status is w1c. So only write BIT(channel)
> > is safe.
> >
> > Need careful handle irq enable/disable.
>
> Yeah, I agree it is unavoidable in this model.
>
> >
> > Or you can defer all actual DMA transfer to EP side, you can append
> > MSI write at last item of link to notify RC side about DMA done. (actually
> > RIE should do the same thing)
> >
> > >
> > > >
> > > > >
> > > > > To avoid that, in my current implementation, the RC side handles the
> > > > > status/int_clear registers in the usual way, and the EP side only tries to
> > > > > suppress needless edma_int as much as possible.
> > > > >
> > > > > That said, I'm now wondering if it would be better to set LIE=0/RIE=1 for
> > > > > the DMA transfer channels and LIE=1/RIE=0 for the notification channel.
> > > > > That would require some changes on dw-edma core.
> > > >
> > > > If dw-edma work as remote DMA, which should enable RIE. like
> > > > dw-edma-pcie.c, but not one actually use it recently.
> > > >
> > > > Use EDMA as doorbell should be new case and I think it is quite useful.
> > > >
> > > > > >
> > > > > > > + struct ntb_edma_interrupt *v = data;
> > > > > > > + u32 mask = BIT(EDMA_RD_CH_NUM);
> > > > > > > + u32 i, val;
> > > > > > > +
> > > > ...
> > > > > > > + ret = dw_edma_probe(chip);
> > > > > >
> > > > > > I think dw_edma_probe() should be in ntb_hw_epf.c, which provide DMA
> > > > > > dma engine support.
> > > > > >
> > > > > > EP side, suppose default dwc controller driver already setup edma engine,
> > > > > > so use correct filter function, you should get dma chan.
> > > > >
> > > > > I intentionally hid edma for EP side in .dts patch in [RFC PATCH v2 26/27]
> > > > > so that RC side only manages eDMA remotely and avoids the potential race
> > > > > condition I mentioned above.
> > > >
> > > > Improve eDMA core to suppport some dma channel work at local, some for
> > > > remote.
> > >
> > > Right, Firstly I experimented a bit more with different LIE/RIE settings and
> > > ended up with the following observations:
> > >
> > > * LIE=0/RIE=1 does not seem to work at the hardware level. When I tried this for
> > > DMA transfer channels, the RC side never received any interrupt. The databook
> > > (5.40a, 8.2.2 "Interrupts and Error Handling") has a hint that says
> > > "If you want a remote interrupt and not a local interrupt then: Set LIE and
> > > RIE [...]", so I think this behaviour is expected.
> >
> > Actually, you can append MSI write at last one of DMA descriptor link. So
> > it will not depend on eDMA's IRQ at all.
>
> For RC->EP interrupts on R-Car S4 in EP mode, using ITS_TRANSLATER as the
> IB iATU target did not appear to work in practice. Indeed that was the
> motivation for the RFC v1 series [2]. I have not tried using ITS_TRANSLATER
> as the eDMA read transfer DAR.
>
> But in any case, simply masking the local interrupt is sufficient here. I
> mainly wanted to point out that my naive idea of LIE=0/RIE=1 is not
> implementable with this hardware. This whole LIE/RIE topic is a bit
> off-track, sorry for the noise.
>
> [2] For the record, RFC v2 is conceptually orthogonal and introduces a
> broader concept ie. remote eDMA model, but I reused many of the
> preparatory commits from v1, which is why this is RFC v2 rather than a
> separate series.
>
> >
> > > * LIE=1/RIE=0 does work at the hardware level, but is problematic for my current
> > > design, where the RC issues the DMA transfer for the notification via
> > > ntb_edma_notify_peer(). With RIE=0, the RC never calls
> > > dw_edma_core_handle_int() for that channel, which means that internal state
> > > such as dw_edma_chan.status is never managed correctly.
> >
> > If you append on MSI write at DMA link, you needn't check status register,
> > just check current LL pos to know which descrptor already done.
> >
> > Or you also enable LIE and disable related IRQ line(without register
> > irq handler), so Local IRQ will be ignore by GIC, you can safe handle at
> > RC side.
>
> What I was worried about here is that, with RIE=0 the current dw-edma
> handling of struct dw_edma_chan::status field (not status register) would
> not run for that channel, which could affect subsequent tx submissions. But
> your suggestion also makes sense, thank you.
>
> --8<--
>
> So anyway the key point seems that we should avoid such hard-coded register
> handling in [RFC PATCH v2 20/27] and rely only on the standard dw-edma
> interfaces (possibly with some extensions to the dw-edma core). From your
> feedback, I feel this is the essential direction.
>
> From that perspective, I'm leaning toward (b) (which I wrote above in a
> reply comment) with a Kconfig guard, i.e. in dw_pcie_ep_init_registers(),
> if IS_ENABLED(CONFIG_DW_REMOTE_EDMA) we only configure the notification
> channel. In practice, a DT-based variant of (b) (for example a new property
> such as "dma-notification-channel = <N>;" and making
> dw_pcie_ep_init_registers() honour it) would be very handy for users, but I
> suspect putting this kind of policy into DT is not acceptable.
>
> Assuming careful handling, (c) might actually be the simplest approach. I
> may need to add a small hook for the notification channel in
> dw_edma_done_interrupt(), via a new API such as
> dw_edma_chan_register_notify().
I reply everything here for overall design
EDMA actually can access all memory at both EP and RC side regardless PCI
map windows. NTB defination is that only access part of both system memory,
so anyway need once memcpy. Although NTB can't take 100% eDMA advantage, it
is still easiest path now. I have a draft idea without touch NTB core code
(most likley).
EP side RC side
1: Control bar
2: Doorbell bar
3: WM1
MW1 is fixed sized array [ntb_payload_header + data]. Current NTB built
queue in system memory, transfer data (RW) to this array.
Use EDMA only one side, RC/EP. use EP as example.
In 1 (control bar, resever memory space, which call B)
In ntb_hw_epf.c driver, create a simple 'fake' DMA memcpy driver, which
just implement device_prep_dma_memcpy(). That just put src\dest\size info
to memory space B, then push doorbell.
in EP side's a workqueue, fetch info from B, the send to EDMA queue to
do actual transfer, after EP DMA finish, mark done at B, then raise msi irq,
'fake' DMA memcpy driver will be triggered.
Futher, 3 WM1 is not necessary existed at all, because both side don't
access it directly.
For example:
case RC TX, EP RX
RC ntb_async_tx_submit() use device_prep_dma_memcpy() copy user space
memory (0xRC_1000 to PCI_1000, size 0x1000), put into share bar0 position
0xRC_1000 -> 0xPCI_1000 0x1000
EP side, there RX request ntb_async_rx_submit(), from 0xPCI_1000 to
0xEP_8000 size 0x20000.
so setup eDMA transfer form 0xRC_1000 -> 0xEP_8000 size 1000. After complete
mark both side done, then trigger related callback functions.
You can see 0xPCI_1000 is not used at all. Actually 0xPCI_1000 is trouble
maker, RC and EP system PCI space is not necesary the same as CPU space,
PCI controller may do address convert.
Frank
>
> Thank you for your time and review,
> Koichiro
>
> >
> > Frank
> > >
> > > >
> > > > Frank
> > > > >
> > > > > Thanks for reviewing,
> > > > > Koichiro
> > > > >
> > > > > >
> > > > > > Frank
> > > > > >
> > > > > > > + if (ret) {
> > > > > > > + dev_err(&ndev->dev, "dw_edma_probe failed: %d\n", ret);
> > > > > > > + return ret;
> > > > > > > + }
> > > > > > > +
> > > > > > > + return 0;
> > > > > > > +}
> > > > > > > +
> > > > ...
> > > >
> > > > > > > +{
> > > > > > > + spin_lock_init(&qp->ep_tx_lock);
> > > > > > > + spin_lock_init(&qp->ep_rx_lock);
> > > > > > > + spin_lock_init(&qp->rc_lock);
> > > > > > > +}
> > > > > > > +
> > > > > > > +static const struct ntb_transport_backend_ops edma_backend_ops = {
> > > > > > > + .setup_qp_mw = ntb_transport_edma_setup_qp_mw,
> > > > > > > + .tx_free_entry = ntb_transport_edma_tx_free_entry,
> > > > > > > + .tx_enqueue = ntb_transport_edma_tx_enqueue,
> > > > > > > + .rx_enqueue = ntb_transport_edma_rx_enqueue,
> > > > > > > + .rx_poll = ntb_transport_edma_rx_poll,
> > > > > > > + .debugfs_stats_show = ntb_transport_edma_debugfs_stats_show,
> > > > > > > +};
> > > > > > > +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> > > > > > > +
> > > > > > > /**
> > > > > > > * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
> > > > > > > * @qp: NTB transport layer queue to be enabled
> > > > > > > --
> > > > > > > 2.48.1
> > > > > > >
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-12-04 20:16 ` Frank Li
@ 2025-12-05 3:04 ` Koichiro Den
2025-12-05 15:06 ` Frank Li
0 siblings, 1 reply; 19+ messages in thread
From: Koichiro Den @ 2025-12-05 3:04 UTC (permalink / raw)
To: Frank Li
Cc: ntb, linux-pci, dmaengine, linux-kernel, mani, kwilczynski,
kishon, bhelgaas, corbet, vkoul, jdmason, dave.jiang, allenbh,
Basavaraj.Natikar, Shyam-sundar.S-k, kurt.schwemmer, logang,
jingoohan1, lpieralisi, robh, jbrunet, fancer.lancer, arnd,
pstanner, elfring
On Thu, Dec 04, 2025 at 03:16:25PM -0500, Frank Li wrote:
> On Fri, Dec 05, 2025 at 12:42:03AM +0900, Koichiro Den wrote:
> > On Wed, Dec 03, 2025 at 11:14:43AM -0500, Frank Li wrote:
> > > On Wed, Dec 03, 2025 at 05:53:03PM +0900, Koichiro Den wrote:
> > > > On Tue, Dec 02, 2025 at 10:42:29AM -0500, Frank Li wrote:
> > > > > On Tue, Dec 02, 2025 at 03:43:10PM +0900, Koichiro Den wrote:
> > > > > > On Mon, Dec 01, 2025 at 04:41:05PM -0500, Frank Li wrote:
> > > > > > > On Sun, Nov 30, 2025 at 01:03:58AM +0900, Koichiro Den wrote:
> > > > > > > > Add a new transport backend that uses a remote DesignWare eDMA engine
> > > > > > > > located on the NTB endpoint to move data between host and endpoint.
> > > > > > > >
> > > > > ...
> > > > > > > > +#include "ntb_edma.h"
> > > > > > > > +
> > > > > > > > +/*
> > > > > > > > + * The interrupt register offsets below are taken from the DesignWare
> > > > > > > > + * eDMA "unrolled" register map (EDMA_MF_EDMA_UNROLL). The remote eDMA
> > > > > > > > + * backend currently only supports this layout.
> > > > > > > > + */
> > > > > > > > +#define DMA_WRITE_INT_STATUS_OFF 0x4c
> > > > > > > > +#define DMA_WRITE_INT_MASK_OFF 0x54
> > > > > > > > +#define DMA_WRITE_INT_CLEAR_OFF 0x58
> > > > > > > > +#define DMA_READ_INT_STATUS_OFF 0xa0
> > > > > > > > +#define DMA_READ_INT_MASK_OFF 0xa8
> > > > > > > > +#define DMA_READ_INT_CLEAR_OFF 0xac
> > > > > > >
> > > > > > > Not sure why need access EDMA register because EMDA driver already export
> > > > > > > as dmaengine driver.
> > > > > >
> > > > > > These are intended for EP use. In my current design I intentionally don't
> > > > > > use the standard dw-edma dmaengine driver on the EP side.
> > > > >
> > > > > why not?
> > > >
> > > > Conceptually I agree that using the standard dw-edma driver on both sides
> > > > would be attractive for future extensibility and maintainability. However,
> > > > there are a couple of concerns for me, some of which might be alleviated by
> > > > your suggestion below, and some which are more generic safety concerns that
> > > > I tried to outline in my replies to your other comments.
> > > >
> > > > >
> > > > > >
> > > > > > >
> > > > > > > > +
> > > > > > > > +#define NTB_EDMA_NOTIFY_MAX_QP 64
> > > > > > > > +
> > > > > ...
> > > > > > > > +
> > > > > > > > + virq = irq_create_fwspec_mapping(&fwspec);
> > > > > > > > + of_node_put(parent);
> > > > > > > > + return (virq > 0) ? virq : -EINVAL;
> > > > > > > > +}
> > > > > > > > +
> > > > > > > > +static irqreturn_t ntb_edma_isr(int irq, void *data)
> > > > > > > > +{
> > > > > > >
> > > > > > > Not sue why dw_edma_interrupt_write/read() does work for your case. Suppose
> > > > > > > just register callback for dmeengine.
> > > > > >
> > > > > > If we ran dw_edma_probe() on both the EP and RC sides and let the dmaengine
> > > > > > callbacks handle int_status/int_clear, I think we could hit races. One side
> > > > > > might clear a status bit before the other side has a chance to see it and
> > > > > > invoke its callback. Please correct me if I'm missing something here.
> > > > >
> > > > > You should use difference channel?
> > > >
> > > > Do you mean something like this:
> > > > - on EP side, dw_edma_probe() only set up a dedicated channel for notification,
> > > > - on RC side, do not set up that particular channel via dw_edma_channel_setup(),
> > > > but do other remaining channels for DMA transfers.
> > >
> > > Yes, it may be simple overall. Of course this will waste a channel.
> >
> > So, on the EP side I see two possible approaches:
> >
> > (a) Hide "dma" [1] as in [RFC PATCH v2 26/27] and call dw_edma_probe() with
> > hand-crafted settings (chip->ll_rd_cnt = 1, chip->ll_wr_cnt = 0).
> > (b) Or, teach this special-purpose policy (i.e. configuring only a single
> > notification channel) to the SoC glue driver's dw_pcie_ep_init_registers(),
> > for example via Kconfig. I don't think DT is a good place to describe
> > such a policy.
> >
> > There is also another option, which do not necessarily run dw_edma_probe()
> > by ourselves:
> >
> > (c) Leave the default initialization by the SoC glue as-is, and override the
> > per-channel role via some new dw-edma interface, with the guarantee
> > that all channels except the notification channel remain unused on its
> > side afterwards. In this model, the EP side builds the LL locations
> > for data transfers and the RC configures all channels, but it sets up
> > the notification channel in a special manner.
> >
> > [1] https://github.com/jonmason/ntb/blob/68113d260674/Documentation/devicetree/bindings/pci/snps%2Cdw-pcie-ep.yaml#L83
> >
> > >
> > > >
> > > > Also, is it generically safe to have dw_edma_probe() executed from both ends on
> > > > the same eDMA instance, as long as the channels are carefully partitioned
> > > > between them?
> > >
> > > Channel register MMIO space is sperated. Some channel register shared
> > > into one 32bit register.
> > >
> > > But the critical one, interrupt status is w1c. So only write BIT(channel)
> > > is safe.
> > >
> > > Need careful handle irq enable/disable.
> >
> > Yeah, I agree it is unavoidable in this model.
> >
> > >
> > > Or you can defer all actual DMA transfer to EP side, you can append
> > > MSI write at last item of link to notify RC side about DMA done. (actually
> > > RIE should do the same thing)
> > >
> > > >
> > > > >
> > > > > >
> > > > > > To avoid that, in my current implementation, the RC side handles the
> > > > > > status/int_clear registers in the usual way, and the EP side only tries to
> > > > > > suppress needless edma_int as much as possible.
> > > > > >
> > > > > > That said, I'm now wondering if it would be better to set LIE=0/RIE=1 for
> > > > > > the DMA transfer channels and LIE=1/RIE=0 for the notification channel.
> > > > > > That would require some changes on dw-edma core.
> > > > >
> > > > > If dw-edma work as remote DMA, which should enable RIE. like
> > > > > dw-edma-pcie.c, but not one actually use it recently.
> > > > >
> > > > > Use EDMA as doorbell should be new case and I think it is quite useful.
> > > > >
> > > > > > >
> > > > > > > > + struct ntb_edma_interrupt *v = data;
> > > > > > > > + u32 mask = BIT(EDMA_RD_CH_NUM);
> > > > > > > > + u32 i, val;
> > > > > > > > +
> > > > > ...
> > > > > > > > + ret = dw_edma_probe(chip);
> > > > > > >
> > > > > > > I think dw_edma_probe() should be in ntb_hw_epf.c, which provide DMA
> > > > > > > dma engine support.
> > > > > > >
> > > > > > > EP side, suppose default dwc controller driver already setup edma engine,
> > > > > > > so use correct filter function, you should get dma chan.
> > > > > >
> > > > > > I intentionally hid edma for EP side in .dts patch in [RFC PATCH v2 26/27]
> > > > > > so that RC side only manages eDMA remotely and avoids the potential race
> > > > > > condition I mentioned above.
> > > > >
> > > > > Improve eDMA core to suppport some dma channel work at local, some for
> > > > > remote.
> > > >
> > > > Right, Firstly I experimented a bit more with different LIE/RIE settings and
> > > > ended up with the following observations:
> > > >
> > > > * LIE=0/RIE=1 does not seem to work at the hardware level. When I tried this for
> > > > DMA transfer channels, the RC side never received any interrupt. The databook
> > > > (5.40a, 8.2.2 "Interrupts and Error Handling") has a hint that says
> > > > "If you want a remote interrupt and not a local interrupt then: Set LIE and
> > > > RIE [...]", so I think this behaviour is expected.
> > >
> > > Actually, you can append MSI write at last one of DMA descriptor link. So
> > > it will not depend on eDMA's IRQ at all.
> >
> > For RC->EP interrupts on R-Car S4 in EP mode, using ITS_TRANSLATER as the
> > IB iATU target did not appear to work in practice. Indeed that was the
> > motivation for the RFC v1 series [2]. I have not tried using ITS_TRANSLATER
> > as the eDMA read transfer DAR.
> >
> > But in any case, simply masking the local interrupt is sufficient here. I
> > mainly wanted to point out that my naive idea of LIE=0/RIE=1 is not
> > implementable with this hardware. This whole LIE/RIE topic is a bit
> > off-track, sorry for the noise.
> >
> > [2] For the record, RFC v2 is conceptually orthogonal and introduces a
> > broader concept ie. remote eDMA model, but I reused many of the
> > preparatory commits from v1, which is why this is RFC v2 rather than a
> > separate series.
> >
> > >
> > > > * LIE=1/RIE=0 does work at the hardware level, but is problematic for my current
> > > > design, where the RC issues the DMA transfer for the notification via
> > > > ntb_edma_notify_peer(). With RIE=0, the RC never calls
> > > > dw_edma_core_handle_int() for that channel, which means that internal state
> > > > such as dw_edma_chan.status is never managed correctly.
> > >
> > > If you append on MSI write at DMA link, you needn't check status register,
> > > just check current LL pos to know which descrptor already done.
> > >
> > > Or you also enable LIE and disable related IRQ line(without register
> > > irq handler), so Local IRQ will be ignore by GIC, you can safe handle at
> > > RC side.
> >
> > What I was worried about here is that, with RIE=0 the current dw-edma
> > handling of struct dw_edma_chan::status field (not status register) would
> > not run for that channel, which could affect subsequent tx submissions. But
> > your suggestion also makes sense, thank you.
> >
> > --8<--
> >
> > So anyway the key point seems that we should avoid such hard-coded register
> > handling in [RFC PATCH v2 20/27] and rely only on the standard dw-edma
> > interfaces (possibly with some extensions to the dw-edma core). From your
> > feedback, I feel this is the essential direction.
> >
> > From that perspective, I'm leaning toward (b) (which I wrote above in a
> > reply comment) with a Kconfig guard, i.e. in dw_pcie_ep_init_registers(),
> > if IS_ENABLED(CONFIG_DW_REMOTE_EDMA) we only configure the notification
> > channel. In practice, a DT-based variant of (b) (for example a new property
> > such as "dma-notification-channel = <N>;" and making
> > dw_pcie_ep_init_registers() honour it) would be very handy for users, but I
> > suspect putting this kind of policy into DT is not acceptable.
> >
> > Assuming careful handling, (c) might actually be the simplest approach. I
> > may need to add a small hook for the notification channel in
> > dw_edma_done_interrupt(), via a new API such as
> > dw_edma_chan_register_notify().
>
> I reply everything here for overall design
>
> EDMA actually can access all memory at both EP and RC side regardless PCI
> map windows. NTB defination is that only access part of both system memory,
> so anyway need once memcpy. Although NTB can't take 100% eDMA advantage, it
> is still easiest path now. I have a draft idea without touch NTB core code
> (most likley).
>
> EP side RC side
> 1: Control bar
> 2: Doorbell bar
> 3: WM1
>
> MW1 is fixed sized array [ntb_payload_header + data]. Current NTB built
> queue in system memory, transfer data (RW) to this array.
>
> Use EDMA only one side, RC/EP. use EP as example.
>
> In 1 (control bar, resever memory space, which call B)
>
> In ntb_hw_epf.c driver, create a simple 'fake' DMA memcpy driver, which
> just implement device_prep_dma_memcpy(). That just put src\dest\size info
> to memory space B, then push doorbell.
>
> in EP side's a workqueue, fetch info from B, the send to EDMA queue to
> do actual transfer, after EP DMA finish, mark done at B, then raise msi irq,
> 'fake' DMA memcpy driver will be triggered.
>
> Futher, 3 WM1 is not necessary existed at all, because both side don't
> access it directly.
>
> For example:
>
> case RC TX, EP RX
>
> RC ntb_async_tx_submit() use device_prep_dma_memcpy() copy user space
> memory (0xRC_1000 to PCI_1000, size 0x1000), put into share bar0 position
>
> 0xRC_1000 -> 0xPCI_1000 0x1000
>
> EP side, there RX request ntb_async_rx_submit(), from 0xPCI_1000 to
> 0xEP_8000 size 0x20000.
>
> so setup eDMA transfer form 0xRC_1000 -> 0xEP_8000 size 1000. After complete
> mark both side done, then trigger related callback functions.
>
> You can see 0xPCI_1000 is not used at all. Actually 0xPCI_1000 is trouble
> maker, RC and EP system PCI space is not necesary the same as CPU space,
> PCI controller may do address convert.
Thanks for the detailed explanation.
Just to clarify, regarding your comments about the number of memcpy
operations and not using the 0xPCI_1000 window for data path, I think RFC
v2 is already similar to what you're describing.
To me it seems the key differences in your proposal are mainly two-fold:
(1) the layering, and (2) local eDMA use rather than remote.
For (1), instead of adding more eDMA-specific handling into ntb_transport
layer, your approach would keep changes to ntb_transport minimal and
encapsulate the eDMA usage inside the "fake DMA memcpy driver" as much as
possible. In that design, would the MW1 layout change? Leaving the existing
layout as-is would waste the space (so RFC v2 had introduced a new layout).
Also, one point I'm still unsure about is the opposite direction (ie.
EP->RC). In that case, do you also expect the EP to trigger its local eDMA
engine? If yes, then, similar to the RC->EP direction in RFC v2, the EP
would need to know the RC-side receive buffer address (e.g. 0xRC_1000) in
advance.
You also mentioned that you already have some draft. Are you planning to
post that as a patch series? If not, I can of course try to
implement/prototype this approach based on your suggestion.
Please let me know if the above understanding does not match what you had
in mind.
Thank you,
Koichiro
>
> Frank
> >
> > Thank you for your time and review,
> > Koichiro
> >
> > >
> > > Frank
> > > >
> > > > >
> > > > > Frank
> > > > > >
> > > > > > Thanks for reviewing,
> > > > > > Koichiro
> > > > > >
> > > > > > >
> > > > > > > Frank
> > > > > > >
> > > > > > > > + if (ret) {
> > > > > > > > + dev_err(&ndev->dev, "dw_edma_probe failed: %d\n", ret);
> > > > > > > > + return ret;
> > > > > > > > + }
> > > > > > > > +
> > > > > > > > + return 0;
> > > > > > > > +}
> > > > > > > > +
> > > > > ...
> > > > >
> > > > > > > > +{
> > > > > > > > + spin_lock_init(&qp->ep_tx_lock);
> > > > > > > > + spin_lock_init(&qp->ep_rx_lock);
> > > > > > > > + spin_lock_init(&qp->rc_lock);
> > > > > > > > +}
> > > > > > > > +
> > > > > > > > +static const struct ntb_transport_backend_ops edma_backend_ops = {
> > > > > > > > + .setup_qp_mw = ntb_transport_edma_setup_qp_mw,
> > > > > > > > + .tx_free_entry = ntb_transport_edma_tx_free_entry,
> > > > > > > > + .tx_enqueue = ntb_transport_edma_tx_enqueue,
> > > > > > > > + .rx_enqueue = ntb_transport_edma_rx_enqueue,
> > > > > > > > + .rx_poll = ntb_transport_edma_rx_poll,
> > > > > > > > + .debugfs_stats_show = ntb_transport_edma_debugfs_stats_show,
> > > > > > > > +};
> > > > > > > > +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> > > > > > > > +
> > > > > > > > /**
> > > > > > > > * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
> > > > > > > > * @qp: NTB transport layer queue to be enabled
> > > > > > > > --
> > > > > > > > 2.48.1
> > > > > > > >
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-12-05 3:04 ` Koichiro Den
@ 2025-12-05 15:06 ` Frank Li
2025-12-18 4:34 ` Koichiro Den
0 siblings, 1 reply; 19+ messages in thread
From: Frank Li @ 2025-12-05 15:06 UTC (permalink / raw)
To: Koichiro Den
Cc: ntb, linux-pci, dmaengine, linux-kernel, mani, kwilczynski,
kishon, bhelgaas, corbet, vkoul, jdmason, dave.jiang, allenbh,
Basavaraj.Natikar, Shyam-sundar.S-k, kurt.schwemmer, logang,
jingoohan1, lpieralisi, robh, jbrunet, fancer.lancer, arnd,
pstanner, elfring
On Fri, Dec 05, 2025 at 12:04:24PM +0900, Koichiro Den wrote:
> On Thu, Dec 04, 2025 at 03:16:25PM -0500, Frank Li wrote:
> > On Fri, Dec 05, 2025 at 12:42:03AM +0900, Koichiro Den wrote:
> > > On Wed, Dec 03, 2025 at 11:14:43AM -0500, Frank Li wrote:
> > > > On Wed, Dec 03, 2025 at 05:53:03PM +0900, Koichiro Den wrote:
> > > > > On Tue, Dec 02, 2025 at 10:42:29AM -0500, Frank Li wrote:
> > > > > > On Tue, Dec 02, 2025 at 03:43:10PM +0900, Koichiro Den wrote:
> > > > > > > On Mon, Dec 01, 2025 at 04:41:05PM -0500, Frank Li wrote:
> > > > > > > > On Sun, Nov 30, 2025 at 01:03:58AM +0900, Koichiro Den wrote:
> > > > > > > > > Add a new transport backend that uses a remote DesignWare eDMA engine
> > > > > > > > > located on the NTB endpoint to move data between host and endpoint.
> > > > > > > > >
> > > > > > ...
> > > > > > > > > +#include "ntb_edma.h"
> > > > > > > > > +
> > > > > > > > > +/*
> > > > > > > > > + * The interrupt register offsets below are taken from the DesignWare
> > > > > > > > > + * eDMA "unrolled" register map (EDMA_MF_EDMA_UNROLL). The remote eDMA
> > > > > > > > > + * backend currently only supports this layout.
> > > > > > > > > + */
> > > > > > > > > +#define DMA_WRITE_INT_STATUS_OFF 0x4c
> > > > > > > > > +#define DMA_WRITE_INT_MASK_OFF 0x54
> > > > > > > > > +#define DMA_WRITE_INT_CLEAR_OFF 0x58
> > > > > > > > > +#define DMA_READ_INT_STATUS_OFF 0xa0
> > > > > > > > > +#define DMA_READ_INT_MASK_OFF 0xa8
> > > > > > > > > +#define DMA_READ_INT_CLEAR_OFF 0xac
> > > > > > > >
> > > > > > > > Not sure why need access EDMA register because EMDA driver already export
> > > > > > > > as dmaengine driver.
> > > > > > >
> > > > > > > These are intended for EP use. In my current design I intentionally don't
> > > > > > > use the standard dw-edma dmaengine driver on the EP side.
> > > > > >
> > > > > > why not?
> > > > >
> > > > > Conceptually I agree that using the standard dw-edma driver on both sides
> > > > > would be attractive for future extensibility and maintainability. However,
> > > > > there are a couple of concerns for me, some of which might be alleviated by
> > > > > your suggestion below, and some which are more generic safety concerns that
> > > > > I tried to outline in my replies to your other comments.
> > > > >
> > > > > >
> > > > > > >
> > > > > > > >
> > > > > > > > > +
> > > > > > > > > +#define NTB_EDMA_NOTIFY_MAX_QP 64
> > > > > > > > > +
> > > > > > ...
> > > > > > > > > +
> > > > > > > > > + virq = irq_create_fwspec_mapping(&fwspec);
> > > > > > > > > + of_node_put(parent);
> > > > > > > > > + return (virq > 0) ? virq : -EINVAL;
> > > > > > > > > +}
> > > > > > > > > +
> > > > > > > > > +static irqreturn_t ntb_edma_isr(int irq, void *data)
> > > > > > > > > +{
> > > > > > > >
> > > > > > > > Not sue why dw_edma_interrupt_write/read() does work for your case. Suppose
> > > > > > > > just register callback for dmeengine.
> > > > > > >
> > > > > > > If we ran dw_edma_probe() on both the EP and RC sides and let the dmaengine
> > > > > > > callbacks handle int_status/int_clear, I think we could hit races. One side
> > > > > > > might clear a status bit before the other side has a chance to see it and
> > > > > > > invoke its callback. Please correct me if I'm missing something here.
> > > > > >
> > > > > > You should use difference channel?
> > > > >
> > > > > Do you mean something like this:
> > > > > - on EP side, dw_edma_probe() only set up a dedicated channel for notification,
> > > > > - on RC side, do not set up that particular channel via dw_edma_channel_setup(),
> > > > > but do other remaining channels for DMA transfers.
> > > >
> > > > Yes, it may be simple overall. Of course this will waste a channel.
> > >
> > > So, on the EP side I see two possible approaches:
> > >
> > > (a) Hide "dma" [1] as in [RFC PATCH v2 26/27] and call dw_edma_probe() with
> > > hand-crafted settings (chip->ll_rd_cnt = 1, chip->ll_wr_cnt = 0).
> > > (b) Or, teach this special-purpose policy (i.e. configuring only a single
> > > notification channel) to the SoC glue driver's dw_pcie_ep_init_registers(),
> > > for example via Kconfig. I don't think DT is a good place to describe
> > > such a policy.
> > >
> > > There is also another option, which do not necessarily run dw_edma_probe()
> > > by ourselves:
> > >
> > > (c) Leave the default initialization by the SoC glue as-is, and override the
> > > per-channel role via some new dw-edma interface, with the guarantee
> > > that all channels except the notification channel remain unused on its
> > > side afterwards. In this model, the EP side builds the LL locations
> > > for data transfers and the RC configures all channels, but it sets up
> > > the notification channel in a special manner.
> > >
> > > [1] https://github.com/jonmason/ntb/blob/68113d260674/Documentation/devicetree/bindings/pci/snps%2Cdw-pcie-ep.yaml#L83
> > >
> > > >
> > > > >
> > > > > Also, is it generically safe to have dw_edma_probe() executed from both ends on
> > > > > the same eDMA instance, as long as the channels are carefully partitioned
> > > > > between them?
> > > >
> > > > Channel register MMIO space is sperated. Some channel register shared
> > > > into one 32bit register.
> > > >
> > > > But the critical one, interrupt status is w1c. So only write BIT(channel)
> > > > is safe.
> > > >
> > > > Need careful handle irq enable/disable.
> > >
> > > Yeah, I agree it is unavoidable in this model.
> > >
> > > >
> > > > Or you can defer all actual DMA transfer to EP side, you can append
> > > > MSI write at last item of link to notify RC side about DMA done. (actually
> > > > RIE should do the same thing)
> > > >
> > > > >
> > > > > >
> > > > > > >
> > > > > > > To avoid that, in my current implementation, the RC side handles the
> > > > > > > status/int_clear registers in the usual way, and the EP side only tries to
> > > > > > > suppress needless edma_int as much as possible.
> > > > > > >
> > > > > > > That said, I'm now wondering if it would be better to set LIE=0/RIE=1 for
> > > > > > > the DMA transfer channels and LIE=1/RIE=0 for the notification channel.
> > > > > > > That would require some changes on dw-edma core.
> > > > > >
> > > > > > If dw-edma work as remote DMA, which should enable RIE. like
> > > > > > dw-edma-pcie.c, but not one actually use it recently.
> > > > > >
> > > > > > Use EDMA as doorbell should be new case and I think it is quite useful.
> > > > > >
> > > > > > > >
> > > > > > > > > + struct ntb_edma_interrupt *v = data;
> > > > > > > > > + u32 mask = BIT(EDMA_RD_CH_NUM);
> > > > > > > > > + u32 i, val;
> > > > > > > > > +
> > > > > > ...
> > > > > > > > > + ret = dw_edma_probe(chip);
> > > > > > > >
> > > > > > > > I think dw_edma_probe() should be in ntb_hw_epf.c, which provide DMA
> > > > > > > > dma engine support.
> > > > > > > >
> > > > > > > > EP side, suppose default dwc controller driver already setup edma engine,
> > > > > > > > so use correct filter function, you should get dma chan.
> > > > > > >
> > > > > > > I intentionally hid edma for EP side in .dts patch in [RFC PATCH v2 26/27]
> > > > > > > so that RC side only manages eDMA remotely and avoids the potential race
> > > > > > > condition I mentioned above.
> > > > > >
> > > > > > Improve eDMA core to suppport some dma channel work at local, some for
> > > > > > remote.
> > > > >
> > > > > Right, Firstly I experimented a bit more with different LIE/RIE settings and
> > > > > ended up with the following observations:
> > > > >
> > > > > * LIE=0/RIE=1 does not seem to work at the hardware level. When I tried this for
> > > > > DMA transfer channels, the RC side never received any interrupt. The databook
> > > > > (5.40a, 8.2.2 "Interrupts and Error Handling") has a hint that says
> > > > > "If you want a remote interrupt and not a local interrupt then: Set LIE and
> > > > > RIE [...]", so I think this behaviour is expected.
> > > >
> > > > Actually, you can append MSI write at last one of DMA descriptor link. So
> > > > it will not depend on eDMA's IRQ at all.
> > >
> > > For RC->EP interrupts on R-Car S4 in EP mode, using ITS_TRANSLATER as the
> > > IB iATU target did not appear to work in practice. Indeed that was the
> > > motivation for the RFC v1 series [2]. I have not tried using ITS_TRANSLATER
> > > as the eDMA read transfer DAR.
> > >
> > > But in any case, simply masking the local interrupt is sufficient here. I
> > > mainly wanted to point out that my naive idea of LIE=0/RIE=1 is not
> > > implementable with this hardware. This whole LIE/RIE topic is a bit
> > > off-track, sorry for the noise.
> > >
> > > [2] For the record, RFC v2 is conceptually orthogonal and introduces a
> > > broader concept ie. remote eDMA model, but I reused many of the
> > > preparatory commits from v1, which is why this is RFC v2 rather than a
> > > separate series.
> > >
> > > >
> > > > > * LIE=1/RIE=0 does work at the hardware level, but is problematic for my current
> > > > > design, where the RC issues the DMA transfer for the notification via
> > > > > ntb_edma_notify_peer(). With RIE=0, the RC never calls
> > > > > dw_edma_core_handle_int() for that channel, which means that internal state
> > > > > such as dw_edma_chan.status is never managed correctly.
> > > >
> > > > If you append on MSI write at DMA link, you needn't check status register,
> > > > just check current LL pos to know which descrptor already done.
> > > >
> > > > Or you also enable LIE and disable related IRQ line(without register
> > > > irq handler), so Local IRQ will be ignore by GIC, you can safe handle at
> > > > RC side.
> > >
> > > What I was worried about here is that, with RIE=0 the current dw-edma
> > > handling of struct dw_edma_chan::status field (not status register) would
> > > not run for that channel, which could affect subsequent tx submissions. But
> > > your suggestion also makes sense, thank you.
> > >
> > > --8<--
> > >
> > > So anyway the key point seems that we should avoid such hard-coded register
> > > handling in [RFC PATCH v2 20/27] and rely only on the standard dw-edma
> > > interfaces (possibly with some extensions to the dw-edma core). From your
> > > feedback, I feel this is the essential direction.
> > >
> > > From that perspective, I'm leaning toward (b) (which I wrote above in a
> > > reply comment) with a Kconfig guard, i.e. in dw_pcie_ep_init_registers(),
> > > if IS_ENABLED(CONFIG_DW_REMOTE_EDMA) we only configure the notification
> > > channel. In practice, a DT-based variant of (b) (for example a new property
> > > such as "dma-notification-channel = <N>;" and making
> > > dw_pcie_ep_init_registers() honour it) would be very handy for users, but I
> > > suspect putting this kind of policy into DT is not acceptable.
> > >
> > > Assuming careful handling, (c) might actually be the simplest approach. I
> > > may need to add a small hook for the notification channel in
> > > dw_edma_done_interrupt(), via a new API such as
> > > dw_edma_chan_register_notify().
> >
> > I reply everything here for overall design
> >
> > EDMA actually can access all memory at both EP and RC side regardless PCI
> > map windows. NTB defination is that only access part of both system memory,
> > so anyway need once memcpy. Although NTB can't take 100% eDMA advantage, it
> > is still easiest path now. I have a draft idea without touch NTB core code
> > (most likley).
> >
> > EP side RC side
> > 1: Control bar
> > 2: Doorbell bar
> > 3: WM1
> >
> > MW1 is fixed sized array [ntb_payload_header + data]. Current NTB built
> > queue in system memory, transfer data (RW) to this array.
> >
> > Use EDMA only one side, RC/EP. use EP as example.
> >
> > In 1 (control bar, resever memory space, which call B)
> >
> > In ntb_hw_epf.c driver, create a simple 'fake' DMA memcpy driver, which
> > just implement device_prep_dma_memcpy(). That just put src\dest\size info
> > to memory space B, then push doorbell.
> >
> > in EP side's a workqueue, fetch info from B, the send to EDMA queue to
> > do actual transfer, after EP DMA finish, mark done at B, then raise msi irq,
> > 'fake' DMA memcpy driver will be triggered.
> >
> > Futher, 3 WM1 is not necessary existed at all, because both side don't
> > access it directly.
> >
> > For example:
> >
> > case RC TX, EP RX
> >
> > RC ntb_async_tx_submit() use device_prep_dma_memcpy() copy user space
> > memory (0xRC_1000 to PCI_1000, size 0x1000), put into share bar0 position
> >
> > 0xRC_1000 -> 0xPCI_1000 0x1000
> >
> > EP side, there RX request ntb_async_rx_submit(), from 0xPCI_1000 to
> > 0xEP_8000 size 0x20000.
> >
> > so setup eDMA transfer form 0xRC_1000 -> 0xEP_8000 size 1000. After complete
> > mark both side done, then trigger related callback functions.
> >
> > You can see 0xPCI_1000 is not used at all. Actually 0xPCI_1000 is trouble
> > maker, RC and EP system PCI space is not necesary the same as CPU space,
> > PCI controller may do address convert.
>
> Thanks for the detailed explanation.
>
> Just to clarify, regarding your comments about the number of memcpy
> operations and not using the 0xPCI_1000 window for data path, I think RFC
> v2 is already similar to what you're describing.
>
> To me it seems the key differences in your proposal are mainly two-fold:
> (1) the layering, and (2) local eDMA use rather than remote.
Not big difference between remote and local DMA. My major means just use
oneside is enough. If eDMA handle in remote, EP side need virtual memcpy
and RC side to handle actual transfer.
I use EP as example, just because some logic R/W is reverted between EP/RC.
RC's write is EP's read.
>
> For (1), instead of adding more eDMA-specific handling into ntb_transport
> layer, your approach would keep changes to ntb_transport minimal and
> encapsulate the eDMA usage inside the "fake DMA memcpy driver" as much as
> possible. In that design, would the MW1 layout change? Leaving the existing
> layout as-is would waste the space (so RFC v2 had introduced a new layout).
It is fine if NTB maintainer agree it.
>
> Also, one point I'm still unsure about is the opposite direction (ie.
> EP->RC). In that case, do you also expect the EP to trigger its local eDMA
> engine? If yes, then, similar to the RC->EP direction in RFC v2, the EP
> would need to know the RC-side receive buffer address (e.g. 0xRC_1000) in
> advance.
'fake DMA memcpy driver' already put 0xRC_1000 to one shared memory place.
>
> You also mentioned that you already have some draft. Are you planning to
> post that as a patch series? If not, I can of course try to
> implement/prototype this approach based on your suggestion.
Sorry, I have not actually worked for ntb eDMA before. My work base on RDMA
framework. Idealy, RDMA can do user-space(EP) to user space (RC) data
transfer with zero copy.
But I think NTB is also a good path since RDMA is over complexed.
Frank
>
> Please let me know if the above understanding does not match what you had
> in mind.
>
> Thank you,
> Koichiro
>
>
> >
> > Frank
> > >
> > > Thank you for your time and review,
> > > Koichiro
> > >
> > > >
> > > > Frank
> > > > >
> > > > > >
> > > > > > Frank
> > > > > > >
> > > > > > > Thanks for reviewing,
> > > > > > > Koichiro
> > > > > > >
> > > > > > > >
> > > > > > > > Frank
> > > > > > > >
> > > > > > > > > + if (ret) {
> > > > > > > > > + dev_err(&ndev->dev, "dw_edma_probe failed: %d\n", ret);
> > > > > > > > > + return ret;
> > > > > > > > > + }
> > > > > > > > > +
> > > > > > > > > + return 0;
> > > > > > > > > +}
> > > > > > > > > +
> > > > > > ...
> > > > > >
> > > > > > > > > +{
> > > > > > > > > + spin_lock_init(&qp->ep_tx_lock);
> > > > > > > > > + spin_lock_init(&qp->ep_rx_lock);
> > > > > > > > > + spin_lock_init(&qp->rc_lock);
> > > > > > > > > +}
> > > > > > > > > +
> > > > > > > > > +static const struct ntb_transport_backend_ops edma_backend_ops = {
> > > > > > > > > + .setup_qp_mw = ntb_transport_edma_setup_qp_mw,
> > > > > > > > > + .tx_free_entry = ntb_transport_edma_tx_free_entry,
> > > > > > > > > + .tx_enqueue = ntb_transport_edma_tx_enqueue,
> > > > > > > > > + .rx_enqueue = ntb_transport_edma_rx_enqueue,
> > > > > > > > > + .rx_poll = ntb_transport_edma_rx_poll,
> > > > > > > > > + .debugfs_stats_show = ntb_transport_edma_debugfs_stats_show,
> > > > > > > > > +};
> > > > > > > > > +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> > > > > > > > > +
> > > > > > > > > /**
> > > > > > > > > * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
> > > > > > > > > * @qp: NTB transport layer queue to be enabled
> > > > > > > > > --
> > > > > > > > > 2.48.1
> > > > > > > > >
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-12-05 15:06 ` Frank Li
@ 2025-12-18 4:34 ` Koichiro Den
0 siblings, 0 replies; 19+ messages in thread
From: Koichiro Den @ 2025-12-18 4:34 UTC (permalink / raw)
To: Frank Li
Cc: ntb, linux-pci, dmaengine, linux-kernel, mani, kwilczynski,
kishon, bhelgaas, corbet, vkoul, jdmason, dave.jiang, allenbh,
Basavaraj.Natikar, Shyam-sundar.S-k, kurt.schwemmer, logang,
jingoohan1, lpieralisi, robh, jbrunet, fancer.lancer, arnd,
pstanner, elfring
On Fri, Dec 05, 2025 at 10:06:30AM -0500, Frank Li wrote:
> On Fri, Dec 05, 2025 at 12:04:24PM +0900, Koichiro Den wrote:
> > On Thu, Dec 04, 2025 at 03:16:25PM -0500, Frank Li wrote:
> > > On Fri, Dec 05, 2025 at 12:42:03AM +0900, Koichiro Den wrote:
> > > > On Wed, Dec 03, 2025 at 11:14:43AM -0500, Frank Li wrote:
> > > > > On Wed, Dec 03, 2025 at 05:53:03PM +0900, Koichiro Den wrote:
> > > > > > On Tue, Dec 02, 2025 at 10:42:29AM -0500, Frank Li wrote:
> > > > > > > On Tue, Dec 02, 2025 at 03:43:10PM +0900, Koichiro Den wrote:
> > > > > > > > On Mon, Dec 01, 2025 at 04:41:05PM -0500, Frank Li wrote:
> > > > > > > > > On Sun, Nov 30, 2025 at 01:03:58AM +0900, Koichiro Den wrote:
> > > > > > > > > > Add a new transport backend that uses a remote DesignWare eDMA engine
> > > > > > > > > > located on the NTB endpoint to move data between host and endpoint.
> > > > > > > > > >
> > > > > > > ...
> > > > > > > > > > +#include "ntb_edma.h"
> > > > > > > > > > +
> > > > > > > > > > +/*
> > > > > > > > > > + * The interrupt register offsets below are taken from the DesignWare
> > > > > > > > > > + * eDMA "unrolled" register map (EDMA_MF_EDMA_UNROLL). The remote eDMA
> > > > > > > > > > + * backend currently only supports this layout.
> > > > > > > > > > + */
> > > > > > > > > > +#define DMA_WRITE_INT_STATUS_OFF 0x4c
> > > > > > > > > > +#define DMA_WRITE_INT_MASK_OFF 0x54
> > > > > > > > > > +#define DMA_WRITE_INT_CLEAR_OFF 0x58
> > > > > > > > > > +#define DMA_READ_INT_STATUS_OFF 0xa0
> > > > > > > > > > +#define DMA_READ_INT_MASK_OFF 0xa8
> > > > > > > > > > +#define DMA_READ_INT_CLEAR_OFF 0xac
> > > > > > > > >
> > > > > > > > > Not sure why need access EDMA register because EMDA driver already export
> > > > > > > > > as dmaengine driver.
> > > > > > > >
> > > > > > > > These are intended for EP use. In my current design I intentionally don't
> > > > > > > > use the standard dw-edma dmaengine driver on the EP side.
> > > > > > >
> > > > > > > why not?
> > > > > >
> > > > > > Conceptually I agree that using the standard dw-edma driver on both sides
> > > > > > would be attractive for future extensibility and maintainability. However,
> > > > > > there are a couple of concerns for me, some of which might be alleviated by
> > > > > > your suggestion below, and some which are more generic safety concerns that
> > > > > > I tried to outline in my replies to your other comments.
> > > > > >
> > > > > > >
> > > > > > > >
> > > > > > > > >
> > > > > > > > > > +
> > > > > > > > > > +#define NTB_EDMA_NOTIFY_MAX_QP 64
> > > > > > > > > > +
> > > > > > > ...
> > > > > > > > > > +
> > > > > > > > > > + virq = irq_create_fwspec_mapping(&fwspec);
> > > > > > > > > > + of_node_put(parent);
> > > > > > > > > > + return (virq > 0) ? virq : -EINVAL;
> > > > > > > > > > +}
> > > > > > > > > > +
> > > > > > > > > > +static irqreturn_t ntb_edma_isr(int irq, void *data)
> > > > > > > > > > +{
> > > > > > > > >
> > > > > > > > > Not sue why dw_edma_interrupt_write/read() does work for your case. Suppose
> > > > > > > > > just register callback for dmeengine.
> > > > > > > >
> > > > > > > > If we ran dw_edma_probe() on both the EP and RC sides and let the dmaengine
> > > > > > > > callbacks handle int_status/int_clear, I think we could hit races. One side
> > > > > > > > might clear a status bit before the other side has a chance to see it and
> > > > > > > > invoke its callback. Please correct me if I'm missing something here.
> > > > > > >
> > > > > > > You should use difference channel?
> > > > > >
> > > > > > Do you mean something like this:
> > > > > > - on EP side, dw_edma_probe() only set up a dedicated channel for notification,
> > > > > > - on RC side, do not set up that particular channel via dw_edma_channel_setup(),
> > > > > > but do other remaining channels for DMA transfers.
> > > > >
> > > > > Yes, it may be simple overall. Of course this will waste a channel.
> > > >
> > > > So, on the EP side I see two possible approaches:
> > > >
> > > > (a) Hide "dma" [1] as in [RFC PATCH v2 26/27] and call dw_edma_probe() with
> > > > hand-crafted settings (chip->ll_rd_cnt = 1, chip->ll_wr_cnt = 0).
> > > > (b) Or, teach this special-purpose policy (i.e. configuring only a single
> > > > notification channel) to the SoC glue driver's dw_pcie_ep_init_registers(),
> > > > for example via Kconfig. I don't think DT is a good place to describe
> > > > such a policy.
> > > >
> > > > There is also another option, which do not necessarily run dw_edma_probe()
> > > > by ourselves:
> > > >
> > > > (c) Leave the default initialization by the SoC glue as-is, and override the
> > > > per-channel role via some new dw-edma interface, with the guarantee
> > > > that all channels except the notification channel remain unused on its
> > > > side afterwards. In this model, the EP side builds the LL locations
> > > > for data transfers and the RC configures all channels, but it sets up
> > > > the notification channel in a special manner.
> > > >
> > > > [1] https://github.com/jonmason/ntb/blob/68113d260674/Documentation/devicetree/bindings/pci/snps%2Cdw-pcie-ep.yaml#L83
> > > >
> > > > >
> > > > > >
> > > > > > Also, is it generically safe to have dw_edma_probe() executed from both ends on
> > > > > > the same eDMA instance, as long as the channels are carefully partitioned
> > > > > > between them?
> > > > >
> > > > > Channel register MMIO space is sperated. Some channel register shared
> > > > > into one 32bit register.
> > > > >
> > > > > But the critical one, interrupt status is w1c. So only write BIT(channel)
> > > > > is safe.
> > > > >
> > > > > Need careful handle irq enable/disable.
> > > >
> > > > Yeah, I agree it is unavoidable in this model.
> > > >
> > > > >
> > > > > Or you can defer all actual DMA transfer to EP side, you can append
> > > > > MSI write at last item of link to notify RC side about DMA done. (actually
> > > > > RIE should do the same thing)
> > > > >
> > > > > >
> > > > > > >
> > > > > > > >
> > > > > > > > To avoid that, in my current implementation, the RC side handles the
> > > > > > > > status/int_clear registers in the usual way, and the EP side only tries to
> > > > > > > > suppress needless edma_int as much as possible.
> > > > > > > >
> > > > > > > > That said, I'm now wondering if it would be better to set LIE=0/RIE=1 for
> > > > > > > > the DMA transfer channels and LIE=1/RIE=0 for the notification channel.
> > > > > > > > That would require some changes on dw-edma core.
> > > > > > >
> > > > > > > If dw-edma work as remote DMA, which should enable RIE. like
> > > > > > > dw-edma-pcie.c, but not one actually use it recently.
> > > > > > >
> > > > > > > Use EDMA as doorbell should be new case and I think it is quite useful.
> > > > > > >
> > > > > > > > >
> > > > > > > > > > + struct ntb_edma_interrupt *v = data;
> > > > > > > > > > + u32 mask = BIT(EDMA_RD_CH_NUM);
> > > > > > > > > > + u32 i, val;
> > > > > > > > > > +
> > > > > > > ...
> > > > > > > > > > + ret = dw_edma_probe(chip);
> > > > > > > > >
> > > > > > > > > I think dw_edma_probe() should be in ntb_hw_epf.c, which provide DMA
> > > > > > > > > dma engine support.
> > > > > > > > >
> > > > > > > > > EP side, suppose default dwc controller driver already setup edma engine,
> > > > > > > > > so use correct filter function, you should get dma chan.
> > > > > > > >
> > > > > > > > I intentionally hid edma for EP side in .dts patch in [RFC PATCH v2 26/27]
> > > > > > > > so that RC side only manages eDMA remotely and avoids the potential race
> > > > > > > > condition I mentioned above.
> > > > > > >
> > > > > > > Improve eDMA core to suppport some dma channel work at local, some for
> > > > > > > remote.
> > > > > >
> > > > > > Right, Firstly I experimented a bit more with different LIE/RIE settings and
> > > > > > ended up with the following observations:
> > > > > >
> > > > > > * LIE=0/RIE=1 does not seem to work at the hardware level. When I tried this for
> > > > > > DMA transfer channels, the RC side never received any interrupt. The databook
> > > > > > (5.40a, 8.2.2 "Interrupts and Error Handling") has a hint that says
> > > > > > "If you want a remote interrupt and not a local interrupt then: Set LIE and
> > > > > > RIE [...]", so I think this behaviour is expected.
> > > > >
> > > > > Actually, you can append MSI write at last one of DMA descriptor link. So
> > > > > it will not depend on eDMA's IRQ at all.
> > > >
> > > > For RC->EP interrupts on R-Car S4 in EP mode, using ITS_TRANSLATER as the
> > > > IB iATU target did not appear to work in practice. Indeed that was the
> > > > motivation for the RFC v1 series [2]. I have not tried using ITS_TRANSLATER
> > > > as the eDMA read transfer DAR.
> > > >
> > > > But in any case, simply masking the local interrupt is sufficient here. I
> > > > mainly wanted to point out that my naive idea of LIE=0/RIE=1 is not
> > > > implementable with this hardware. This whole LIE/RIE topic is a bit
> > > > off-track, sorry for the noise.
> > > >
> > > > [2] For the record, RFC v2 is conceptually orthogonal and introduces a
> > > > broader concept ie. remote eDMA model, but I reused many of the
> > > > preparatory commits from v1, which is why this is RFC v2 rather than a
> > > > separate series.
> > > >
> > > > >
> > > > > > * LIE=1/RIE=0 does work at the hardware level, but is problematic for my current
> > > > > > design, where the RC issues the DMA transfer for the notification via
> > > > > > ntb_edma_notify_peer(). With RIE=0, the RC never calls
> > > > > > dw_edma_core_handle_int() for that channel, which means that internal state
> > > > > > such as dw_edma_chan.status is never managed correctly.
> > > > >
> > > > > If you append on MSI write at DMA link, you needn't check status register,
> > > > > just check current LL pos to know which descrptor already done.
> > > > >
> > > > > Or you also enable LIE and disable related IRQ line(without register
> > > > > irq handler), so Local IRQ will be ignore by GIC, you can safe handle at
> > > > > RC side.
> > > >
> > > > What I was worried about here is that, with RIE=0 the current dw-edma
> > > > handling of struct dw_edma_chan::status field (not status register) would
> > > > not run for that channel, which could affect subsequent tx submissions. But
> > > > your suggestion also makes sense, thank you.
> > > >
> > > > --8<--
> > > >
> > > > So anyway the key point seems that we should avoid such hard-coded register
> > > > handling in [RFC PATCH v2 20/27] and rely only on the standard dw-edma
> > > > interfaces (possibly with some extensions to the dw-edma core). From your
> > > > feedback, I feel this is the essential direction.
> > > >
> > > > From that perspective, I'm leaning toward (b) (which I wrote above in a
> > > > reply comment) with a Kconfig guard, i.e. in dw_pcie_ep_init_registers(),
> > > > if IS_ENABLED(CONFIG_DW_REMOTE_EDMA) we only configure the notification
> > > > channel. In practice, a DT-based variant of (b) (for example a new property
> > > > such as "dma-notification-channel = <N>;" and making
> > > > dw_pcie_ep_init_registers() honour it) would be very handy for users, but I
> > > > suspect putting this kind of policy into DT is not acceptable.
> > > >
> > > > Assuming careful handling, (c) might actually be the simplest approach. I
> > > > may need to add a small hook for the notification channel in
> > > > dw_edma_done_interrupt(), via a new API such as
> > > > dw_edma_chan_register_notify().
> > >
> > > I reply everything here for overall design
> > >
> > > EDMA actually can access all memory at both EP and RC side regardless PCI
> > > map windows. NTB defination is that only access part of both system memory,
> > > so anyway need once memcpy. Although NTB can't take 100% eDMA advantage, it
> > > is still easiest path now. I have a draft idea without touch NTB core code
> > > (most likley).
> > >
> > > EP side RC side
> > > 1: Control bar
> > > 2: Doorbell bar
> > > 3: WM1
> > >
> > > MW1 is fixed sized array [ntb_payload_header + data]. Current NTB built
> > > queue in system memory, transfer data (RW) to this array.
> > >
> > > Use EDMA only one side, RC/EP. use EP as example.
> > >
> > > In 1 (control bar, resever memory space, which call B)
> > >
> > > In ntb_hw_epf.c driver, create a simple 'fake' DMA memcpy driver, which
> > > just implement device_prep_dma_memcpy(). That just put src\dest\size info
> > > to memory space B, then push doorbell.
> > >
> > > in EP side's a workqueue, fetch info from B, the send to EDMA queue to
> > > do actual transfer, after EP DMA finish, mark done at B, then raise msi irq,
> > > 'fake' DMA memcpy driver will be triggered.
> > >
> > > Futher, 3 WM1 is not necessary existed at all, because both side don't
> > > access it directly.
> > >
> > > For example:
> > >
> > > case RC TX, EP RX
> > >
> > > RC ntb_async_tx_submit() use device_prep_dma_memcpy() copy user space
> > > memory (0xRC_1000 to PCI_1000, size 0x1000), put into share bar0 position
> > >
> > > 0xRC_1000 -> 0xPCI_1000 0x1000
> > >
> > > EP side, there RX request ntb_async_rx_submit(), from 0xPCI_1000 to
> > > 0xEP_8000 size 0x20000.
> > >
> > > so setup eDMA transfer form 0xRC_1000 -> 0xEP_8000 size 1000. After complete
> > > mark both side done, then trigger related callback functions.
> > >
> > > You can see 0xPCI_1000 is not used at all. Actually 0xPCI_1000 is trouble
> > > maker, RC and EP system PCI space is not necesary the same as CPU space,
> > > PCI controller may do address convert.
> >
> > Thanks for the detailed explanation.
> >
> > Just to clarify, regarding your comments about the number of memcpy
> > operations and not using the 0xPCI_1000 window for data path, I think RFC
> > v2 is already similar to what you're describing.
> >
> > To me it seems the key differences in your proposal are mainly two-fold:
> > (1) the layering, and (2) local eDMA use rather than remote.
>
> Not big difference between remote and local DMA. My major means just use
> oneside is enough. If eDMA handle in remote, EP side need virtual memcpy
> and RC side to handle actual transfer.
>
> I use EP as example, just because some logic R/W is reverted between EP/RC.
> RC's write is EP's read.
>
> >
> > For (1), instead of adding more eDMA-specific handling into ntb_transport
> > layer, your approach would keep changes to ntb_transport minimal and
> > encapsulate the eDMA usage inside the "fake DMA memcpy driver" as much as
> > possible. In that design, would the MW1 layout change? Leaving the existing
> > layout as-is would waste the space (so RFC v2 had introduced a new layout).
>
> It is fine if NTB maintainer agree it.
>
> >
> > Also, one point I'm still unsure about is the opposite direction (ie.
> > EP->RC). In that case, do you also expect the EP to trigger its local eDMA
> > engine? If yes, then, similar to the RC->EP direction in RFC v2, the EP
> > would need to know the RC-side receive buffer address (e.g. 0xRC_1000) in
> > advance.
>
> 'fake DMA memcpy driver' already put 0xRC_1000 to one shared memory place.
>
> >
> > You also mentioned that you already have some draft. Are you planning to
> > post that as a patch series? If not, I can of course try to
> > implement/prototype this approach based on your suggestion.
>
> Sorry, I have not actually worked for ntb eDMA before. My work base on RDMA
> framework. Idealy, RDMA can do user-space(EP) to user space (RC) data
> transfer with zero copy.
>
> But I think NTB is also a good path since RDMA is over complexed.
>
> Frank
Hi Frank,
Thank you for the review and the discussion. Apologies for the delayed
response here in this thread, I needed some time to think through the
re-design.
After further consideration, I sent RFC v3 [1] with the following design:
* all the read channels (including a channel for notification) are driven
by host (RC)
* all the write channels are driven by endpoint (EP)
This way we can avoid both ends touching and updating per-direction
registers concurrently at runtime [2]. Also the data plane behaviour
becomes symmetric in both directions, resulting in a simpler data path in
the NTB transport layer compared to RFC v2. As you commented earlier, RFC
v3 no longer relies on the duplicate hard-coded register offsets, and leave
dma_device/dma_chan initialization to the standard path. RFC v3 no longer
hides eDMA instance on enpoint side, like I did in [RFC PATCH v2 26/28]
[3].
But still I didn't implement the fake DMA memcpy driver idea in RFC v3.
Instead, I chose MW1 layout optimized for eDMA-backed transport, since it
reduces MW1 usage and makes it possible to scale to multiple queue pairs
with deeper ring buffers, which helps fully exploit the potential of the
eDMA-backed transport.
[1] https://lore.kernel.org/all/20251217151609.3162665-1-den@valinux.co.jp/
[2] as a somewhat relevant topic, I've found an existing issue that becomes
observable under heavy load across multiple channels.
https://lore.kernel.org/all/20251217151609.3162665-23-den@valinux.co.jp/
[3] https://lore.kernel.org/all/20251129160405.2568284-27-den@valinux.co.jp/
Thank you again for your time and for the review,
Koichiro
>
> >
> > Please let me know if the above understanding does not match what you had
> > in mind.
> >
> > Thank you,
> > Koichiro
> >
> >
> > >
> > > Frank
> > > >
> > > > Thank you for your time and review,
> > > > Koichiro
> > > >
> > > > >
> > > > > Frank
> > > > > >
> > > > > > >
> > > > > > > Frank
> > > > > > > >
> > > > > > > > Thanks for reviewing,
> > > > > > > > Koichiro
> > > > > > > >
> > > > > > > > >
> > > > > > > > > Frank
> > > > > > > > >
> > > > > > > > > > + if (ret) {
> > > > > > > > > > + dev_err(&ndev->dev, "dw_edma_probe failed: %d\n", ret);
> > > > > > > > > > + return ret;
> > > > > > > > > > + }
> > > > > > > > > > +
> > > > > > > > > > + return 0;
> > > > > > > > > > +}
> > > > > > > > > > +
> > > > > > > ...
> > > > > > >
> > > > > > > > > > +{
> > > > > > > > > > + spin_lock_init(&qp->ep_tx_lock);
> > > > > > > > > > + spin_lock_init(&qp->ep_rx_lock);
> > > > > > > > > > + spin_lock_init(&qp->rc_lock);
> > > > > > > > > > +}
> > > > > > > > > > +
> > > > > > > > > > +static const struct ntb_transport_backend_ops edma_backend_ops = {
> > > > > > > > > > + .setup_qp_mw = ntb_transport_edma_setup_qp_mw,
> > > > > > > > > > + .tx_free_entry = ntb_transport_edma_tx_free_entry,
> > > > > > > > > > + .tx_enqueue = ntb_transport_edma_tx_enqueue,
> > > > > > > > > > + .rx_enqueue = ntb_transport_edma_rx_enqueue,
> > > > > > > > > > + .rx_poll = ntb_transport_edma_rx_poll,
> > > > > > > > > > + .debugfs_stats_show = ntb_transport_edma_debugfs_stats_show,
> > > > > > > > > > +};
> > > > > > > > > > +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> > > > > > > > > > +
> > > > > > > > > > /**
> > > > > > > > > > * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
> > > > > > > > > > * @qp: NTB transport layer queue to be enabled
> > > > > > > > > > --
> > > > > > > > > > 2.48.1
> > > > > > > > > >
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-11-29 16:03 ` [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode Koichiro Den
` (3 preceding siblings ...)
2025-12-01 21:41 ` Frank Li
@ 2025-12-01 21:46 ` Dave Jiang
2025-12-02 6:59 ` Koichiro Den
4 siblings, 1 reply; 19+ messages in thread
From: Dave Jiang @ 2025-12-01 21:46 UTC (permalink / raw)
To: Koichiro Den, ntb, linux-pci, dmaengine, linux-kernel, Frank.Li
Cc: mani, kwilczynski, kishon, bhelgaas, corbet, vkoul, jdmason,
allenbh, Basavaraj.Natikar, Shyam-sundar.S-k, kurt.schwemmer,
logang, jingoohan1, lpieralisi, robh, jbrunet, fancer.lancer,
arnd, pstanner, elfring
On 11/29/25 9:03 AM, Koichiro Den wrote:
> Add a new transport backend that uses a remote DesignWare eDMA engine
> located on the NTB endpoint to move data between host and endpoint.
>
> In this mode:
>
> - The endpoint exposes a dedicated memory window that contains the
> eDMA register block followed by a small control structure (struct
> ntb_edma_info) and per-channel linked-list (LL) rings.
>
> - On the endpoint side, ntb_edma_setup_mws() allocates the control
> structure and LL rings in endpoint memory, then programs an inbound
> iATU region so that the host can access them via a peer MW.
>
> - On the host side, ntb_edma_setup_peer() ioremaps the peer MW, reads
> ntb_edma_info and configures a dw-edma DMA device to use the LL
> rings provided by the endpoint.
>
> - ntb_transport is extended with a new backend_ops implementation that
> routes TX and RX enqueue/poll operations through the remote eDMA
> rings while keeping the existing shared-memory backend intact.
>
> - The host signals the endpoint via a dedicated DMA read channel.
> 'use_msi' module option is ignored when 'use_remote_edma=1'.
>
> The new mode is guarded by a Kconfig option (NTB_TRANSPORT_EDMA) and a
> module parameter (use_remote_edma). When disabled, the existing
> ntb_transport behaviour is unchanged.
>
> Signed-off-by: Koichiro Den <den@valinux.co.jp>
> ---
> drivers/ntb/Kconfig | 11 +
> drivers/ntb/Makefile | 3 +
> drivers/ntb/ntb_edma.c | 628 ++++++++
> drivers/ntb/ntb_edma.h | 128 ++
I briefly looked over the code. It feels like the EDMA bits should go in drivers/ntb/hw/ rather than drivers/ntb/ given it's pretty specific to the designware hardware. What sits in drivers/ntb should be generic APIs where a different vendor can utilize it and not having to adopt to designware hardware specifics. So maybe a bit more abstractions are needed?
> .../{ntb_transport.c => ntb_transport_core.c} | 1281 ++++++++++++++++-
> 5 files changed, 2048 insertions(+), 3 deletions(-)
> create mode 100644 drivers/ntb/ntb_edma.c
> create mode 100644 drivers/ntb/ntb_edma.h
> rename drivers/ntb/{ntb_transport.c => ntb_transport_core.c} (65%)
>
> diff --git a/drivers/ntb/Kconfig b/drivers/ntb/Kconfig
> index df16c755b4da..db63f02bb116 100644
> --- a/drivers/ntb/Kconfig
> +++ b/drivers/ntb/Kconfig
> @@ -37,4 +37,15 @@ config NTB_TRANSPORT
>
> If unsure, say N.
>
> +config NTB_TRANSPORT_EDMA
> + bool "NTB Transport backed by remote eDMA"
> + depends on NTB_TRANSPORT
> + depends on PCI
> + select DMA_ENGINE
> + help
> + Enable a transport backend that uses a remote DesignWare eDMA engine
> + exposed through a dedicated NTB memory window. The host uses the
> + endpoint's eDMA engine to move data in both directions.
> + Say Y here if you intend to use the 'use_remote_edma' module parameter.
> +
> endif # NTB
> diff --git a/drivers/ntb/Makefile b/drivers/ntb/Makefile
> index 3a6fa181ff99..51f0e1e3aec7 100644
> --- a/drivers/ntb/Makefile
> +++ b/drivers/ntb/Makefile
> @@ -4,3 +4,6 @@ obj-$(CONFIG_NTB_TRANSPORT) += ntb_transport.o
>
> ntb-y := core.o
> ntb-$(CONFIG_NTB_MSI) += msi.o
> +
> +ntb_transport-y := ntb_transport_core.o
> +ntb_transport-$(CONFIG_NTB_TRANSPORT_EDMA) += ntb_edma.o
> diff --git a/drivers/ntb/ntb_edma.c b/drivers/ntb/ntb_edma.c
> new file mode 100644
> index 000000000000..cb35e0d56aa8
> --- /dev/null
> +++ b/drivers/ntb/ntb_edma.c
> @@ -0,0 +1,628 @@
> +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
> +
> +#include <linux/module.h>
> +#include <linux/device.h>
> +#include <linux/pci.h>
> +#include <linux/ntb.h>
> +#include <linux/io.h>
> +#include <linux/iommu.h>
> +#include <linux/dmaengine.h>
> +#include <linux/pci-epc.h>
> +#include <linux/dma/edma.h>
> +#include <linux/irq.h>
> +#include <linux/irqdomain.h>
> +#include <linux/of.h>
> +#include <linux/of_irq.h>
> +#include <dt-bindings/interrupt-controller/arm-gic.h>
> +
> +#include "ntb_edma.h"
> +
> +/*
> + * The interrupt register offsets below are taken from the DesignWare
> + * eDMA "unrolled" register map (EDMA_MF_EDMA_UNROLL). The remote eDMA
> + * backend currently only supports this layout.
> + */
> +#define DMA_WRITE_INT_STATUS_OFF 0x4c
> +#define DMA_WRITE_INT_MASK_OFF 0x54
> +#define DMA_WRITE_INT_CLEAR_OFF 0x58
> +#define DMA_READ_INT_STATUS_OFF 0xa0
> +#define DMA_READ_INT_MASK_OFF 0xa8
> +#define DMA_READ_INT_CLEAR_OFF 0xac
> +
> +#define NTB_EDMA_NOTIFY_MAX_QP 64
> +
> +static unsigned int edma_spi = 417; /* 0x1a1 */
> +module_param(edma_spi, uint, 0644);
> +MODULE_PARM_DESC(edma_spi, "SPI number used by remote eDMA interrupt (EP local)");
> +
> +static u64 edma_regs_phys = 0xe65d5000;
> +module_param(edma_regs_phys, ullong, 0644);
> +MODULE_PARM_DESC(edma_regs_phys, "Physical base address of local eDMA registers (EP)");
> +
> +static unsigned long edma_regs_size = 0x1200;
> +module_param(edma_regs_size, ulong, 0644);
> +MODULE_PARM_DESC(edma_regs_size, "Size of the local eDMA register space (EP)");
> +
> +struct ntb_edma_intr {
> + u32 db[NTB_EDMA_NOTIFY_MAX_QP];
> +};
> +
> +struct ntb_edma_ctx {
> + void *ll_wr_virt[EDMA_WR_CH_NUM];
> + dma_addr_t ll_wr_phys[EDMA_WR_CH_NUM];
> + void *ll_rd_virt[EDMA_RD_CH_NUM + 1];
> + dma_addr_t ll_rd_phys[EDMA_RD_CH_NUM + 1];
> +
> + struct ntb_edma_intr *intr_ep_virt;
> + dma_addr_t intr_ep_phys;
> + struct ntb_edma_intr *intr_rc_virt;
> + dma_addr_t intr_rc_phys;
> + u32 notify_qp_max;
> +
> + bool initialized;
> +};
> +
> +static struct ntb_edma_ctx edma_ctx;
> +
> +typedef void (*ntb_edma_interrupt_cb_t)(void *data, int qp_num);
> +
> +struct ntb_edma_interrupt {
> + int virq;
> + void __iomem *base;
> + ntb_edma_interrupt_cb_t cb;
> + void *data;
> +};
> +
> +static struct ntb_edma_interrupt ntb_edma_intr;
> +
> +static int ntb_edma_map_spi_to_virq(struct device *dev, unsigned int spi)
> +{
> + struct device_node *np = dev_of_node(dev);
> + struct device_node *parent;
> + struct irq_fwspec fwspec = { 0 };
> + int virq;
> +
> + parent = of_irq_find_parent(np);
> + if (!parent)
> + return -ENODEV;
> +
> + fwspec.fwnode = of_fwnode_handle(parent);
> + fwspec.param_count = 3;
> + fwspec.param[0] = GIC_SPI;
> + fwspec.param[1] = spi;
> + fwspec.param[2] = IRQ_TYPE_LEVEL_HIGH;
> +
> + virq = irq_create_fwspec_mapping(&fwspec);
> + of_node_put(parent);
> + return (virq > 0) ? virq : -EINVAL;
> +}
> +
> +static irqreturn_t ntb_edma_isr(int irq, void *data)
> +{
> + struct ntb_edma_interrupt *v = data;
> + u32 mask = BIT(EDMA_RD_CH_NUM);
> + u32 i, val;
> +
> + /*
> + * We do not ack interrupts here but instead we mask all local interrupt
> + * sources except the read channel used for notification. This reduces
> + * needless ISR invocations.
> + *
> + * In theory we could configure LIE=1/RIE=0 only for the notification
> + * transfer (keeping all other channels at LIE=1/RIE=1), but that would
> + * require intrusive changes to the dw-edma core.
> + *
> + * Note: The host side may have already cleared the read interrupt used
> + * for notification, so reading DMA_READ_INT_CLEAR_OFF is not a reliable
> + * way to detect it. As a result, we cannot reliably tell which specific
> + * channel triggered this interrupt. intr_ep_virt->db[i] teaches us
> + * instead.
> + */
> + iowrite32(~0x0, v->base + DMA_WRITE_INT_MASK_OFF);
> + iowrite32(~mask, v->base + DMA_READ_INT_MASK_OFF);
> +
> + if (!v->cb || !edma_ctx.intr_ep_virt)
> + return IRQ_HANDLED;
> +
> + for (i = 0; i < edma_ctx.notify_qp_max; i++) {
> + val = READ_ONCE(edma_ctx.intr_ep_virt->db[i]);
> + if (!val)
> + continue;
> +
> + WRITE_ONCE(edma_ctx.intr_ep_virt->db[i], 0);
> + v->cb(v->data, i);
> + }
> +
> + return IRQ_HANDLED;
> +}
> +
> +int ntb_edma_setup_isr(struct device *dev, struct device *epc_dev,
> + ntb_edma_interrupt_cb_t cb, void *data)
> +{
> + struct ntb_edma_interrupt *v = &ntb_edma_intr;
> + int virq = ntb_edma_map_spi_to_virq(epc_dev->parent, edma_spi);
> + int ret;
> +
> + if (virq < 0) {
> + dev_err(dev, "failed to get virq (%d)\n", virq);
> + return virq;
> + }
> +
> + v->virq = virq;
> + v->cb = cb;
> + v->data = data;
> + if (edma_regs_phys && !v->base)
> + v->base = devm_ioremap(dev, edma_regs_phys, edma_regs_size);
> + if (!v->base) {
> + dev_err(dev, "failed to setup v->base\n");
> + return -1;
> + }
> + ret = devm_request_irq(dev, v->virq, ntb_edma_isr, 0, "ntb-edma", v);
> + if (ret)
> + return ret;
> +
> + if (v->base) {
> + iowrite32(0x0, v->base + DMA_WRITE_INT_MASK_OFF);
> + iowrite32(0x0, v->base + DMA_READ_INT_MASK_OFF);
> + }
> + return 0;
> +}
> +
> +void ntb_edma_teardown_isr(struct device *dev)
> +{
> + struct ntb_edma_interrupt *v = &ntb_edma_intr;
> +
> + /* Mask all write/read interrupts so we don't get called again. */
> + if (v->base) {
> + iowrite32(~0x0, v->base + DMA_WRITE_INT_MASK_OFF);
> + iowrite32(~0x0, v->base + DMA_READ_INT_MASK_OFF);
> + }
> +
> + if (v->virq > 0)
> + devm_free_irq(dev, v->virq, v);
> +
> + if (v->base)
> + devm_iounmap(dev, v->base);
> +
> + v->virq = 0;
> + v->cb = NULL;
> + v->data = NULL;
> +}
> +
> +int ntb_edma_setup_mws(struct ntb_dev *ndev)
> +{
> + const size_t info_bytes = PAGE_SIZE;
> + resource_size_t size_max, offset;
> + dma_addr_t intr_phys, info_phys;
> + u32 wr_done = 0, rd_done = 0;
> + struct ntb_edma_intr *intr;
> + struct ntb_edma_info *info;
> + int peer_mw, mw_index, rc;
> + struct iommu_domain *dom;
> + bool reg_mapped = false;
> + size_t ll_bytes, size;
> + struct pci_epc *epc;
> + struct device *dev;
> + unsigned long iova;
> + phys_addr_t phys;
> + u64 need;
> + u32 i;
> +
> + /* +1 is for interruption */
> + ll_bytes = (EDMA_WR_CH_NUM + EDMA_RD_CH_NUM + 1) * DMA_LLP_MEM_SIZE;
> + need = EDMA_REG_SIZE + info_bytes + ll_bytes;
> +
> + epc = ntb_get_pci_epc(ndev);
> + if (!epc)
> + return -ENODEV;
> + dev = epc->dev.parent;
> +
> + if (edma_ctx.initialized)
> + return 0;
> +
> + info = dma_alloc_coherent(dev, info_bytes, &info_phys, GFP_KERNEL);
> + if (!info)
> + return -ENOMEM;
> +
> + memset(info, 0, info_bytes);
> + info->magic = NTB_EDMA_INFO_MAGIC;
> + info->wr_cnt = EDMA_WR_CH_NUM;
> + info->rd_cnt = EDMA_RD_CH_NUM + 1; /* +1 for interruption */
> + info->regs_phys = edma_regs_phys;
> + info->ll_stride = DMA_LLP_MEM_SIZE;
> +
> + for (i = 0; i < EDMA_WR_CH_NUM; i++) {
> + edma_ctx.ll_wr_virt[i] = dma_alloc_attrs(dev, DMA_LLP_MEM_SIZE,
> + &edma_ctx.ll_wr_phys[i],
> + GFP_KERNEL,
> + DMA_ATTR_FORCE_CONTIGUOUS);
> + if (!edma_ctx.ll_wr_virt[i]) {
> + rc = -ENOMEM;
> + goto err_free_ll;
> + }
> + wr_done++;
> + info->ll_wr_phys[i] = edma_ctx.ll_wr_phys[i];
> + }
> + for (i = 0; i < EDMA_RD_CH_NUM + 1; i++) {
> + edma_ctx.ll_rd_virt[i] = dma_alloc_attrs(dev, DMA_LLP_MEM_SIZE,
> + &edma_ctx.ll_rd_phys[i],
> + GFP_KERNEL,
> + DMA_ATTR_FORCE_CONTIGUOUS);
> + if (!edma_ctx.ll_rd_virt[i]) {
> + rc = -ENOMEM;
> + goto err_free_ll;
> + }
> + rd_done++;
> + info->ll_rd_phys[i] = edma_ctx.ll_rd_phys[i];
> + }
> +
> + /* For interruption */
> + edma_ctx.notify_qp_max = NTB_EDMA_NOTIFY_MAX_QP;
> + intr = dma_alloc_coherent(dev, sizeof(*intr), &intr_phys, GFP_KERNEL);
> + if (!intr) {
> + rc = -ENOMEM;
> + goto err_free_ll;
> + }
> + memset(intr, 0, sizeof(*intr));
> + edma_ctx.intr_ep_virt = intr;
> + edma_ctx.intr_ep_phys = intr_phys;
> + info->intr_dar_base = intr_phys;
> +
> + peer_mw = ntb_peer_mw_count(ndev);
> + if (peer_mw <= 0) {
> + rc = -ENODEV;
> + goto err_free_ll;
> + }
> +
> + mw_index = peer_mw - 1; /* last MW */
> +
> + rc = ntb_mw_get_align(ndev, 0, mw_index, 0, NULL, &size_max,
> + &offset);
> + if (rc)
> + goto err_free_ll;
> +
> + if (size_max < need) {
> + rc = -ENOSPC;
> + goto err_free_ll;
> + }
> +
> + /* Map register space (direct) */
> + dom = iommu_get_domain_for_dev(dev);
> + if (dom) {
> + phys = edma_regs_phys & PAGE_MASK;
> + size = PAGE_ALIGN(EDMA_REG_SIZE + edma_regs_phys - phys);
> + iova = phys;
> +
> + rc = iommu_map(dom, iova, phys, EDMA_REG_SIZE,
> + IOMMU_READ | IOMMU_WRITE | IOMMU_MMIO, GFP_KERNEL);
> + if (rc)
> + dev_err(&ndev->dev, "failed to create direct mapping for eDMA reg space\n");
> + reg_mapped = true;
> + }
> +
> + rc = ntb_mw_set_trans(ndev, 0, mw_index, edma_regs_phys, EDMA_REG_SIZE, offset);
> + if (rc)
> + goto err_unmap_reg;
> +
> + offset += EDMA_REG_SIZE;
> +
> + /* Map ntb_edma_info */
> + rc = ntb_mw_set_trans(ndev, 0, mw_index, info_phys, info_bytes, offset);
> + if (rc)
> + goto err_clear_trans;
> + offset += info_bytes;
> +
> + /* Map LL location */
> + for (i = 0; i < EDMA_WR_CH_NUM; i++) {
> + rc = ntb_mw_set_trans(ndev, 0, mw_index, edma_ctx.ll_wr_phys[i],
> + DMA_LLP_MEM_SIZE, offset);
> + if (rc)
> + goto err_clear_trans;
> + offset += DMA_LLP_MEM_SIZE;
> + }
> + for (i = 0; i < EDMA_RD_CH_NUM + 1; i++) {
> + rc = ntb_mw_set_trans(ndev, 0, mw_index, edma_ctx.ll_rd_phys[i],
> + DMA_LLP_MEM_SIZE, offset);
> + if (rc)
> + goto err_clear_trans;
> + offset += DMA_LLP_MEM_SIZE;
> + }
> + edma_ctx.initialized = true;
> +
> + return 0;
> +
> +err_clear_trans:
> + /*
> + * Tear down the NTB translation window used for the eDMA MW.
> + * There is no sub-range clear API for ntb_mw_set_trans(), so we
> + * unconditionally drop the whole mapping on error.
> + */
> + ntb_mw_clear_trans(ndev, 0, mw_index);
> +
> +err_unmap_reg:
> + if (reg_mapped)
> + iommu_unmap(dom, iova, size);
> +err_free_ll:
> + while (rd_done--)
> + dma_free_attrs(dev, DMA_LLP_MEM_SIZE,
> + edma_ctx.ll_rd_virt[rd_done],
> + edma_ctx.ll_rd_phys[rd_done],
> + DMA_ATTR_FORCE_CONTIGUOUS);
> + while (wr_done--)
> + dma_free_attrs(dev, DMA_LLP_MEM_SIZE,
> + edma_ctx.ll_wr_virt[wr_done],
> + edma_ctx.ll_wr_phys[wr_done],
> + DMA_ATTR_FORCE_CONTIGUOUS);
> + if (edma_ctx.intr_ep_virt)
> + dma_free_coherent(dev, sizeof(struct ntb_edma_intr),
> + edma_ctx.intr_ep_virt,
> + edma_ctx.intr_ep_phys);
> + dma_free_coherent(dev, info_bytes, info, info_phys);
> + return rc;
> +}
> +
> +static int ntb_edma_irq_vector(struct device *dev, unsigned int nr)
> +{
> + struct pci_dev *pdev = to_pci_dev(dev);
> + int ret, nvec;
> +
> + nvec = pci_msi_vec_count(pdev);
> + for (; nr < nvec; nr++) {
> + ret = pci_irq_vector(pdev, nr);
> + if (!irq_has_action(ret))
> + return ret;
> + }
> + return 0;
> +}
> +
> +static const struct dw_edma_plat_ops ntb_edma_ops = {
> + .irq_vector = ntb_edma_irq_vector,
> +};
> +
> +int ntb_edma_setup_peer(struct ntb_dev *ndev)
> +{
> + struct ntb_edma_info *info;
> + unsigned int wr_cnt, rd_cnt;
> + struct dw_edma_chip *chip;
> + void __iomem *edma_virt;
> + phys_addr_t edma_phys;
> + resource_size_t mw_size;
> + u64 off = EDMA_REG_SIZE;
> + int peer_mw, mw_index;
> + unsigned int i;
> + int ret;
> +
> + peer_mw = ntb_peer_mw_count(ndev);
> + if (peer_mw <= 0)
> + return -ENODEV;
> +
> + mw_index = peer_mw - 1; /* last MW */
> +
> + ret = ntb_peer_mw_get_addr(ndev, mw_index, &edma_phys,
> + &mw_size);
> + if (ret)
> + return -1;
> +
> + edma_virt = ioremap(edma_phys, mw_size);
> +
> + chip = devm_kzalloc(&ndev->dev, sizeof(*chip), GFP_KERNEL);
> + if (!chip) {
> + ret = -ENOMEM;
> + return ret;
> + }
> +
> + chip->dev = &ndev->pdev->dev;
> + chip->nr_irqs = 4;
> + chip->ops = &ntb_edma_ops;
> + chip->flags = 0;
> + chip->reg_base = edma_virt;
> + chip->mf = EDMA_MF_EDMA_UNROLL;
> +
> + info = edma_virt + off;
> + if (info->magic != NTB_EDMA_INFO_MAGIC)
> + return -EINVAL;
> + wr_cnt = info->wr_cnt;
> + rd_cnt = info->rd_cnt;
> + chip->ll_wr_cnt = wr_cnt;
> + chip->ll_rd_cnt = rd_cnt;
> + off += PAGE_SIZE;
> +
> + edma_ctx.notify_qp_max = NTB_EDMA_NOTIFY_MAX_QP;
> + edma_ctx.intr_ep_phys = info->intr_dar_base;
> + if (edma_ctx.intr_ep_phys) {
> + edma_ctx.intr_rc_virt =
> + dma_alloc_coherent(&ndev->pdev->dev,
> + sizeof(struct ntb_edma_intr),
> + &edma_ctx.intr_rc_phys,
> + GFP_KERNEL);
> + if (!edma_ctx.intr_rc_virt)
> + return -ENOMEM;
> + memset(edma_ctx.intr_rc_virt, 0,
> + sizeof(struct ntb_edma_intr));
> + }
> +
> + for (i = 0; i < wr_cnt; i++) {
> + chip->ll_region_wr[i].vaddr.io = edma_virt + off;
> + chip->ll_region_wr[i].paddr = info->ll_wr_phys[i];
> + chip->ll_region_wr[i].sz = DMA_LLP_MEM_SIZE;
> + off += DMA_LLP_MEM_SIZE;
> + }
> + for (i = 0; i < rd_cnt; i++) {
> + chip->ll_region_rd[i].vaddr.io = edma_virt + off;
> + chip->ll_region_rd[i].paddr = info->ll_rd_phys[i];
> + chip->ll_region_rd[i].sz = DMA_LLP_MEM_SIZE;
> + off += DMA_LLP_MEM_SIZE;
> + }
> +
> + if (!pci_dev_msi_enabled(ndev->pdev))
> + return -ENXIO;
> +
> + ret = dw_edma_probe(chip);
> + if (ret) {
> + dev_err(&ndev->dev, "dw_edma_probe failed: %d\n", ret);
> + return ret;
> + }
> +
> + return 0;
> +}
> +
> +struct ntb_edma_filter {
> + struct device *dma_dev;
> + u32 direction;
> +};
> +
> +static bool ntb_edma_filter_fn(struct dma_chan *chan, void *arg)
> +{
> + struct ntb_edma_filter *filter = arg;
> + u32 dir = filter->direction;
> + struct dma_slave_caps caps;
> + int ret;
> +
> + if (chan->device->dev != filter->dma_dev)
> + return false;
> +
> + ret = dma_get_slave_caps(chan, &caps);
> + if (ret < 0)
> + return false;
> +
> + return !!(caps.directions & dir);
> +}
> +
> +void ntb_edma_teardown_chans(struct ntb_edma_chans *edma)
> +{
> + unsigned int i;
> +
> + for (i = 0; i < edma->num_wr_chan; i++)
> + dma_release_channel(edma->wr_chan[i]);
> +
> + for (i = 0; i < edma->num_rd_chan; i++)
> + dma_release_channel(edma->rd_chan[i]);
> +
> + if (edma->intr_chan)
> + dma_release_channel(edma->intr_chan);
> +}
> +
> +int ntb_edma_setup_chans(struct device *dma_dev, struct ntb_edma_chans *edma)
> +{
> + struct ntb_edma_filter filter;
> + dma_cap_mask_t dma_mask;
> + unsigned int i;
> +
> + dma_cap_zero(dma_mask);
> + dma_cap_set(DMA_SLAVE, dma_mask);
> +
> + memset(edma, 0, sizeof(*edma));
> + edma->dev = dma_dev;
> +
> + filter.dma_dev = dma_dev;
> + filter.direction = BIT(DMA_DEV_TO_MEM);
> + for (i = 0; i < EDMA_WR_CH_NUM; i++) {
> + edma->wr_chan[i] = dma_request_channel(dma_mask,
> + ntb_edma_filter_fn,
> + &filter);
> + if (!edma->wr_chan[i])
> + break;
> + edma->num_wr_chan++;
> + }
> +
> + filter.direction = BIT(DMA_MEM_TO_DEV);
> + for (i = 0; i < EDMA_RD_CH_NUM; i++) {
> + edma->rd_chan[i] = dma_request_channel(dma_mask,
> + ntb_edma_filter_fn,
> + &filter);
> + if (!edma->rd_chan[i])
> + break;
> + edma->num_rd_chan++;
> + }
> +
> + edma->intr_chan = dma_request_channel(dma_mask, ntb_edma_filter_fn,
> + &filter);
> + if (!edma->intr_chan)
> + dev_warn(dma_dev,
> + "Remote eDMA notify channel could not be allocated\n");
> +
> + if (!edma->num_wr_chan || !edma->num_rd_chan) {
> + dev_warn(dma_dev, "Remote eDMA channels failed to initialize\n");
> + ntb_edma_teardown_chans(edma);
> + return -ENODEV;
> + }
> + return 0;
> +}
> +
> +struct dma_chan *ntb_edma_pick_chan(struct ntb_edma_chans *edma,
> + remote_edma_dir_t dir)
> +{
> + unsigned int n, cur, idx;
> + struct dma_chan **chans;
> + atomic_t *cur_chan;
> +
> + if (dir == REMOTE_EDMA_WRITE) {
> + n = edma->num_wr_chan;
> + chans = edma->wr_chan;
> + cur_chan = &edma->cur_wr_chan;
> + } else {
> + n = edma->num_rd_chan;
> + chans = edma->rd_chan;
> + cur_chan = &edma->cur_rd_chan;
> + }
> + if (WARN_ON_ONCE(!n))
> + return NULL;
> +
> + /* Simple round-robin */
> + cur = (unsigned int)atomic_inc_return(cur_chan) - 1;
> + idx = cur % n;
> + return chans[idx];
> +}
> +
> +int ntb_edma_notify_peer(struct ntb_edma_chans *edma, int qp_num)
> +{
> + struct dma_async_tx_descriptor *txd;
> + struct dma_slave_config cfg;
> + struct scatterlist sgl;
> + dma_cookie_t cookie;
> + struct device *dev;
> +
> + if (!edma || !edma->intr_chan)
> + return -ENXIO;
> +
> + if (qp_num < 0 || qp_num >= edma_ctx.notify_qp_max)
> + return -EINVAL;
> +
> + if (!edma_ctx.intr_rc_virt || !edma_ctx.intr_ep_phys)
> + return -EINVAL;
> +
> + dev = edma->dev;
> + if (!dev)
> + return -ENODEV;
> +
> + WRITE_ONCE(edma_ctx.intr_rc_virt->db[qp_num], 1);
> +
> + /* Ensure store is visible before kicking the DMA transfer */
> + wmb();
> +
> + sg_init_table(&sgl, 1);
> + sg_dma_address(&sgl) = edma_ctx.intr_rc_phys + qp_num * sizeof(u32);
> + sg_dma_len(&sgl) = sizeof(u32);
> +
> + memset(&cfg, 0, sizeof(cfg));
> + cfg.dst_addr = edma_ctx.intr_ep_phys + qp_num * sizeof(u32);
> + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> + cfg.direction = DMA_MEM_TO_DEV;
> +
> + if (dmaengine_slave_config(edma->intr_chan, &cfg))
> + return -EINVAL;
> +
> + txd = dmaengine_prep_slave_sg(edma->intr_chan, &sgl, 1,
> + DMA_MEM_TO_DEV,
> + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> + if (!txd)
> + return -ENOSPC;
> +
> + cookie = dmaengine_submit(txd);
> + if (dma_submit_error(cookie))
> + return -ENOSPC;
> +
> + dma_async_issue_pending(edma->intr_chan);
> + return 0;
> +}
> diff --git a/drivers/ntb/ntb_edma.h b/drivers/ntb/ntb_edma.h
> new file mode 100644
> index 000000000000..da0451827edb
> --- /dev/null
> +++ b/drivers/ntb/ntb_edma.h
> @@ -0,0 +1,128 @@
> +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
> +#ifndef _NTB_EDMA_H_
> +#define _NTB_EDMA_H_
> +
> +#include <linux/completion.h>
> +#include <linux/device.h>
> +#include <linux/interrupt.h>
> +
> +#define EDMA_REG_SIZE SZ_64K
> +#define DMA_LLP_MEM_SIZE SZ_4K
> +#define EDMA_WR_CH_NUM 4
> +#define EDMA_RD_CH_NUM 4
> +#define NTB_EDMA_MAX_CH 8
> +
> +#define NTB_EDMA_INFO_MAGIC 0x45444D41 /* "EDMA" */
> +#define NTB_EDMA_INFO_OFF EDMA_REG_SIZE
> +
> +#define NTB_EDMA_RING_ORDER 7
> +#define NTB_EDMA_RING_ENTRIES (1U << NTB_EDMA_RING_ORDER)
> +#define NTB_EDMA_RING_MASK (NTB_EDMA_RING_ENTRIES - 1)
> +
> +typedef void (*ntb_edma_interrupt_cb_t)(void *data, int qp_num);
> +
> +/*
> + * REMOTE_EDMA_EP:
> + * Endpoint owns the eDMA engine and pushes descriptors into a shared MW.
> + *
> + * REMOTE_EDMA_RC:
> + * Root Complex controls the endpoint eDMA through the shared MW and
> + * drives reads/writes on behalf of the host.
> + */
> +typedef enum {
> + REMOTE_EDMA_UNKNOWN,
> + REMOTE_EDMA_EP,
> + REMOTE_EDMA_RC,
> +} remote_edma_mode_t;
> +
> +typedef enum {
> + REMOTE_EDMA_WRITE,
> + REMOTE_EDMA_READ,
> +} remote_edma_dir_t;
> +
> +/*
> + * Layout of remote eDMA MW (EP local address space, RC sees via peer MW):
> + *
> + * 0 .. EDMA_REG_SIZE-1 : DesignWare eDMA registers
> + * EDMA_REG_SIZE .. +PAGE_SIZE : struct ntb_edma_info (EP writes, RC reads)
> + * +PAGE_SIZE .. : LL ring buffers (EP allocates phys addresses,
> + * RC configures via dw_edma)
> + *
> + * ntb_edma_setup_mws() on EP:
> + * - allocates ntb_edma_info and LLs in EP memory
> + * - programs inbound iATU so that RC peer MW[n] points at this block
> + *
> + * ntb_edma_setup_peer() on RC:
> + * - ioremaps peer MW[n]
> + * - reads ntb_edma_info
> + * - sets up dw_edma_chip ll_region_* from that info
> + */
> +struct ntb_edma_info {
> + u32 magic;
> + u16 wr_cnt;
> + u16 rd_cnt;
> + u64 regs_phys;
> + u32 ll_stride;
> + u32 rsvd;
> + u64 ll_wr_phys[NTB_EDMA_MAX_CH];
> + u64 ll_rd_phys[NTB_EDMA_MAX_CH];
> +
> + u64 intr_dar_base;
> +} __packed;
> +
> +struct ll_dma_addrs {
> + dma_addr_t wr[EDMA_WR_CH_NUM];
> + dma_addr_t rd[EDMA_RD_CH_NUM];
> +};
> +
> +struct ntb_edma_chans {
> + struct device *dev;
> +
> + struct dma_chan *wr_chan[EDMA_WR_CH_NUM];
> + struct dma_chan *rd_chan[EDMA_RD_CH_NUM];
> + struct dma_chan *intr_chan;
> +
> + unsigned int num_wr_chan;
> + unsigned int num_rd_chan;
> + atomic_t cur_wr_chan;
> + atomic_t cur_rd_chan;
> +};
> +
> +static __always_inline u32 ntb_edma_ring_idx(u32 v)
> +{
> + return v & NTB_EDMA_RING_MASK;
> +}
> +
> +static __always_inline u32 ntb_edma_ring_used_entry(u32 head, u32 tail)
> +{
> + if (head >= tail) {
> + WARN_ON_ONCE((head - tail) > (NTB_EDMA_RING_ENTRIES - 1));
> + return head - tail;
> + }
> +
> + WARN_ON_ONCE((U32_MAX - tail + head + 1) > (NTB_EDMA_RING_ENTRIES - 1));
> + return U32_MAX - tail + head + 1;
> +}
> +
> +static __always_inline u32 ntb_edma_ring_free_entry(u32 head, u32 tail)
> +{
> + return NTB_EDMA_RING_ENTRIES - ntb_edma_ring_used_entry(head, tail) - 1;
> +}
> +
> +static __always_inline bool ntb_edma_ring_full(u32 head, u32 tail)
> +{
> + return ntb_edma_ring_free_entry(head, tail) == 0;
> +}
> +
> +int ntb_edma_setup_isr(struct device *dev, struct device *epc_dev,
> + ntb_edma_interrupt_cb_t cb, void *data);
> +void ntb_edma_teardown_isr(struct device *dev);
> +int ntb_edma_setup_mws(struct ntb_dev *ndev);
> +int ntb_edma_setup_peer(struct ntb_dev *ndev);
> +int ntb_edma_setup_chans(struct device *dma_dev, struct ntb_edma_chans *edma);
> +struct dma_chan *ntb_edma_pick_chan(struct ntb_edma_chans *edma,
> + remote_edma_dir_t dir);
> +void ntb_edma_teardown_chans(struct ntb_edma_chans *edma);
> +int ntb_edma_notify_peer(struct ntb_edma_chans *edma, int qp_num);
> +
> +#endif
> diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport_core.c
> similarity index 65%
> rename from drivers/ntb/ntb_transport.c
> rename to drivers/ntb/ntb_transport_core.c
> index 907db6c93d4d..48d48921978d 100644
> --- a/drivers/ntb/ntb_transport.c
> +++ b/drivers/ntb/ntb_transport_core.c
> @@ -47,6 +47,9 @@
> * Contact Information:
> * Jon Mason <jon.mason@intel.com>
> */
> +#include <linux/atomic.h>
> +#include <linux/bug.h>
> +#include <linux/compiler.h>
> #include <linux/debugfs.h>
> #include <linux/delay.h>
> #include <linux/dmaengine.h>
> @@ -71,6 +74,8 @@
> #define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB"
> #define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2)
>
> +#define NTB_EDMA_MAX_POLL 32
> +
> MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
> MODULE_VERSION(NTB_TRANSPORT_VER);
> MODULE_LICENSE("Dual BSD/GPL");
> @@ -102,6 +107,13 @@ module_param(use_msi, bool, 0644);
> MODULE_PARM_DESC(use_msi, "Use MSI interrupts instead of doorbells");
> #endif
>
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
This is a comment throughout this patch. Doing ifdefs inside C source is pretty frowed upon in the kernel. The preferred way is to only have ifdefs in the header files. So please give this a bit more consideration and see if it can be done differently to address this.
> +#include "ntb_edma.h"
> +static bool use_remote_edma;
> +module_param(use_remote_edma, bool, 0644);
> +MODULE_PARM_DESC(use_remote_edma, "Use remote eDMA mode (when enabled, use_msi is ignored)");
> +#endif
> +
> static struct dentry *nt_debugfs_dir;
>
> /* Only two-ports NTB devices are supported */
> @@ -125,6 +137,14 @@ struct ntb_queue_entry {
> struct ntb_payload_header __iomem *tx_hdr;
> struct ntb_payload_header *rx_hdr;
> };
> +
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + dma_addr_t addr;
> +
> + /* Used by RC side only */
> + struct scatterlist sgl;
> + struct work_struct dma_work;
> +#endif
> };
>
> struct ntb_rx_info {
> @@ -202,6 +222,33 @@ struct ntb_transport_qp {
> int msi_irq;
> struct ntb_msi_desc msi_desc;
> struct ntb_msi_desc peer_msi_desc;
> +
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + /*
> + * For ensuring peer notification in non-atomic context.
> + * ntb_peer_db_set might sleep or schedule.
> + */
> + struct work_struct db_work;
> +
> + /*
> + * wr: remote eDMA write transfer (EP -> RC direction)
> + * rd: remote eDMA read transfer (RC -> EP direction)
> + */
> + u32 wr_cons;
> + u32 rd_cons;
> + u32 wr_prod;
> + u32 rd_prod;
> + u32 wr_issue;
> + u32 rd_issue;
> +
> + spinlock_t ep_tx_lock;
> + spinlock_t ep_rx_lock;
> + spinlock_t rc_lock;
> +
> + /* Completion work for read/write transfers. */
> + struct work_struct read_work;
> + struct work_struct write_work;
> +#endif
For something like this, maybe it needs its own struct instead of an ifdef chunk. Perhaps 'ntb_rx_info' can serve as a core data struct with EDMA having 'ntb_rx_info_edma' and embed 'ntb_rx_info'.
DJ
> };
>
> struct ntb_transport_mw {
> @@ -249,6 +296,13 @@ struct ntb_transport_ctx {
>
> /* Make sure workq of link event be executed serially */
> struct mutex link_event_lock;
> +
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + remote_edma_mode_t remote_edma_mode;
> + struct device *dma_dev;
> + struct workqueue_struct *wq;
> + struct ntb_edma_chans edma;
> +#endif
> };
>
> enum {
> @@ -262,6 +316,19 @@ struct ntb_payload_header {
> unsigned int flags;
> };
>
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> +static void ntb_transport_edma_uninit(struct ntb_transport_ctx *nt);
> +static int ntb_transport_edma_init(struct ntb_transport_ctx *nt,
> + unsigned int *mw_count);
> +static void ntb_transport_edma_init_queue(struct ntb_transport_ctx *nt,
> + unsigned int qp_num);
> +static void ntb_transport_edma_create_queue(struct ntb_transport_ctx *nt,
> + struct ntb_transport_qp *qp);
> +static int ntb_transport_edma_rc_init(struct ntb_transport_ctx *nt);
> +static int ntb_transport_edma_ep_init(struct ntb_transport_ctx *nt);
> +static void ntb_transport_edma_rc_dma_work(struct work_struct *work);
> +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> +
> /*
> * Return the device that should be used for DMA mapping.
> *
> @@ -298,7 +365,7 @@ enum {
> container_of((__drv), struct ntb_transport_client, driver)
>
> #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count)
> -#define NTB_QP_DEF_NUM_ENTRIES 100
> +#define NTB_QP_DEF_NUM_ENTRIES 128
> #define NTB_LINK_DOWN_TIMEOUT 10
>
> static void ntb_transport_rxc_db(unsigned long data);
> @@ -1015,6 +1082,10 @@ static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
> count = ntb_spad_count(nt->ndev);
> for (i = 0; i < count; i++)
> ntb_spad_write(nt->ndev, i, 0);
> +
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + ntb_edma_teardown_chans(&nt->edma);
> +#endif
> }
>
> static void ntb_transport_link_cleanup_work(struct work_struct *work)
> @@ -1051,6 +1122,14 @@ static void ntb_transport_link_work(struct work_struct *work)
>
> /* send the local info, in the opposite order of the way we read it */
>
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + rc = ntb_transport_edma_ep_init(nt);
> + if (rc) {
> + dev_err(&pdev->dev, "Failed to init EP: %d\n", rc);
> + return;
> + }
> +#endif
> +
> if (nt->use_msi) {
> rc = ntb_msi_setup_mws(ndev);
> if (rc) {
> @@ -1132,6 +1211,14 @@ static void ntb_transport_link_work(struct work_struct *work)
>
> nt->link_is_up = true;
>
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + rc = ntb_transport_edma_rc_init(nt);
> + if (rc) {
> + dev_err(&pdev->dev, "Failed to init RC: %d\n", rc);
> + goto out1;
> + }
> +#endif
> +
> for (i = 0; i < nt->qp_count; i++) {
> struct ntb_transport_qp *qp = &nt->qp_vec[i];
>
> @@ -1277,6 +1364,8 @@ static const struct ntb_transport_backend_ops default_backend_ops = {
> .debugfs_stats_show = ntb_transport_default_debugfs_stats_show,
> };
>
> +static const struct ntb_transport_backend_ops edma_backend_ops;
> +
> static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> {
> struct ntb_transport_ctx *nt;
> @@ -1311,7 +1400,23 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
>
> nt->ndev = ndev;
>
> - nt->backend_ops = default_backend_ops;
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + if (use_remote_edma) {
> + rc = ntb_transport_edma_init(nt, &mw_count);
> + if (rc) {
> + nt->mw_count = 0;
> + goto err;
> + }
> + nt->backend_ops = edma_backend_ops;
> +
> + /*
> + * On remote eDMA mode, we reserve a read channel for Host->EP
> + * interruption.
> + */
> + use_msi = false;
> + } else
> +#endif
> + nt->backend_ops = default_backend_ops;
>
> /*
> * If we are using MSI, and have at least one extra memory window,
> @@ -1402,6 +1507,10 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> rc = ntb_transport_init_queue(nt, i);
> if (rc)
> goto err2;
> +
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + ntb_transport_edma_init_queue(nt, i);
> +#endif
> }
>
> INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);
> @@ -1433,6 +1542,9 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> }
> kfree(nt->mw_vec);
> err:
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + ntb_transport_edma_uninit(nt);
> +#endif
> kfree(nt);
> return rc;
> }
> @@ -2055,11 +2167,16 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
>
> nt->qp_bitmap_free &= ~qp_bit;
>
> + qp->qp_bit = qp_bit;
> qp->cb_data = data;
> qp->rx_handler = handlers->rx_handler;
> qp->tx_handler = handlers->tx_handler;
> qp->event_handler = handlers->event_handler;
>
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + ntb_transport_edma_create_queue(nt, qp);
> +#endif
> +
> dma_cap_zero(dma_mask);
> dma_cap_set(DMA_MEMCPY, dma_mask);
>
> @@ -2105,6 +2222,9 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
> goto err1;
>
> entry->qp = qp;
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + INIT_WORK(&entry->dma_work, ntb_transport_edma_rc_dma_work);
> +#endif
> ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> &qp->rx_free_q);
> }
> @@ -2156,8 +2276,8 @@ EXPORT_SYMBOL_GPL(ntb_transport_create_queue);
> */
> void ntb_transport_free_queue(struct ntb_transport_qp *qp)
> {
> - struct pci_dev *pdev;
> struct ntb_queue_entry *entry;
> + struct pci_dev *pdev;
> u64 qp_bit;
>
> if (!qp)
> @@ -2208,6 +2328,10 @@ void ntb_transport_free_queue(struct ntb_transport_qp *qp)
> tasklet_kill(&qp->rxc_db_work);
>
> cancel_delayed_work_sync(&qp->link_work);
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> + cancel_work_sync(&qp->read_work);
> + cancel_work_sync(&qp->write_work);
> +#endif
>
> qp->cb_data = NULL;
> qp->rx_handler = NULL;
> @@ -2346,6 +2470,1157 @@ int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
> }
> EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue);
>
> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> +/*
> + * Remote eDMA mode implementation
> + */
> +struct ntb_edma_desc {
> + u32 len;
> + u32 flags;
> + u64 addr; /* DMA address */
> + u64 data;
> +};
> +
> +struct ntb_edma_ring {
> + struct ntb_edma_desc desc[NTB_EDMA_RING_ENTRIES];
> + u32 head;
> + u32 tail;
> +};
> +
> +#define NTB_EDMA_DESC_OFF(i) ((size_t)(i) * sizeof(struct ntb_edma_desc))
> +
> +#define __NTB_EDMA_CHECK_INDEX(_i) \
> +({ \
> + unsigned long __i = (unsigned long)(_i); \
> + WARN_ONCE(__i >= (unsigned long)NTB_EDMA_RING_ENTRIES, \
> + "ntb_edma: index i=%lu >= ring_entries=%lu\n", \
> + __i, (unsigned long)NTB_EDMA_RING_ENTRIES); \
> + __i; \
> +})
> +
> +#define NTB_EDMA_DESC_I(qp, i, n) \
> +({ \
> + typeof(qp) __qp = (qp); \
> + unsigned long __i = __NTB_EDMA_CHECK_INDEX(i); \
> + (struct ntb_edma_desc *) \
> + ((char *)(__qp)->rx_buff + \
> + (sizeof(struct ntb_edma_ring) * n) + \
> + NTB_EDMA_DESC_OFF(__i)); \
> +})
> +
> +#define NTB_EDMA_DESC_O(qp, i, n) \
> +({ \
> + typeof(qp) __qp = (qp); \
> + unsigned long __i = __NTB_EDMA_CHECK_INDEX(i); \
> + (struct ntb_edma_desc __iomem *) \
> + ((char __iomem *)(__qp)->tx_mw + \
> + (sizeof(struct ntb_edma_ring) * n) + \
> + NTB_EDMA_DESC_OFF(__i)); \
> +})
> +
> +#define NTB_EDMA_HEAD_I(qp, n) ((u32 *)((char *)qp->rx_buff + \
> + (sizeof(struct ntb_edma_ring) * n) + \
> + offsetof(struct ntb_edma_ring, head)))
> +#define NTB_EDMA_HEAD_O(qp, n) ((u32 *)((char __iomem *)qp->tx_mw + \
> + (sizeof(struct ntb_edma_ring) * n) + \
> + offsetof(struct ntb_edma_ring, head)))
> +#define NTB_EDMA_TAIL_I(qp, n) ((u32 *)((char *)qp->rx_buff + \
> + (sizeof(struct ntb_edma_ring) * n) + \
> + offsetof(struct ntb_edma_ring, tail)))
> +#define NTB_EDMA_TAIL_O(qp, n) ((u32 *)((char __iomem *)qp->tx_mw + \
> + (sizeof(struct ntb_edma_ring) * n) + \
> + offsetof(struct ntb_edma_ring, tail)))
> +
> +/*
> + * Macro naming rule:
> + * NTB_DESC_RD_EP_I (as an example)
> + * ^^ ^^ ^
> + * : : `-- I(n) or O(ut). In = Read, Out = Write.
> + * : `----- Who uses this macro.
> + * `-------- DESC / HEAD / TAIL
> + *
> + * Read transfers (RC->EP):
> + *
> + * EP view (outbound, written via NTB):
> + * - descs: NTB_DESC_RD_EP_O(qp, i) / NTB_DESC_RD_EP_I(qp, i)
> + * [ len ][ flags ][ addr ][ data ]
> + * [ len ][ flags ][ addr ][ data ]
> + * :
> + * [ len ][ flags ][ addr ][ data ]
> + * - head: NTB_HEAD_RD_EP_O(qp)
> + * - tail: NTB_TAIL_RD_EP_I(qp)
> + *
> + * RC view (inbound, local mapping):
> + * - descs: NTB_DESC_RD_RC_I(qp, i) / NTB_DESC_RD_RC_O(qp, i)
> + * [ len ][ flags ][ addr ][ data ]
> + * [ len ][ flags ][ addr ][ data ]
> + * :
> + * [ len ][ flags ][ addr ][ data ]
> + * - head: NTB_HEAD_RD_RC_I(qp)
> + * - tail: NTB_TAIL_RD_RC_O(qp)
> + *
> + * Write transfers (EP -> RC) are analogous but use
> + * NTB_DESC_WR_{EP_O,RC_I}(), NTB_HEAD_WR_{EP_O,RC_I}(),
> + * and NTB_TAIL_WR_{EP_I,RC_O}().
> + */
> +#define NTB_DESC_RD_EP_I(qp, i) NTB_EDMA_DESC_I(qp, i, 0)
> +#define NTB_DESC_RD_EP_O(qp, i) NTB_EDMA_DESC_O(qp, i, 0)
> +#define NTB_DESC_WR_EP_I(qp, i) NTB_EDMA_DESC_I(qp, i, 1)
> +#define NTB_DESC_WR_EP_O(qp, i) NTB_EDMA_DESC_O(qp, i, 1)
> +#define NTB_DESC_RD_RC_I(qp, i) NTB_EDMA_DESC_I(qp, i, 0)
> +#define NTB_DESC_RD_RC_O(qp, i) NTB_EDMA_DESC_O(qp, i, 0)
> +#define NTB_DESC_WR_RC_I(qp, i) NTB_EDMA_DESC_I(qp, i, 1)
> +#define NTB_DESC_WR_RC_O(qp, i) NTB_EDMA_DESC_O(qp, i, 1)
> +
> +#define NTB_HEAD_RD_EP_O(qp) NTB_EDMA_HEAD_O(qp, 0)
> +#define NTB_HEAD_WR_EP_O(qp) NTB_EDMA_HEAD_O(qp, 1)
> +#define NTB_HEAD_RD_RC_I(qp) NTB_EDMA_HEAD_I(qp, 0)
> +#define NTB_HEAD_WR_RC_I(qp) NTB_EDMA_HEAD_I(qp, 1)
> +
> +#define NTB_TAIL_RD_EP_I(qp) NTB_EDMA_TAIL_I(qp, 0)
> +#define NTB_TAIL_WR_EP_I(qp) NTB_EDMA_TAIL_I(qp, 1)
> +#define NTB_TAIL_RD_RC_O(qp) NTB_EDMA_TAIL_O(qp, 0)
> +#define NTB_TAIL_WR_RC_O(qp) NTB_EDMA_TAIL_O(qp, 1)
> +
> +static inline bool ntb_qp_edma_is_rc(struct ntb_transport_qp *qp)
> +{
> + return qp->transport->remote_edma_mode == REMOTE_EDMA_RC;
> +}
> +
> +static inline bool ntb_qp_edma_is_ep(struct ntb_transport_qp *qp)
> +{
> + return qp->transport->remote_edma_mode == REMOTE_EDMA_EP;
> +}
> +
> +static inline bool ntb_qp_edma_enabled(struct ntb_transport_qp *qp)
> +{
> + return ntb_qp_edma_is_rc(qp) || ntb_qp_edma_is_ep(qp);
> +}
> +
> +static unsigned int ntb_transport_edma_tx_free_entry(struct ntb_transport_qp *qp)
> +{
> + unsigned int head, tail;
> +
> + if (ntb_qp_edma_is_ep(qp)) {
> + scoped_guard(spinlock_irqsave, &qp->ep_tx_lock) {
> + /* In this scope, only 'head' might proceed */
> + tail = READ_ONCE(qp->wr_cons);
> + head = READ_ONCE(qp->wr_prod);
> + }
> + return ntb_edma_ring_free_entry(head, tail);
> + }
> +
> + scoped_guard(spinlock_irqsave, &qp->rc_lock) {
> + /* In this scope, only 'head' might proceed */
> + tail = READ_ONCE(qp->rd_issue);
> + head = READ_ONCE(*NTB_HEAD_RD_RC_I(qp));
> + }
> + /*
> + * On RC side, 'used' amount indicates how much EP side
> + * has refilled, which are available for us to use for TX.
> + */
> + return ntb_edma_ring_used_entry(head, tail);
> +}
> +
> +static void ntb_transport_edma_debugfs_stats_show(struct seq_file *s,
> + struct ntb_transport_qp *qp)
> +{
> + seq_printf(s, "rx_bytes - \t%llu\n", qp->rx_bytes);
> + seq_printf(s, "rx_pkts - \t%llu\n", qp->rx_pkts);
> + seq_printf(s, "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
> + seq_printf(s, "rx_buff - \t0x%p\n", qp->rx_buff);
> + seq_printf(s, "rx_max_entry - \t%u\n", qp->rx_max_entry);
> + seq_printf(s, "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry);
> +
> + seq_printf(s, "tx_bytes - \t%llu\n", qp->tx_bytes);
> + seq_printf(s, "tx_pkts - \t%llu\n", qp->tx_pkts);
> + seq_printf(s, "tx_ring_full - \t%llu\n", qp->tx_ring_full);
> + seq_printf(s, "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
> + seq_printf(s, "tx_mw - \t0x%p\n", qp->tx_mw);
> + seq_printf(s, "tx_max_entry - \t%u\n", qp->tx_max_entry);
> + seq_printf(s, "free tx - \t%u\n", ntb_transport_tx_free_entry(qp));
> + seq_putc(s, '\n');
> +
> + seq_puts(s, "Using Remote eDMA - Yes\n");
> + seq_printf(s, "QP Link - \t%s\n", qp->link_is_up ? "Up" : "Down");
> +}
> +
> +static void ntb_transport_edma_uninit(struct ntb_transport_ctx *nt)
> +{
> + struct ntb_dev *ndev = nt->ndev;
> +
> + if (nt->remote_edma_mode == REMOTE_EDMA_EP && ndev && ndev->pdev)
> + ntb_edma_teardown_isr(&ndev->pdev->dev);
> +
> + if (nt->wq)
> + destroy_workqueue(nt->wq);
> + nt->wq = NULL;
> +}
> +
> +static int ntb_transport_edma_init(struct ntb_transport_ctx *nt,
> + unsigned int *mw_count)
> +{
> + struct ntb_dev *ndev = nt->ndev;
> +
> + /*
> + * We need at least one MW for the transport plus one MW reserved
> + * for the remote eDMA window (see ntb_edma_setup_mws/peer).
> + */
> + if (*mw_count <= 1) {
> + dev_err(&ndev->dev,
> + "remote eDMA requires at least two MWS (have %u)\n",
> + *mw_count);
> + return -ENODEV;
> + }
> +
> + nt->wq = alloc_workqueue("ntb-edma-wq", WQ_UNBOUND | WQ_SYSFS, 0);
> + if (!nt->wq) {
> + ntb_transport_edma_uninit(nt);
> + return -ENOMEM;
> + }
> +
> + /* Reserve the last peer MW exclusively for the eDMA window. */
> + *mw_count -= 1;
> +
> + return 0;
> +}
> +
> +static void ntb_transport_edma_db_work(struct work_struct *work)
> +{
> + struct ntb_transport_qp *qp =
> + container_of(work, struct ntb_transport_qp, db_work);
> +
> + ntb_peer_db_set(qp->ndev, qp->qp_bit);
> +}
> +
> +static void ntb_transport_edma_notify_peer(struct ntb_transport_qp *qp)
> +{
> + if (ntb_qp_edma_is_rc(qp))
> + if (!ntb_edma_notify_peer(&qp->transport->edma, qp->qp_num))
> + return;
> +
> + /*
> + * Called from contexts that may be atomic. Since ntb_peer_db_set()
> + * may sleep, delegate the actual doorbell write to a workqueue.
> + */
> + queue_work(system_highpri_wq, &qp->db_work);
> +}
> +
> +static void ntb_transport_edma_isr(void *data, int qp_num)
> +{
> + struct ntb_transport_ctx *nt = data;
> + struct ntb_transport_qp *qp;
> +
> + if (qp_num < 0 || qp_num >= nt->qp_count)
> + return;
> +
> + qp = &nt->qp_vec[qp_num];
> + if (WARN_ON(!qp))
> + return;
> +
> + queue_work(nt->wq, &qp->read_work);
> + queue_work(nt->wq, &qp->write_work);
> +}
> +
> +static int ntb_transport_edma_rc_init(struct ntb_transport_ctx *nt)
> +{
> + struct ntb_dev *ndev = nt->ndev;
> + struct pci_dev *pdev = ndev->pdev;
> + int rc;
> +
> + if (!use_remote_edma || nt->remote_edma_mode != REMOTE_EDMA_UNKNOWN)
> + return 0;
> +
> + rc = ntb_edma_setup_peer(ndev);
> + if (rc) {
> + dev_err(&pdev->dev, "Failed to enable remote eDMA: %d\n", rc);
> + return rc;
> + }
> +
> + rc = ntb_edma_setup_chans(get_dma_dev(ndev), &nt->edma);
> + if (rc) {
> + dev_err(&pdev->dev, "Failed to setup eDMA channels: %d\n", rc);
> + return rc;
> + }
> +
> + nt->remote_edma_mode = REMOTE_EDMA_RC;
> + return 0;
> +}
> +
> +static int ntb_transport_edma_ep_init(struct ntb_transport_ctx *nt)
> +{
> + struct ntb_dev *ndev = nt->ndev;
> + struct pci_dev *pdev = ndev->pdev;
> + struct pci_epc *epc;
> + int rc;
> +
> + if (!use_remote_edma || nt->remote_edma_mode == REMOTE_EDMA_EP)
> + return 0;
> +
> + /* Only EP side can return pci_epc */
> + epc = ntb_get_pci_epc(ndev);
> + if (!epc)
> + return 0;
> +
> + rc = ntb_edma_setup_mws(ndev);
> + if (rc) {
> + dev_err(&pdev->dev,
> + "Failed to set up memory window for eDMA: %d\n", rc);
> + return rc;
> + }
> +
> + rc = ntb_edma_setup_isr(&pdev->dev, &epc->dev, ntb_transport_edma_isr, nt);
> + if (rc) {
> + dev_err(&pdev->dev, "Failed to setup eDMA ISR (%d)\n", rc);
> + return rc;
> + }
> +
> + nt->remote_edma_mode = REMOTE_EDMA_EP;
> + return 0;
> +}
> +
> +static int ntb_transport_edma_setup_qp_mw(struct ntb_transport_ctx *nt,
> + unsigned int qp_num)
> +{
> + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
> + struct ntb_dev *ndev = nt->ndev;
> + struct ntb_queue_entry *entry;
> + struct ntb_transport_mw *mw;
> + unsigned int mw_num, mw_count, qp_count;
> + unsigned int qp_offset, rx_info_offset;
> + unsigned int mw_size, mw_size_per_qp;
> + unsigned int num_qps_mw;
> + size_t edma_total;
> + unsigned int i;
> + int node;
> +
> + mw_count = nt->mw_count;
> + qp_count = nt->qp_count;
> +
> + mw_num = QP_TO_MW(nt, qp_num);
> + mw = &nt->mw_vec[mw_num];
> +
> + if (!mw->virt_addr)
> + return -ENOMEM;
> +
> + if (mw_num < qp_count % mw_count)
> + num_qps_mw = qp_count / mw_count + 1;
> + else
> + num_qps_mw = qp_count / mw_count;
> +
> + mw_size = min(nt->mw_vec[mw_num].phys_size, mw->xlat_size);
> + if (max_mw_size && mw_size > max_mw_size)
> + mw_size = max_mw_size;
> +
> + mw_size_per_qp = round_down((unsigned int)mw_size / num_qps_mw, SZ_64);
> + qp_offset = mw_size_per_qp * (qp_num / mw_count);
> + rx_info_offset = mw_size_per_qp - sizeof(struct ntb_rx_info);
> +
> + qp->tx_mw_size = mw_size_per_qp;
> + qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;
> + if (!qp->tx_mw)
> + return -EINVAL;
> + qp->tx_mw_phys = nt->mw_vec[mw_num].phys_addr + qp_offset;
> + if (!qp->tx_mw_phys)
> + return -EINVAL;
> + qp->rx_info = qp->tx_mw + rx_info_offset;
> + qp->rx_buff = mw->virt_addr + qp_offset;
> + qp->remote_rx_info = qp->rx_buff + rx_info_offset;
> +
> + /* Due to housekeeping, there must be at least 2 buffs */
> + qp->tx_max_frame = min(transport_mtu, mw_size_per_qp / 2);
> + qp->rx_max_frame = min(transport_mtu, mw_size_per_qp / 2);
> +
> + /* In eDMA mode, decouple from MW sizing and force ring-sized entries */
> + edma_total = 2 * sizeof(struct ntb_edma_ring);
> + if (rx_info_offset < edma_total) {
> + dev_err(&ndev->dev, "Ring space requires %luB (>=%uB)\n",
> + edma_total, rx_info_offset);
> + return -EINVAL;
> + }
> + qp->tx_max_entry = NTB_EDMA_RING_ENTRIES;
> + qp->rx_max_entry = NTB_EDMA_RING_ENTRIES;
> +
> + /*
> + * Checking to see if we have more entries than the default.
> + * We should add additional entries if that is the case so we
> + * can be in sync with the transport frames.
> + */
> + node = dev_to_node(&ndev->dev);
> + for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) {
> + entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
> + if (!entry)
> + return -ENOMEM;
> +
> + entry->qp = qp;
> + INIT_WORK(&entry->dma_work, ntb_transport_edma_rc_dma_work);
> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> + &qp->rx_free_q);
> + qp->rx_alloc_entry++;
> + }
> +
> + memset(qp->rx_buff, 0, edma_total);
> +
> + qp->rx_pkts = 0;
> + qp->tx_pkts = 0;
> +
> + return 0;
> +}
> +
> +static int ntb_transport_edma_ep_read_complete(struct ntb_transport_qp *qp)
> +{
> + struct device *dma_dev = get_dma_dev(qp->ndev);
> + struct ntb_queue_entry *entry;
> + struct ntb_edma_desc *in;
> + unsigned int len;
> + u32 idx;
> +
> + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_TAIL_RD_EP_I(qp)),
> + qp->rd_cons) == 0)
> + return 0;
> +
> + idx = ntb_edma_ring_idx(qp->rd_cons);
> + in = NTB_DESC_RD_EP_I(qp, idx);
> + if (!(in->flags & DESC_DONE_FLAG))
> + return 0;
> +
> + in->flags = 0;
> + len = in->len; /* might be smaller than entry->len */
> +
> + entry = (struct ntb_queue_entry *)(in->data);
> + if (WARN_ON(!entry))
> + return 0;
> +
> + if (in->flags & LINK_DOWN_FLAG) {
> + ntb_qp_link_down(qp);
> + qp->rd_cons++;
> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> + return 1;
> + }
> +
> + dma_unmap_single(dma_dev, entry->addr, entry->len, DMA_FROM_DEVICE);
> +
> + qp->rx_bytes += len;
> + qp->rx_pkts++;
> + qp->rd_cons++;
> +
> + if (qp->rx_handler && qp->client_ready)
> + qp->rx_handler(qp, qp->cb_data, entry->cb_data, len);
> +
> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> + return 1;
> +}
> +
> +static int ntb_transport_edma_ep_write_complete(struct ntb_transport_qp *qp)
> +{
> + struct ntb_queue_entry *entry;
> + struct ntb_edma_desc *in;
> + u32 idx;
> +
> + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_TAIL_WR_EP_I(qp)),
> + qp->wr_cons) == 0)
> + return 0;
> +
> + idx = ntb_edma_ring_idx(qp->wr_cons);
> + in = NTB_DESC_WR_EP_I(qp, idx);
> +
> + entry = (struct ntb_queue_entry *)(in->data);
> + if (WARN_ON(!entry))
> + return 0;
> +
> + qp->wr_cons++;
> +
> + if (qp->tx_handler)
> + qp->tx_handler(qp, qp->cb_data, entry->cb_data, entry->len);
> +
> + ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q);
> + return 1;
> +}
> +
> +static void ntb_transport_edma_ep_read_work(struct work_struct *work)
> +{
> + struct ntb_transport_qp *qp = container_of(
> + work, struct ntb_transport_qp, read_work);
> + unsigned int i;
> +
> + for (i = 0; i < NTB_EDMA_MAX_POLL; i++) {
> + if (!ntb_transport_edma_ep_read_complete(qp))
> + break;
> + }
> +
> + if (ntb_transport_edma_ep_read_complete(qp))
> + queue_work(qp->transport->wq, &qp->read_work);
> +}
> +
> +static void ntb_transport_edma_ep_write_work(struct work_struct *work)
> +{
> + struct ntb_transport_qp *qp = container_of(
> + work, struct ntb_transport_qp, write_work);
> + unsigned int i;
> +
> + for (i = 0; i < NTB_EDMA_MAX_POLL; i++) {
> + if (!ntb_transport_edma_ep_write_complete(qp))
> + break;
> + }
> +
> + if (ntb_transport_edma_ep_write_complete(qp))
> + queue_work(qp->transport->wq, &qp->write_work);
> +}
> +
> +static void ntb_transport_edma_rc_write_complete_work(struct work_struct *work)
> +{
> + struct ntb_transport_qp *qp = container_of(
> + work, struct ntb_transport_qp, write_work);
> + struct ntb_queue_entry *entry;
> + struct ntb_edma_desc *in;
> + unsigned int len;
> + void *cb_data;
> + u32 idx;
> +
> + while (ntb_edma_ring_used_entry(READ_ONCE(qp->wr_issue),
> + qp->wr_cons) != 0) {
> + /* Paired with smp_wmb() in ntb_transport_edma_rc_poll() */
> + smp_rmb();
> +
> + idx = ntb_edma_ring_idx(qp->wr_cons);
> + in = NTB_DESC_WR_RC_I(qp, idx);
> + entry = (struct ntb_queue_entry *)READ_ONCE(in->data);
> + if (!entry || !(entry->flags & DESC_DONE_FLAG))
> + break;
> +
> + in->data = 0;
> +
> + cb_data = entry->cb_data;
> + len = entry->len;
> +
> + iowrite32(++qp->wr_cons, NTB_TAIL_WR_RC_O(qp));
> +
> + if (unlikely(entry->flags & LINK_DOWN_FLAG)) {
> + ntb_qp_link_down(qp);
> + continue;
> + }
> +
> + ntb_transport_edma_notify_peer(qp);
> +
> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> +
> + if (qp->rx_handler && qp->client_ready)
> + qp->rx_handler(qp, qp->cb_data, cb_data, len);
> +
> + /* stat updates */
> + qp->rx_bytes += len;
> + qp->rx_pkts++;
> + }
> +}
> +
> +static void ntb_transport_edma_rc_write_cb(void *data,
> + const struct dmaengine_result *res)
> +{
> + struct ntb_queue_entry *entry = data;
> + struct ntb_transport_qp *qp = entry->qp;
> + struct ntb_transport_ctx *nt = qp->transport;
> + enum dmaengine_tx_result dma_err = res->result;
> + struct device *dma_dev = get_dma_dev(qp->ndev);
> +
> + switch (dma_err) {
> + case DMA_TRANS_READ_FAILED:
> + case DMA_TRANS_WRITE_FAILED:
> + case DMA_TRANS_ABORTED:
> + entry->errors++;
> + entry->len = -EIO;
> + break;
> + case DMA_TRANS_NOERROR:
> + default:
> + break;
> + }
> + dma_unmap_sg(dma_dev, &entry->sgl, 1, DMA_FROM_DEVICE);
> + sg_dma_address(&entry->sgl) = 0;
> +
> + entry->flags |= DESC_DONE_FLAG;
> +
> + queue_work(nt->wq, &qp->write_work);
> +}
> +
> +static void ntb_transport_edma_rc_read_complete_work(struct work_struct *work)
> +{
> + struct ntb_transport_qp *qp = container_of(
> + work, struct ntb_transport_qp, read_work);
> + struct ntb_edma_desc *in, __iomem *out;
> + struct ntb_queue_entry *entry;
> + unsigned int len;
> + void *cb_data;
> + u32 idx;
> +
> + while (ntb_edma_ring_used_entry(READ_ONCE(qp->rd_issue),
> + qp->rd_cons) != 0) {
> + /* Paired with smp_wmb() in ntb_transport_edma_rc_tx_enqueue() */
> + smp_rmb();
> +
> + idx = ntb_edma_ring_idx(qp->rd_cons);
> + in = NTB_DESC_RD_RC_I(qp, idx);
> + entry = (struct ntb_queue_entry *)in->data;
> + if (!entry || !(entry->flags & DESC_DONE_FLAG))
> + break;
> +
> + in->data = 0;
> +
> + cb_data = entry->cb_data;
> + len = entry->len;
> +
> + out = NTB_DESC_RD_RC_O(qp, idx);
> +
> + WRITE_ONCE(qp->rd_cons, qp->rd_cons + 1);
> +
> + /*
> + * No need to add barrier in-between to enforce ordering here.
> + * The other side proceeds only after both flags and tail are
> + * updated.
> + */
> + iowrite32(entry->flags, &out->flags);
> + iowrite32(qp->rd_cons, NTB_TAIL_RD_RC_O(qp));
> +
> + ntb_transport_edma_notify_peer(qp);
> +
> + ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
> + &qp->tx_free_q);
> +
> + if (qp->tx_handler)
> + qp->tx_handler(qp, qp->cb_data, cb_data, len);
> +
> + /* stat updates */
> + qp->tx_bytes += len;
> + qp->tx_pkts++;
> + }
> +}
> +
> +static void ntb_transport_edma_rc_read_cb(void *data,
> + const struct dmaengine_result *res)
> +{
> + struct ntb_queue_entry *entry = data;
> + struct ntb_transport_qp *qp = entry->qp;
> + struct ntb_transport_ctx *nt = qp->transport;
> + struct device *dma_dev = get_dma_dev(qp->ndev);
> + enum dmaengine_tx_result dma_err = res->result;
> +
> + switch (dma_err) {
> + case DMA_TRANS_READ_FAILED:
> + case DMA_TRANS_WRITE_FAILED:
> + case DMA_TRANS_ABORTED:
> + entry->errors++;
> + entry->len = -EIO;
> + break;
> + case DMA_TRANS_NOERROR:
> + default:
> + break;
> + }
> + dma_unmap_sg(dma_dev, &entry->sgl, 1, DMA_TO_DEVICE);
> + sg_dma_address(&entry->sgl) = 0;
> +
> + entry->flags |= DESC_DONE_FLAG;
> +
> + queue_work(nt->wq, &qp->read_work);
> +}
> +
> +static int ntb_transport_edma_rc_write_start(struct device *d,
> + struct dma_chan *chan, size_t len,
> + dma_addr_t ep_src, void *rc_dst,
> + struct ntb_queue_entry *entry)
> +{
> + struct scatterlist *sgl = &entry->sgl;
> + struct dma_async_tx_descriptor *txd;
> + struct dma_slave_config cfg;
> + dma_cookie_t cookie;
> + int nents, rc;
> +
> + if (!d)
> + return -ENODEV;
> +
> + if (!chan)
> + return -ENXIO;
> +
> + if (WARN_ON(!ep_src || !rc_dst))
> + return -EINVAL;
> +
> + if (WARN_ON(sg_dma_address(sgl)))
> + return -EINVAL;
> +
> + sg_init_one(sgl, rc_dst, len);
> + nents = dma_map_sg(d, sgl, 1, DMA_FROM_DEVICE);
> + if (nents <= 0)
> + return -EIO;
> +
> + memset(&cfg, 0, sizeof(cfg));
> + cfg.src_addr = ep_src;
> + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> + cfg.direction = DMA_DEV_TO_MEM;
> + rc = dmaengine_slave_config(chan, &cfg);
> + if (rc)
> + goto out_unmap;
> +
> + txd = dmaengine_prep_slave_sg(chan, sgl, 1, DMA_DEV_TO_MEM,
> + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> + if (!txd) {
> + rc = -EIO;
> + goto out_unmap;
> + }
> +
> + txd->callback_result = ntb_transport_edma_rc_write_cb;
> + txd->callback_param = entry;
> +
> + cookie = dmaengine_submit(txd);
> + if (dma_submit_error(cookie)) {
> + rc = -EIO;
> + goto out_unmap;
> + }
> + dma_async_issue_pending(chan);
> + return 0;
> +out_unmap:
> + dma_unmap_sg(d, sgl, 1, DMA_FROM_DEVICE);
> + return rc;
> +}
> +
> +static int ntb_transport_edma_rc_read_start(struct device *d,
> + struct dma_chan *chan, size_t len,
> + void *rc_src, dma_addr_t ep_dst,
> + struct ntb_queue_entry *entry)
> +{
> + struct scatterlist *sgl = &entry->sgl;
> + struct dma_async_tx_descriptor *txd;
> + struct dma_slave_config cfg;
> + dma_cookie_t cookie;
> + int nents, rc;
> +
> + if (!d)
> + return -ENODEV;
> +
> + if (!chan)
> + return -ENXIO;
> +
> + if (WARN_ON(!rc_src || !ep_dst))
> + return -EINVAL;
> +
> + if (WARN_ON(sg_dma_address(sgl)))
> + return -EINVAL;
> +
> + sg_init_one(sgl, rc_src, len);
> + nents = dma_map_sg(d, sgl, 1, DMA_TO_DEVICE);
> + if (nents <= 0)
> + return -EIO;
> +
> + memset(&cfg, 0, sizeof(cfg));
> + cfg.dst_addr = ep_dst;
> + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> + cfg.direction = DMA_MEM_TO_DEV;
> + rc = dmaengine_slave_config(chan, &cfg);
> + if (rc)
> + goto out_unmap;
> +
> + txd = dmaengine_prep_slave_sg(chan, sgl, 1, DMA_MEM_TO_DEV,
> + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> + if (!txd) {
> + rc = -EIO;
> + goto out_unmap;
> + }
> +
> + txd->callback_result = ntb_transport_edma_rc_read_cb;
> + txd->callback_param = entry;
> +
> + cookie = dmaengine_submit(txd);
> + if (dma_submit_error(cookie)) {
> + rc = -EIO;
> + goto out_unmap;
> + }
> + dma_async_issue_pending(chan);
> + return 0;
> +out_unmap:
> + dma_unmap_sg(d, sgl, 1, DMA_TO_DEVICE);
> + return rc;
> +}
> +
> +static void ntb_transport_edma_rc_dma_work(struct work_struct *work)
> +{
> + struct ntb_queue_entry *entry = container_of(
> + work, struct ntb_queue_entry, dma_work);
> + struct ntb_transport_qp *qp = entry->qp;
> + struct ntb_transport_ctx *nt = qp->transport;
> + struct device *dma_dev = get_dma_dev(qp->ndev);
> + struct dma_chan *chan;
> + int rc;
> +
> + chan = ntb_edma_pick_chan(&nt->edma, REMOTE_EDMA_WRITE);
> + rc = ntb_transport_edma_rc_write_start(dma_dev, chan, entry->len,
> + entry->addr, entry->buf, entry);
> + if (rc) {
> + entry->errors++;
> + entry->len = -EIO;
> + entry->flags |= DESC_DONE_FLAG;
> + queue_work(nt->wq, &qp->write_work);
> + return;
> + }
> +}
> +
> +static void ntb_transport_edma_rc_poll(struct ntb_transport_qp *qp)
> +{
> + struct ntb_transport_ctx *nt = qp->transport;
> + unsigned int budget = NTB_EDMA_MAX_POLL;
> + struct ntb_queue_entry *entry;
> + struct ntb_edma_desc *in;
> + dma_addr_t ep_src;
> + u32 len, idx;
> +
> + while (budget--) {
> + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_HEAD_WR_RC_I(qp)),
> + qp->wr_issue) == 0)
> + break;
> +
> + idx = ntb_edma_ring_idx(qp->wr_issue);
> + in = NTB_DESC_WR_RC_I(qp, idx);
> +
> + len = READ_ONCE(in->len);
> + ep_src = (dma_addr_t)READ_ONCE(in->addr);
> +
> + /* Prepare 'entry' for write completion */
> + entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q);
> + if (!entry) {
> + qp->rx_err_no_buf++;
> + break;
> + }
> + if (WARN_ON(entry->flags & DESC_DONE_FLAG))
> + entry->flags &= ~DESC_DONE_FLAG;
> + entry->len = len; /* NB. entry->len can be <=0 */
> + entry->addr = ep_src;
> +
> + /*
> + * ntb_transport_edma_rc_write_complete_work() checks entry->flags
> + * so it needs to be set before wr_issue++.
> + */
> + in->data = (uintptr_t)entry;
> +
> + /* Ensure in->data visible before wr_issue++ */
> + smp_wmb();
> +
> + WRITE_ONCE(qp->wr_issue, qp->wr_issue + 1);
> +
> + if (!len) {
> + entry->flags |= DESC_DONE_FLAG;
> + queue_work(nt->wq, &qp->write_work);
> + continue;
> + }
> +
> + if (in->flags & LINK_DOWN_FLAG) {
> + dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n");
> + entry->flags |= DESC_DONE_FLAG | LINK_DOWN_FLAG;
> + queue_work(nt->wq, &qp->write_work);
> + continue;
> + }
> +
> + queue_work(nt->wq, &entry->dma_work);
> + }
> +
> + if (!budget)
> + tasklet_schedule(&qp->rxc_db_work);
> +}
> +
> +static int ntb_transport_edma_rc_tx_enqueue(struct ntb_transport_qp *qp,
> + struct ntb_queue_entry *entry)
> +{
> + struct device *dma_dev = get_dma_dev(qp->ndev);
> + struct ntb_transport_ctx *nt = qp->transport;
> + struct ntb_edma_desc *in, __iomem *out;
> + unsigned int len = entry->len;
> + struct dma_chan *chan;
> + u32 issue, idx, head;
> + dma_addr_t ep_dst;
> + int rc;
> +
> + WARN_ON_ONCE(entry->flags & DESC_DONE_FLAG);
> +
> + scoped_guard(spinlock_irqsave, &qp->rc_lock) {
> + head = READ_ONCE(*NTB_HEAD_RD_RC_I(qp));
> + issue = qp->rd_issue;
> + if (ntb_edma_ring_used_entry(head, issue) == 0) {
> + qp->tx_ring_full++;
> + return -ENOSPC;
> + }
> +
> + /*
> + * ntb_transport_edma_rc_read_complete_work() checks entry->flags
> + * so it needs to be set before rd_issue++.
> + */
> + idx = ntb_edma_ring_idx(issue);
> + in = NTB_DESC_RD_RC_I(qp, idx);
> + in->data = (uintptr_t)entry;
> +
> + /* Make in->data visible before rd_issue++ */
> + smp_wmb();
> +
> + WRITE_ONCE(qp->rd_issue, qp->rd_issue + 1);
> + }
> +
> + /* Publish the final transfer length to the EP side */
> + out = NTB_DESC_RD_RC_O(qp, idx);
> + iowrite32(len, &out->len);
> + ioread32(&out->len);
> +
> + if (unlikely(!len)) {
> + entry->flags |= DESC_DONE_FLAG;
> + queue_work(nt->wq, &qp->read_work);
> + return 0;
> + }
> +
> + /* Paired with dma_wmb() in ntb_transport_edma_ep_rx_enqueue() */
> + dma_rmb();
> +
> + /* kick remote eDMA read transfer */
> + ep_dst = (dma_addr_t)in->addr;
> + chan = ntb_edma_pick_chan(&nt->edma, REMOTE_EDMA_READ);
> + rc = ntb_transport_edma_rc_read_start(dma_dev, chan, len,
> + entry->buf, ep_dst, entry);
> + if (rc) {
> + entry->errors++;
> + entry->len = -EIO;
> + entry->flags |= DESC_DONE_FLAG;
> + queue_work(nt->wq, &qp->read_work);
> + }
> + return 0;
> +}
> +
> +static int ntb_transport_edma_ep_tx_enqueue(struct ntb_transport_qp *qp,
> + struct ntb_queue_entry *entry)
> +{
> + struct device *dma_dev = get_dma_dev(qp->ndev);
> + struct ntb_edma_desc *in, __iomem *out;
> + unsigned int len = entry->len;
> + dma_addr_t ep_src = 0;
> + u32 idx;
> + int rc;
> +
> + if (likely(len)) {
> + ep_src = dma_map_single(dma_dev, entry->buf, len,
> + DMA_TO_DEVICE);
> + rc = dma_mapping_error(dma_dev, ep_src);
> + if (rc)
> + return rc;
> + }
> +
> + scoped_guard(spinlock_irqsave, &qp->ep_tx_lock) {
> + if (ntb_edma_ring_full(qp->wr_prod, qp->wr_cons)) {
> + rc = -ENOSPC;
> + qp->tx_ring_full++;
> + goto out_unmap;
> + }
> +
> + idx = ntb_edma_ring_idx(qp->wr_prod);
> + in = NTB_DESC_WR_EP_I(qp, idx);
> + out = NTB_DESC_WR_EP_O(qp, idx);
> +
> + WARN_ON(in->flags & DESC_DONE_FLAG);
> + WARN_ON(entry->flags & DESC_DONE_FLAG);
> + in->flags = 0;
> + in->data = (uintptr_t)entry;
> + entry->addr = ep_src;
> +
> + iowrite32(len, &out->len);
> + iowrite32(entry->flags, &out->flags);
> + iowrite64(ep_src, &out->addr);
> + WRITE_ONCE(qp->wr_prod, qp->wr_prod + 1);
> +
> + dma_wmb();
> + iowrite32(qp->wr_prod, NTB_HEAD_WR_EP_O(qp));
> +
> + qp->tx_bytes += len;
> + qp->tx_pkts++;
> + }
> +
> + ntb_transport_edma_notify_peer(qp);
> +
> + return 0;
> +out_unmap:
> + if (likely(len))
> + dma_unmap_single(dma_dev, ep_src, len, DMA_TO_DEVICE);
> + return rc;
> +}
> +
> +static int ntb_transport_edma_tx_enqueue(struct ntb_transport_qp *qp,
> + struct ntb_queue_entry *entry,
> + void *cb, void *data, unsigned int len,
> + unsigned int flags)
> +{
> + struct device *dma_dev;
> +
> + if (entry->addr) {
> + /* Deferred unmap */
> + dma_dev = get_dma_dev(qp->ndev);
> + dma_unmap_single(dma_dev, entry->addr, entry->len, DMA_TO_DEVICE);
> + }
> +
> + entry->cb_data = cb;
> + entry->buf = data;
> + entry->len = len;
> + entry->flags = flags;
> + entry->errors = 0;
> + entry->addr = 0;
> +
> + WARN_ON_ONCE(!ntb_qp_edma_enabled(qp));
> +
> + if (ntb_qp_edma_is_ep(qp))
> + return ntb_transport_edma_ep_tx_enqueue(qp, entry);
> + else
> + return ntb_transport_edma_rc_tx_enqueue(qp, entry);
> +}
> +
> +static int ntb_transport_edma_ep_rx_enqueue(struct ntb_transport_qp *qp,
> + struct ntb_queue_entry *entry)
> +{
> + struct device *dma_dev = get_dma_dev(qp->ndev);
> + struct ntb_edma_desc *in, __iomem *out;
> + unsigned int len = entry->len;
> + void *data = entry->buf;
> + dma_addr_t ep_dst;
> + u32 idx;
> + int rc;
> +
> + ep_dst = dma_map_single(dma_dev, data, len, DMA_FROM_DEVICE);
> + rc = dma_mapping_error(dma_dev, ep_dst);
> + if (rc)
> + return rc;
> +
> + scoped_guard(spinlock_bh, &qp->ep_rx_lock) {
> + if (ntb_edma_ring_full(READ_ONCE(qp->rd_prod),
> + READ_ONCE(qp->rd_cons))) {
> + rc = -ENOSPC;
> + goto out_unmap;
> + }
> +
> + idx = ntb_edma_ring_idx(qp->rd_prod);
> + in = NTB_DESC_RD_EP_I(qp, idx);
> + out = NTB_DESC_RD_EP_O(qp, idx);
> +
> + iowrite32(len, &out->len);
> + iowrite64(ep_dst, &out->addr);
> +
> + WARN_ON(in->flags & DESC_DONE_FLAG);
> + in->data = (uintptr_t)entry;
> + entry->addr = ep_dst;
> +
> + /* Ensure len/addr are visible before the head update */
> + dma_wmb();
> +
> + WRITE_ONCE(qp->rd_prod, qp->rd_prod + 1);
> + iowrite32(qp->rd_prod, NTB_HEAD_RD_EP_O(qp));
> + }
> + return 0;
> +out_unmap:
> + dma_unmap_single(dma_dev, ep_dst, len, DMA_FROM_DEVICE);
> + return rc;
> +}
> +
> +static int ntb_transport_edma_rx_enqueue(struct ntb_transport_qp *qp,
> + struct ntb_queue_entry *entry)
> +{
> + int rc;
> +
> + /* The behaviour is the same as the default backend for RC side */
> + if (ntb_qp_edma_is_ep(qp)) {
> + rc = ntb_transport_edma_ep_rx_enqueue(qp, entry);
> + if (rc) {
> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> + &qp->rx_free_q);
> + return rc;
> + }
> + }
> +
> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q);
> +
> + if (qp->active)
> + tasklet_schedule(&qp->rxc_db_work);
> +
> + return 0;
> +}
> +
> +static void ntb_transport_edma_rx_poll(struct ntb_transport_qp *qp)
> +{
> + struct ntb_transport_ctx *nt = qp->transport;
> +
> + if (ntb_qp_edma_is_rc(qp))
> + ntb_transport_edma_rc_poll(qp);
> + else if (ntb_qp_edma_is_ep(qp)) {
> + /*
> + * Make sure we poll the rings even if an eDMA interrupt is
> + * cleared on the RC side earlier.
> + */
> + queue_work(nt->wq, &qp->read_work);
> + queue_work(nt->wq, &qp->write_work);
> + } else
> + /* Unreachable */
> + WARN_ON_ONCE(1);
> +}
> +
> +static void ntb_transport_edma_read_work(struct work_struct *work)
> +{
> + struct ntb_transport_qp *qp = container_of(
> + work, struct ntb_transport_qp, read_work);
> +
> + if (ntb_qp_edma_is_rc(qp))
> + ntb_transport_edma_rc_read_complete_work(work);
> + else if (ntb_qp_edma_is_ep(qp))
> + ntb_transport_edma_ep_read_work(work);
> + else
> + /* Unreachable */
> + WARN_ON_ONCE(1);
> +}
> +
> +static void ntb_transport_edma_write_work(struct work_struct *work)
> +{
> + struct ntb_transport_qp *qp = container_of(
> + work, struct ntb_transport_qp, write_work);
> +
> + if (ntb_qp_edma_is_rc(qp))
> + ntb_transport_edma_rc_write_complete_work(work);
> + else if (ntb_qp_edma_is_ep(qp))
> + ntb_transport_edma_ep_write_work(work);
> + else
> + /* Unreachable */
> + WARN_ON_ONCE(1);
> +}
> +
> +static void ntb_transport_edma_init_queue(struct ntb_transport_ctx *nt,
> + unsigned int qp_num)
> +{
> + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
> +
> + qp->wr_cons = 0;
> + qp->rd_cons = 0;
> + qp->wr_prod = 0;
> + qp->rd_prod = 0;
> + qp->wr_issue = 0;
> + qp->rd_issue = 0;
> +
> + INIT_WORK(&qp->db_work, ntb_transport_edma_db_work);
> + INIT_WORK(&qp->read_work, ntb_transport_edma_read_work);
> + INIT_WORK(&qp->write_work, ntb_transport_edma_write_work);
> +}
> +
> +static void ntb_transport_edma_create_queue(struct ntb_transport_ctx *nt,
> + struct ntb_transport_qp *qp)
> +{
> + spin_lock_init(&qp->ep_tx_lock);
> + spin_lock_init(&qp->ep_rx_lock);
> + spin_lock_init(&qp->rc_lock);
> +}
> +
> +static const struct ntb_transport_backend_ops edma_backend_ops = {
> + .setup_qp_mw = ntb_transport_edma_setup_qp_mw,
> + .tx_free_entry = ntb_transport_edma_tx_free_entry,
> + .tx_enqueue = ntb_transport_edma_tx_enqueue,
> + .rx_enqueue = ntb_transport_edma_rx_enqueue,
> + .rx_poll = ntb_transport_edma_rx_poll,
> + .debugfs_stats_show = ntb_transport_edma_debugfs_stats_show,
> +};
> +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> +
> /**
> * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
> * @qp: NTB transport layer queue to be enabled
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-12-01 21:46 ` Dave Jiang
@ 2025-12-02 6:59 ` Koichiro Den
2025-12-02 14:53 ` Dave Jiang
0 siblings, 1 reply; 19+ messages in thread
From: Koichiro Den @ 2025-12-02 6:59 UTC (permalink / raw)
To: Dave Jiang
Cc: ntb, linux-pci, dmaengine, linux-kernel, Frank.Li, mani,
kwilczynski, kishon, bhelgaas, corbet, vkoul, jdmason, allenbh,
Basavaraj.Natikar, Shyam-sundar.S-k, kurt.schwemmer, logang,
jingoohan1, lpieralisi, robh, jbrunet, fancer.lancer, arnd,
pstanner, elfring
On Mon, Dec 01, 2025 at 02:46:41PM -0700, Dave Jiang wrote:
>
>
> On 11/29/25 9:03 AM, Koichiro Den wrote:
> > Add a new transport backend that uses a remote DesignWare eDMA engine
> > located on the NTB endpoint to move data between host and endpoint.
> >
> > In this mode:
> >
> > - The endpoint exposes a dedicated memory window that contains the
> > eDMA register block followed by a small control structure (struct
> > ntb_edma_info) and per-channel linked-list (LL) rings.
> >
> > - On the endpoint side, ntb_edma_setup_mws() allocates the control
> > structure and LL rings in endpoint memory, then programs an inbound
> > iATU region so that the host can access them via a peer MW.
> >
> > - On the host side, ntb_edma_setup_peer() ioremaps the peer MW, reads
> > ntb_edma_info and configures a dw-edma DMA device to use the LL
> > rings provided by the endpoint.
> >
> > - ntb_transport is extended with a new backend_ops implementation that
> > routes TX and RX enqueue/poll operations through the remote eDMA
> > rings while keeping the existing shared-memory backend intact.
> >
> > - The host signals the endpoint via a dedicated DMA read channel.
> > 'use_msi' module option is ignored when 'use_remote_edma=1'.
> >
> > The new mode is guarded by a Kconfig option (NTB_TRANSPORT_EDMA) and a
> > module parameter (use_remote_edma). When disabled, the existing
> > ntb_transport behaviour is unchanged.
> >
> > Signed-off-by: Koichiro Den <den@valinux.co.jp>
> > ---
> > drivers/ntb/Kconfig | 11 +
> > drivers/ntb/Makefile | 3 +
> > drivers/ntb/ntb_edma.c | 628 ++++++++
> > drivers/ntb/ntb_edma.h | 128 ++
>
> I briefly looked over the code. It feels like the EDMA bits should go in drivers/ntb/hw/ rather than drivers/ntb/ given it's pretty specific to the designware hardware. What sits in drivers/ntb should be generic APIs where a different vendor can utilize it and not having to adopt to designware hardware specifics. So maybe a bit more abstractions are needed?
That makes sense, I'll reorganize things. Thank you for the suggestion.
>
> > .../{ntb_transport.c => ntb_transport_core.c} | 1281 ++++++++++++++++-
> > 5 files changed, 2048 insertions(+), 3 deletions(-)
> > create mode 100644 drivers/ntb/ntb_edma.c
> > create mode 100644 drivers/ntb/ntb_edma.h
> > rename drivers/ntb/{ntb_transport.c => ntb_transport_core.c} (65%)
> >
> > diff --git a/drivers/ntb/Kconfig b/drivers/ntb/Kconfig
> > index df16c755b4da..db63f02bb116 100644
> > --- a/drivers/ntb/Kconfig
> > +++ b/drivers/ntb/Kconfig
> > @@ -37,4 +37,15 @@ config NTB_TRANSPORT
> >
> > If unsure, say N.
> >
> > +config NTB_TRANSPORT_EDMA
> > + bool "NTB Transport backed by remote eDMA"
> > + depends on NTB_TRANSPORT
> > + depends on PCI
> > + select DMA_ENGINE
> > + help
> > + Enable a transport backend that uses a remote DesignWare eDMA engine
> > + exposed through a dedicated NTB memory window. The host uses the
> > + endpoint's eDMA engine to move data in both directions.
> > + Say Y here if you intend to use the 'use_remote_edma' module parameter.
> > +
> > endif # NTB
> > diff --git a/drivers/ntb/Makefile b/drivers/ntb/Makefile
> > index 3a6fa181ff99..51f0e1e3aec7 100644
> > --- a/drivers/ntb/Makefile
> > +++ b/drivers/ntb/Makefile
> > @@ -4,3 +4,6 @@ obj-$(CONFIG_NTB_TRANSPORT) += ntb_transport.o
> >
> > ntb-y := core.o
> > ntb-$(CONFIG_NTB_MSI) += msi.o
> > +
> > +ntb_transport-y := ntb_transport_core.o
> > +ntb_transport-$(CONFIG_NTB_TRANSPORT_EDMA) += ntb_edma.o
> > diff --git a/drivers/ntb/ntb_edma.c b/drivers/ntb/ntb_edma.c
> > new file mode 100644
> > index 000000000000..cb35e0d56aa8
> > --- /dev/null
> > +++ b/drivers/ntb/ntb_edma.c
> > @@ -0,0 +1,628 @@
> > +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
> > +
> > +#include <linux/module.h>
> > +#include <linux/device.h>
> > +#include <linux/pci.h>
> > +#include <linux/ntb.h>
> > +#include <linux/io.h>
> > +#include <linux/iommu.h>
> > +#include <linux/dmaengine.h>
> > +#include <linux/pci-epc.h>
> > +#include <linux/dma/edma.h>
> > +#include <linux/irq.h>
> > +#include <linux/irqdomain.h>
> > +#include <linux/of.h>
> > +#include <linux/of_irq.h>
> > +#include <dt-bindings/interrupt-controller/arm-gic.h>
> > +
> > +#include "ntb_edma.h"
> > +
> > +/*
> > + * The interrupt register offsets below are taken from the DesignWare
> > + * eDMA "unrolled" register map (EDMA_MF_EDMA_UNROLL). The remote eDMA
> > + * backend currently only supports this layout.
> > + */
> > +#define DMA_WRITE_INT_STATUS_OFF 0x4c
> > +#define DMA_WRITE_INT_MASK_OFF 0x54
> > +#define DMA_WRITE_INT_CLEAR_OFF 0x58
> > +#define DMA_READ_INT_STATUS_OFF 0xa0
> > +#define DMA_READ_INT_MASK_OFF 0xa8
> > +#define DMA_READ_INT_CLEAR_OFF 0xac
> > +
> > +#define NTB_EDMA_NOTIFY_MAX_QP 64
> > +
> > +static unsigned int edma_spi = 417; /* 0x1a1 */
> > +module_param(edma_spi, uint, 0644);
> > +MODULE_PARM_DESC(edma_spi, "SPI number used by remote eDMA interrupt (EP local)");
> > +
> > +static u64 edma_regs_phys = 0xe65d5000;
> > +module_param(edma_regs_phys, ullong, 0644);
> > +MODULE_PARM_DESC(edma_regs_phys, "Physical base address of local eDMA registers (EP)");
> > +
> > +static unsigned long edma_regs_size = 0x1200;
> > +module_param(edma_regs_size, ulong, 0644);
> > +MODULE_PARM_DESC(edma_regs_size, "Size of the local eDMA register space (EP)");
> > +
> > +struct ntb_edma_intr {
> > + u32 db[NTB_EDMA_NOTIFY_MAX_QP];
> > +};
> > +
> > +struct ntb_edma_ctx {
> > + void *ll_wr_virt[EDMA_WR_CH_NUM];
> > + dma_addr_t ll_wr_phys[EDMA_WR_CH_NUM];
> > + void *ll_rd_virt[EDMA_RD_CH_NUM + 1];
> > + dma_addr_t ll_rd_phys[EDMA_RD_CH_NUM + 1];
> > +
> > + struct ntb_edma_intr *intr_ep_virt;
> > + dma_addr_t intr_ep_phys;
> > + struct ntb_edma_intr *intr_rc_virt;
> > + dma_addr_t intr_rc_phys;
> > + u32 notify_qp_max;
> > +
> > + bool initialized;
> > +};
> > +
> > +static struct ntb_edma_ctx edma_ctx;
> > +
> > +typedef void (*ntb_edma_interrupt_cb_t)(void *data, int qp_num);
> > +
> > +struct ntb_edma_interrupt {
> > + int virq;
> > + void __iomem *base;
> > + ntb_edma_interrupt_cb_t cb;
> > + void *data;
> > +};
> > +
> > +static struct ntb_edma_interrupt ntb_edma_intr;
> > +
> > +static int ntb_edma_map_spi_to_virq(struct device *dev, unsigned int spi)
> > +{
> > + struct device_node *np = dev_of_node(dev);
> > + struct device_node *parent;
> > + struct irq_fwspec fwspec = { 0 };
> > + int virq;
> > +
> > + parent = of_irq_find_parent(np);
> > + if (!parent)
> > + return -ENODEV;
> > +
> > + fwspec.fwnode = of_fwnode_handle(parent);
> > + fwspec.param_count = 3;
> > + fwspec.param[0] = GIC_SPI;
> > + fwspec.param[1] = spi;
> > + fwspec.param[2] = IRQ_TYPE_LEVEL_HIGH;
> > +
> > + virq = irq_create_fwspec_mapping(&fwspec);
> > + of_node_put(parent);
> > + return (virq > 0) ? virq : -EINVAL;
> > +}
> > +
> > +static irqreturn_t ntb_edma_isr(int irq, void *data)
> > +{
> > + struct ntb_edma_interrupt *v = data;
> > + u32 mask = BIT(EDMA_RD_CH_NUM);
> > + u32 i, val;
> > +
> > + /*
> > + * We do not ack interrupts here but instead we mask all local interrupt
> > + * sources except the read channel used for notification. This reduces
> > + * needless ISR invocations.
> > + *
> > + * In theory we could configure LIE=1/RIE=0 only for the notification
> > + * transfer (keeping all other channels at LIE=1/RIE=1), but that would
> > + * require intrusive changes to the dw-edma core.
> > + *
> > + * Note: The host side may have already cleared the read interrupt used
> > + * for notification, so reading DMA_READ_INT_CLEAR_OFF is not a reliable
> > + * way to detect it. As a result, we cannot reliably tell which specific
> > + * channel triggered this interrupt. intr_ep_virt->db[i] teaches us
> > + * instead.
> > + */
> > + iowrite32(~0x0, v->base + DMA_WRITE_INT_MASK_OFF);
> > + iowrite32(~mask, v->base + DMA_READ_INT_MASK_OFF);
> > +
> > + if (!v->cb || !edma_ctx.intr_ep_virt)
> > + return IRQ_HANDLED;
> > +
> > + for (i = 0; i < edma_ctx.notify_qp_max; i++) {
> > + val = READ_ONCE(edma_ctx.intr_ep_virt->db[i]);
> > + if (!val)
> > + continue;
> > +
> > + WRITE_ONCE(edma_ctx.intr_ep_virt->db[i], 0);
> > + v->cb(v->data, i);
> > + }
> > +
> > + return IRQ_HANDLED;
> > +}
> > +
> > +int ntb_edma_setup_isr(struct device *dev, struct device *epc_dev,
> > + ntb_edma_interrupt_cb_t cb, void *data)
> > +{
> > + struct ntb_edma_interrupt *v = &ntb_edma_intr;
> > + int virq = ntb_edma_map_spi_to_virq(epc_dev->parent, edma_spi);
> > + int ret;
> > +
> > + if (virq < 0) {
> > + dev_err(dev, "failed to get virq (%d)\n", virq);
> > + return virq;
> > + }
> > +
> > + v->virq = virq;
> > + v->cb = cb;
> > + v->data = data;
> > + if (edma_regs_phys && !v->base)
> > + v->base = devm_ioremap(dev, edma_regs_phys, edma_regs_size);
> > + if (!v->base) {
> > + dev_err(dev, "failed to setup v->base\n");
> > + return -1;
> > + }
> > + ret = devm_request_irq(dev, v->virq, ntb_edma_isr, 0, "ntb-edma", v);
> > + if (ret)
> > + return ret;
> > +
> > + if (v->base) {
> > + iowrite32(0x0, v->base + DMA_WRITE_INT_MASK_OFF);
> > + iowrite32(0x0, v->base + DMA_READ_INT_MASK_OFF);
> > + }
> > + return 0;
> > +}
> > +
> > +void ntb_edma_teardown_isr(struct device *dev)
> > +{
> > + struct ntb_edma_interrupt *v = &ntb_edma_intr;
> > +
> > + /* Mask all write/read interrupts so we don't get called again. */
> > + if (v->base) {
> > + iowrite32(~0x0, v->base + DMA_WRITE_INT_MASK_OFF);
> > + iowrite32(~0x0, v->base + DMA_READ_INT_MASK_OFF);
> > + }
> > +
> > + if (v->virq > 0)
> > + devm_free_irq(dev, v->virq, v);
> > +
> > + if (v->base)
> > + devm_iounmap(dev, v->base);
> > +
> > + v->virq = 0;
> > + v->cb = NULL;
> > + v->data = NULL;
> > +}
> > +
> > +int ntb_edma_setup_mws(struct ntb_dev *ndev)
> > +{
> > + const size_t info_bytes = PAGE_SIZE;
> > + resource_size_t size_max, offset;
> > + dma_addr_t intr_phys, info_phys;
> > + u32 wr_done = 0, rd_done = 0;
> > + struct ntb_edma_intr *intr;
> > + struct ntb_edma_info *info;
> > + int peer_mw, mw_index, rc;
> > + struct iommu_domain *dom;
> > + bool reg_mapped = false;
> > + size_t ll_bytes, size;
> > + struct pci_epc *epc;
> > + struct device *dev;
> > + unsigned long iova;
> > + phys_addr_t phys;
> > + u64 need;
> > + u32 i;
> > +
> > + /* +1 is for interruption */
> > + ll_bytes = (EDMA_WR_CH_NUM + EDMA_RD_CH_NUM + 1) * DMA_LLP_MEM_SIZE;
> > + need = EDMA_REG_SIZE + info_bytes + ll_bytes;
> > +
> > + epc = ntb_get_pci_epc(ndev);
> > + if (!epc)
> > + return -ENODEV;
> > + dev = epc->dev.parent;
> > +
> > + if (edma_ctx.initialized)
> > + return 0;
> > +
> > + info = dma_alloc_coherent(dev, info_bytes, &info_phys, GFP_KERNEL);
> > + if (!info)
> > + return -ENOMEM;
> > +
> > + memset(info, 0, info_bytes);
> > + info->magic = NTB_EDMA_INFO_MAGIC;
> > + info->wr_cnt = EDMA_WR_CH_NUM;
> > + info->rd_cnt = EDMA_RD_CH_NUM + 1; /* +1 for interruption */
> > + info->regs_phys = edma_regs_phys;
> > + info->ll_stride = DMA_LLP_MEM_SIZE;
> > +
> > + for (i = 0; i < EDMA_WR_CH_NUM; i++) {
> > + edma_ctx.ll_wr_virt[i] = dma_alloc_attrs(dev, DMA_LLP_MEM_SIZE,
> > + &edma_ctx.ll_wr_phys[i],
> > + GFP_KERNEL,
> > + DMA_ATTR_FORCE_CONTIGUOUS);
> > + if (!edma_ctx.ll_wr_virt[i]) {
> > + rc = -ENOMEM;
> > + goto err_free_ll;
> > + }
> > + wr_done++;
> > + info->ll_wr_phys[i] = edma_ctx.ll_wr_phys[i];
> > + }
> > + for (i = 0; i < EDMA_RD_CH_NUM + 1; i++) {
> > + edma_ctx.ll_rd_virt[i] = dma_alloc_attrs(dev, DMA_LLP_MEM_SIZE,
> > + &edma_ctx.ll_rd_phys[i],
> > + GFP_KERNEL,
> > + DMA_ATTR_FORCE_CONTIGUOUS);
> > + if (!edma_ctx.ll_rd_virt[i]) {
> > + rc = -ENOMEM;
> > + goto err_free_ll;
> > + }
> > + rd_done++;
> > + info->ll_rd_phys[i] = edma_ctx.ll_rd_phys[i];
> > + }
> > +
> > + /* For interruption */
> > + edma_ctx.notify_qp_max = NTB_EDMA_NOTIFY_MAX_QP;
> > + intr = dma_alloc_coherent(dev, sizeof(*intr), &intr_phys, GFP_KERNEL);
> > + if (!intr) {
> > + rc = -ENOMEM;
> > + goto err_free_ll;
> > + }
> > + memset(intr, 0, sizeof(*intr));
> > + edma_ctx.intr_ep_virt = intr;
> > + edma_ctx.intr_ep_phys = intr_phys;
> > + info->intr_dar_base = intr_phys;
> > +
> > + peer_mw = ntb_peer_mw_count(ndev);
> > + if (peer_mw <= 0) {
> > + rc = -ENODEV;
> > + goto err_free_ll;
> > + }
> > +
> > + mw_index = peer_mw - 1; /* last MW */
> > +
> > + rc = ntb_mw_get_align(ndev, 0, mw_index, 0, NULL, &size_max,
> > + &offset);
> > + if (rc)
> > + goto err_free_ll;
> > +
> > + if (size_max < need) {
> > + rc = -ENOSPC;
> > + goto err_free_ll;
> > + }
> > +
> > + /* Map register space (direct) */
> > + dom = iommu_get_domain_for_dev(dev);
> > + if (dom) {
> > + phys = edma_regs_phys & PAGE_MASK;
> > + size = PAGE_ALIGN(EDMA_REG_SIZE + edma_regs_phys - phys);
> > + iova = phys;
> > +
> > + rc = iommu_map(dom, iova, phys, EDMA_REG_SIZE,
> > + IOMMU_READ | IOMMU_WRITE | IOMMU_MMIO, GFP_KERNEL);
> > + if (rc)
> > + dev_err(&ndev->dev, "failed to create direct mapping for eDMA reg space\n");
> > + reg_mapped = true;
> > + }
> > +
> > + rc = ntb_mw_set_trans(ndev, 0, mw_index, edma_regs_phys, EDMA_REG_SIZE, offset);
> > + if (rc)
> > + goto err_unmap_reg;
> > +
> > + offset += EDMA_REG_SIZE;
> > +
> > + /* Map ntb_edma_info */
> > + rc = ntb_mw_set_trans(ndev, 0, mw_index, info_phys, info_bytes, offset);
> > + if (rc)
> > + goto err_clear_trans;
> > + offset += info_bytes;
> > +
> > + /* Map LL location */
> > + for (i = 0; i < EDMA_WR_CH_NUM; i++) {
> > + rc = ntb_mw_set_trans(ndev, 0, mw_index, edma_ctx.ll_wr_phys[i],
> > + DMA_LLP_MEM_SIZE, offset);
> > + if (rc)
> > + goto err_clear_trans;
> > + offset += DMA_LLP_MEM_SIZE;
> > + }
> > + for (i = 0; i < EDMA_RD_CH_NUM + 1; i++) {
> > + rc = ntb_mw_set_trans(ndev, 0, mw_index, edma_ctx.ll_rd_phys[i],
> > + DMA_LLP_MEM_SIZE, offset);
> > + if (rc)
> > + goto err_clear_trans;
> > + offset += DMA_LLP_MEM_SIZE;
> > + }
> > + edma_ctx.initialized = true;
> > +
> > + return 0;
> > +
> > +err_clear_trans:
> > + /*
> > + * Tear down the NTB translation window used for the eDMA MW.
> > + * There is no sub-range clear API for ntb_mw_set_trans(), so we
> > + * unconditionally drop the whole mapping on error.
> > + */
> > + ntb_mw_clear_trans(ndev, 0, mw_index);
> > +
> > +err_unmap_reg:
> > + if (reg_mapped)
> > + iommu_unmap(dom, iova, size);
> > +err_free_ll:
> > + while (rd_done--)
> > + dma_free_attrs(dev, DMA_LLP_MEM_SIZE,
> > + edma_ctx.ll_rd_virt[rd_done],
> > + edma_ctx.ll_rd_phys[rd_done],
> > + DMA_ATTR_FORCE_CONTIGUOUS);
> > + while (wr_done--)
> > + dma_free_attrs(dev, DMA_LLP_MEM_SIZE,
> > + edma_ctx.ll_wr_virt[wr_done],
> > + edma_ctx.ll_wr_phys[wr_done],
> > + DMA_ATTR_FORCE_CONTIGUOUS);
> > + if (edma_ctx.intr_ep_virt)
> > + dma_free_coherent(dev, sizeof(struct ntb_edma_intr),
> > + edma_ctx.intr_ep_virt,
> > + edma_ctx.intr_ep_phys);
> > + dma_free_coherent(dev, info_bytes, info, info_phys);
> > + return rc;
> > +}
> > +
> > +static int ntb_edma_irq_vector(struct device *dev, unsigned int nr)
> > +{
> > + struct pci_dev *pdev = to_pci_dev(dev);
> > + int ret, nvec;
> > +
> > + nvec = pci_msi_vec_count(pdev);
> > + for (; nr < nvec; nr++) {
> > + ret = pci_irq_vector(pdev, nr);
> > + if (!irq_has_action(ret))
> > + return ret;
> > + }
> > + return 0;
> > +}
> > +
> > +static const struct dw_edma_plat_ops ntb_edma_ops = {
> > + .irq_vector = ntb_edma_irq_vector,
> > +};
> > +
> > +int ntb_edma_setup_peer(struct ntb_dev *ndev)
> > +{
> > + struct ntb_edma_info *info;
> > + unsigned int wr_cnt, rd_cnt;
> > + struct dw_edma_chip *chip;
> > + void __iomem *edma_virt;
> > + phys_addr_t edma_phys;
> > + resource_size_t mw_size;
> > + u64 off = EDMA_REG_SIZE;
> > + int peer_mw, mw_index;
> > + unsigned int i;
> > + int ret;
> > +
> > + peer_mw = ntb_peer_mw_count(ndev);
> > + if (peer_mw <= 0)
> > + return -ENODEV;
> > +
> > + mw_index = peer_mw - 1; /* last MW */
> > +
> > + ret = ntb_peer_mw_get_addr(ndev, mw_index, &edma_phys,
> > + &mw_size);
> > + if (ret)
> > + return -1;
> > +
> > + edma_virt = ioremap(edma_phys, mw_size);
> > +
> > + chip = devm_kzalloc(&ndev->dev, sizeof(*chip), GFP_KERNEL);
> > + if (!chip) {
> > + ret = -ENOMEM;
> > + return ret;
> > + }
> > +
> > + chip->dev = &ndev->pdev->dev;
> > + chip->nr_irqs = 4;
> > + chip->ops = &ntb_edma_ops;
> > + chip->flags = 0;
> > + chip->reg_base = edma_virt;
> > + chip->mf = EDMA_MF_EDMA_UNROLL;
> > +
> > + info = edma_virt + off;
> > + if (info->magic != NTB_EDMA_INFO_MAGIC)
> > + return -EINVAL;
> > + wr_cnt = info->wr_cnt;
> > + rd_cnt = info->rd_cnt;
> > + chip->ll_wr_cnt = wr_cnt;
> > + chip->ll_rd_cnt = rd_cnt;
> > + off += PAGE_SIZE;
> > +
> > + edma_ctx.notify_qp_max = NTB_EDMA_NOTIFY_MAX_QP;
> > + edma_ctx.intr_ep_phys = info->intr_dar_base;
> > + if (edma_ctx.intr_ep_phys) {
> > + edma_ctx.intr_rc_virt =
> > + dma_alloc_coherent(&ndev->pdev->dev,
> > + sizeof(struct ntb_edma_intr),
> > + &edma_ctx.intr_rc_phys,
> > + GFP_KERNEL);
> > + if (!edma_ctx.intr_rc_virt)
> > + return -ENOMEM;
> > + memset(edma_ctx.intr_rc_virt, 0,
> > + sizeof(struct ntb_edma_intr));
> > + }
> > +
> > + for (i = 0; i < wr_cnt; i++) {
> > + chip->ll_region_wr[i].vaddr.io = edma_virt + off;
> > + chip->ll_region_wr[i].paddr = info->ll_wr_phys[i];
> > + chip->ll_region_wr[i].sz = DMA_LLP_MEM_SIZE;
> > + off += DMA_LLP_MEM_SIZE;
> > + }
> > + for (i = 0; i < rd_cnt; i++) {
> > + chip->ll_region_rd[i].vaddr.io = edma_virt + off;
> > + chip->ll_region_rd[i].paddr = info->ll_rd_phys[i];
> > + chip->ll_region_rd[i].sz = DMA_LLP_MEM_SIZE;
> > + off += DMA_LLP_MEM_SIZE;
> > + }
> > +
> > + if (!pci_dev_msi_enabled(ndev->pdev))
> > + return -ENXIO;
> > +
> > + ret = dw_edma_probe(chip);
> > + if (ret) {
> > + dev_err(&ndev->dev, "dw_edma_probe failed: %d\n", ret);
> > + return ret;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +struct ntb_edma_filter {
> > + struct device *dma_dev;
> > + u32 direction;
> > +};
> > +
> > +static bool ntb_edma_filter_fn(struct dma_chan *chan, void *arg)
> > +{
> > + struct ntb_edma_filter *filter = arg;
> > + u32 dir = filter->direction;
> > + struct dma_slave_caps caps;
> > + int ret;
> > +
> > + if (chan->device->dev != filter->dma_dev)
> > + return false;
> > +
> > + ret = dma_get_slave_caps(chan, &caps);
> > + if (ret < 0)
> > + return false;
> > +
> > + return !!(caps.directions & dir);
> > +}
> > +
> > +void ntb_edma_teardown_chans(struct ntb_edma_chans *edma)
> > +{
> > + unsigned int i;
> > +
> > + for (i = 0; i < edma->num_wr_chan; i++)
> > + dma_release_channel(edma->wr_chan[i]);
> > +
> > + for (i = 0; i < edma->num_rd_chan; i++)
> > + dma_release_channel(edma->rd_chan[i]);
> > +
> > + if (edma->intr_chan)
> > + dma_release_channel(edma->intr_chan);
> > +}
> > +
> > +int ntb_edma_setup_chans(struct device *dma_dev, struct ntb_edma_chans *edma)
> > +{
> > + struct ntb_edma_filter filter;
> > + dma_cap_mask_t dma_mask;
> > + unsigned int i;
> > +
> > + dma_cap_zero(dma_mask);
> > + dma_cap_set(DMA_SLAVE, dma_mask);
> > +
> > + memset(edma, 0, sizeof(*edma));
> > + edma->dev = dma_dev;
> > +
> > + filter.dma_dev = dma_dev;
> > + filter.direction = BIT(DMA_DEV_TO_MEM);
> > + for (i = 0; i < EDMA_WR_CH_NUM; i++) {
> > + edma->wr_chan[i] = dma_request_channel(dma_mask,
> > + ntb_edma_filter_fn,
> > + &filter);
> > + if (!edma->wr_chan[i])
> > + break;
> > + edma->num_wr_chan++;
> > + }
> > +
> > + filter.direction = BIT(DMA_MEM_TO_DEV);
> > + for (i = 0; i < EDMA_RD_CH_NUM; i++) {
> > + edma->rd_chan[i] = dma_request_channel(dma_mask,
> > + ntb_edma_filter_fn,
> > + &filter);
> > + if (!edma->rd_chan[i])
> > + break;
> > + edma->num_rd_chan++;
> > + }
> > +
> > + edma->intr_chan = dma_request_channel(dma_mask, ntb_edma_filter_fn,
> > + &filter);
> > + if (!edma->intr_chan)
> > + dev_warn(dma_dev,
> > + "Remote eDMA notify channel could not be allocated\n");
> > +
> > + if (!edma->num_wr_chan || !edma->num_rd_chan) {
> > + dev_warn(dma_dev, "Remote eDMA channels failed to initialize\n");
> > + ntb_edma_teardown_chans(edma);
> > + return -ENODEV;
> > + }
> > + return 0;
> > +}
> > +
> > +struct dma_chan *ntb_edma_pick_chan(struct ntb_edma_chans *edma,
> > + remote_edma_dir_t dir)
> > +{
> > + unsigned int n, cur, idx;
> > + struct dma_chan **chans;
> > + atomic_t *cur_chan;
> > +
> > + if (dir == REMOTE_EDMA_WRITE) {
> > + n = edma->num_wr_chan;
> > + chans = edma->wr_chan;
> > + cur_chan = &edma->cur_wr_chan;
> > + } else {
> > + n = edma->num_rd_chan;
> > + chans = edma->rd_chan;
> > + cur_chan = &edma->cur_rd_chan;
> > + }
> > + if (WARN_ON_ONCE(!n))
> > + return NULL;
> > +
> > + /* Simple round-robin */
> > + cur = (unsigned int)atomic_inc_return(cur_chan) - 1;
> > + idx = cur % n;
> > + return chans[idx];
> > +}
> > +
> > +int ntb_edma_notify_peer(struct ntb_edma_chans *edma, int qp_num)
> > +{
> > + struct dma_async_tx_descriptor *txd;
> > + struct dma_slave_config cfg;
> > + struct scatterlist sgl;
> > + dma_cookie_t cookie;
> > + struct device *dev;
> > +
> > + if (!edma || !edma->intr_chan)
> > + return -ENXIO;
> > +
> > + if (qp_num < 0 || qp_num >= edma_ctx.notify_qp_max)
> > + return -EINVAL;
> > +
> > + if (!edma_ctx.intr_rc_virt || !edma_ctx.intr_ep_phys)
> > + return -EINVAL;
> > +
> > + dev = edma->dev;
> > + if (!dev)
> > + return -ENODEV;
> > +
> > + WRITE_ONCE(edma_ctx.intr_rc_virt->db[qp_num], 1);
> > +
> > + /* Ensure store is visible before kicking the DMA transfer */
> > + wmb();
> > +
> > + sg_init_table(&sgl, 1);
> > + sg_dma_address(&sgl) = edma_ctx.intr_rc_phys + qp_num * sizeof(u32);
> > + sg_dma_len(&sgl) = sizeof(u32);
> > +
> > + memset(&cfg, 0, sizeof(cfg));
> > + cfg.dst_addr = edma_ctx.intr_ep_phys + qp_num * sizeof(u32);
> > + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> > + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> > + cfg.direction = DMA_MEM_TO_DEV;
> > +
> > + if (dmaengine_slave_config(edma->intr_chan, &cfg))
> > + return -EINVAL;
> > +
> > + txd = dmaengine_prep_slave_sg(edma->intr_chan, &sgl, 1,
> > + DMA_MEM_TO_DEV,
> > + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> > + if (!txd)
> > + return -ENOSPC;
> > +
> > + cookie = dmaengine_submit(txd);
> > + if (dma_submit_error(cookie))
> > + return -ENOSPC;
> > +
> > + dma_async_issue_pending(edma->intr_chan);
> > + return 0;
> > +}
> > diff --git a/drivers/ntb/ntb_edma.h b/drivers/ntb/ntb_edma.h
> > new file mode 100644
> > index 000000000000..da0451827edb
> > --- /dev/null
> > +++ b/drivers/ntb/ntb_edma.h
> > @@ -0,0 +1,128 @@
> > +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
> > +#ifndef _NTB_EDMA_H_
> > +#define _NTB_EDMA_H_
> > +
> > +#include <linux/completion.h>
> > +#include <linux/device.h>
> > +#include <linux/interrupt.h>
> > +
> > +#define EDMA_REG_SIZE SZ_64K
> > +#define DMA_LLP_MEM_SIZE SZ_4K
> > +#define EDMA_WR_CH_NUM 4
> > +#define EDMA_RD_CH_NUM 4
> > +#define NTB_EDMA_MAX_CH 8
> > +
> > +#define NTB_EDMA_INFO_MAGIC 0x45444D41 /* "EDMA" */
> > +#define NTB_EDMA_INFO_OFF EDMA_REG_SIZE
> > +
> > +#define NTB_EDMA_RING_ORDER 7
> > +#define NTB_EDMA_RING_ENTRIES (1U << NTB_EDMA_RING_ORDER)
> > +#define NTB_EDMA_RING_MASK (NTB_EDMA_RING_ENTRIES - 1)
> > +
> > +typedef void (*ntb_edma_interrupt_cb_t)(void *data, int qp_num);
> > +
> > +/*
> > + * REMOTE_EDMA_EP:
> > + * Endpoint owns the eDMA engine and pushes descriptors into a shared MW.
> > + *
> > + * REMOTE_EDMA_RC:
> > + * Root Complex controls the endpoint eDMA through the shared MW and
> > + * drives reads/writes on behalf of the host.
> > + */
> > +typedef enum {
> > + REMOTE_EDMA_UNKNOWN,
> > + REMOTE_EDMA_EP,
> > + REMOTE_EDMA_RC,
> > +} remote_edma_mode_t;
> > +
> > +typedef enum {
> > + REMOTE_EDMA_WRITE,
> > + REMOTE_EDMA_READ,
> > +} remote_edma_dir_t;
> > +
> > +/*
> > + * Layout of remote eDMA MW (EP local address space, RC sees via peer MW):
> > + *
> > + * 0 .. EDMA_REG_SIZE-1 : DesignWare eDMA registers
> > + * EDMA_REG_SIZE .. +PAGE_SIZE : struct ntb_edma_info (EP writes, RC reads)
> > + * +PAGE_SIZE .. : LL ring buffers (EP allocates phys addresses,
> > + * RC configures via dw_edma)
> > + *
> > + * ntb_edma_setup_mws() on EP:
> > + * - allocates ntb_edma_info and LLs in EP memory
> > + * - programs inbound iATU so that RC peer MW[n] points at this block
> > + *
> > + * ntb_edma_setup_peer() on RC:
> > + * - ioremaps peer MW[n]
> > + * - reads ntb_edma_info
> > + * - sets up dw_edma_chip ll_region_* from that info
> > + */
> > +struct ntb_edma_info {
> > + u32 magic;
> > + u16 wr_cnt;
> > + u16 rd_cnt;
> > + u64 regs_phys;
> > + u32 ll_stride;
> > + u32 rsvd;
> > + u64 ll_wr_phys[NTB_EDMA_MAX_CH];
> > + u64 ll_rd_phys[NTB_EDMA_MAX_CH];
> > +
> > + u64 intr_dar_base;
> > +} __packed;
> > +
> > +struct ll_dma_addrs {
> > + dma_addr_t wr[EDMA_WR_CH_NUM];
> > + dma_addr_t rd[EDMA_RD_CH_NUM];
> > +};
> > +
> > +struct ntb_edma_chans {
> > + struct device *dev;
> > +
> > + struct dma_chan *wr_chan[EDMA_WR_CH_NUM];
> > + struct dma_chan *rd_chan[EDMA_RD_CH_NUM];
> > + struct dma_chan *intr_chan;
> > +
> > + unsigned int num_wr_chan;
> > + unsigned int num_rd_chan;
> > + atomic_t cur_wr_chan;
> > + atomic_t cur_rd_chan;
> > +};
> > +
> > +static __always_inline u32 ntb_edma_ring_idx(u32 v)
> > +{
> > + return v & NTB_EDMA_RING_MASK;
> > +}
> > +
> > +static __always_inline u32 ntb_edma_ring_used_entry(u32 head, u32 tail)
> > +{
> > + if (head >= tail) {
> > + WARN_ON_ONCE((head - tail) > (NTB_EDMA_RING_ENTRIES - 1));
> > + return head - tail;
> > + }
> > +
> > + WARN_ON_ONCE((U32_MAX - tail + head + 1) > (NTB_EDMA_RING_ENTRIES - 1));
> > + return U32_MAX - tail + head + 1;
> > +}
> > +
> > +static __always_inline u32 ntb_edma_ring_free_entry(u32 head, u32 tail)
> > +{
> > + return NTB_EDMA_RING_ENTRIES - ntb_edma_ring_used_entry(head, tail) - 1;
> > +}
> > +
> > +static __always_inline bool ntb_edma_ring_full(u32 head, u32 tail)
> > +{
> > + return ntb_edma_ring_free_entry(head, tail) == 0;
> > +}
> > +
> > +int ntb_edma_setup_isr(struct device *dev, struct device *epc_dev,
> > + ntb_edma_interrupt_cb_t cb, void *data);
> > +void ntb_edma_teardown_isr(struct device *dev);
> > +int ntb_edma_setup_mws(struct ntb_dev *ndev);
> > +int ntb_edma_setup_peer(struct ntb_dev *ndev);
> > +int ntb_edma_setup_chans(struct device *dma_dev, struct ntb_edma_chans *edma);
> > +struct dma_chan *ntb_edma_pick_chan(struct ntb_edma_chans *edma,
> > + remote_edma_dir_t dir);
> > +void ntb_edma_teardown_chans(struct ntb_edma_chans *edma);
> > +int ntb_edma_notify_peer(struct ntb_edma_chans *edma, int qp_num);
> > +
> > +#endif
> > diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport_core.c
> > similarity index 65%
> > rename from drivers/ntb/ntb_transport.c
> > rename to drivers/ntb/ntb_transport_core.c
> > index 907db6c93d4d..48d48921978d 100644
> > --- a/drivers/ntb/ntb_transport.c
> > +++ b/drivers/ntb/ntb_transport_core.c
> > @@ -47,6 +47,9 @@
> > * Contact Information:
> > * Jon Mason <jon.mason@intel.com>
> > */
> > +#include <linux/atomic.h>
> > +#include <linux/bug.h>
> > +#include <linux/compiler.h>
> > #include <linux/debugfs.h>
> > #include <linux/delay.h>
> > #include <linux/dmaengine.h>
> > @@ -71,6 +74,8 @@
> > #define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB"
> > #define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2)
> >
> > +#define NTB_EDMA_MAX_POLL 32
> > +
> > MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
> > MODULE_VERSION(NTB_TRANSPORT_VER);
> > MODULE_LICENSE("Dual BSD/GPL");
> > @@ -102,6 +107,13 @@ module_param(use_msi, bool, 0644);
> > MODULE_PARM_DESC(use_msi, "Use MSI interrupts instead of doorbells");
> > #endif
> >
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>
> This is a comment throughout this patch. Doing ifdefs inside C source is pretty frowed upon in the kernel. The preferred way is to only have ifdefs in the header files. So please give this a bit more consideration and see if it can be done differently to address this.
I agree, there is no good reason to keep those remaining ifdefs at all.
I'll clean it up. Thanks for pointing this out.
>
> > +#include "ntb_edma.h"
> > +static bool use_remote_edma;
> > +module_param(use_remote_edma, bool, 0644);
> > +MODULE_PARM_DESC(use_remote_edma, "Use remote eDMA mode (when enabled, use_msi is ignored)");
> > +#endif
> > +
> > static struct dentry *nt_debugfs_dir;
> >
> > /* Only two-ports NTB devices are supported */
> > @@ -125,6 +137,14 @@ struct ntb_queue_entry {
> > struct ntb_payload_header __iomem *tx_hdr;
> > struct ntb_payload_header *rx_hdr;
> > };
> > +
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + dma_addr_t addr;
> > +
> > + /* Used by RC side only */
> > + struct scatterlist sgl;
> > + struct work_struct dma_work;
> > +#endif
> > };
> >
> > struct ntb_rx_info {
> > @@ -202,6 +222,33 @@ struct ntb_transport_qp {
> > int msi_irq;
> > struct ntb_msi_desc msi_desc;
> > struct ntb_msi_desc peer_msi_desc;
> > +
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + /*
> > + * For ensuring peer notification in non-atomic context.
> > + * ntb_peer_db_set might sleep or schedule.
> > + */
> > + struct work_struct db_work;
> > +
> > + /*
> > + * wr: remote eDMA write transfer (EP -> RC direction)
> > + * rd: remote eDMA read transfer (RC -> EP direction)
> > + */
> > + u32 wr_cons;
> > + u32 rd_cons;
> > + u32 wr_prod;
> > + u32 rd_prod;
> > + u32 wr_issue;
> > + u32 rd_issue;
> > +
> > + spinlock_t ep_tx_lock;
> > + spinlock_t ep_rx_lock;
> > + spinlock_t rc_lock;
> > +
> > + /* Completion work for read/write transfers. */
> > + struct work_struct read_work;
> > + struct work_struct write_work;
> > +#endif
>
> For something like this, maybe it needs its own struct instead of an ifdef chunk. Perhaps 'ntb_rx_info' can serve as a core data struct with EDMA having 'ntb_rx_info_edma' and embed 'ntb_rx_info'.
Thanks again for the suggestion. I'll reorganize things.
Koichiro
>
> DJ
>
> > };
> >
> > struct ntb_transport_mw {
> > @@ -249,6 +296,13 @@ struct ntb_transport_ctx {
> >
> > /* Make sure workq of link event be executed serially */
> > struct mutex link_event_lock;
> > +
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + remote_edma_mode_t remote_edma_mode;
> > + struct device *dma_dev;
> > + struct workqueue_struct *wq;
> > + struct ntb_edma_chans edma;
> > +#endif
> > };
> >
> > enum {
> > @@ -262,6 +316,19 @@ struct ntb_payload_header {
> > unsigned int flags;
> > };
> >
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > +static void ntb_transport_edma_uninit(struct ntb_transport_ctx *nt);
> > +static int ntb_transport_edma_init(struct ntb_transport_ctx *nt,
> > + unsigned int *mw_count);
> > +static void ntb_transport_edma_init_queue(struct ntb_transport_ctx *nt,
> > + unsigned int qp_num);
> > +static void ntb_transport_edma_create_queue(struct ntb_transport_ctx *nt,
> > + struct ntb_transport_qp *qp);
> > +static int ntb_transport_edma_rc_init(struct ntb_transport_ctx *nt);
> > +static int ntb_transport_edma_ep_init(struct ntb_transport_ctx *nt);
> > +static void ntb_transport_edma_rc_dma_work(struct work_struct *work);
> > +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> > +
> > /*
> > * Return the device that should be used for DMA mapping.
> > *
> > @@ -298,7 +365,7 @@ enum {
> > container_of((__drv), struct ntb_transport_client, driver)
> >
> > #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count)
> > -#define NTB_QP_DEF_NUM_ENTRIES 100
> > +#define NTB_QP_DEF_NUM_ENTRIES 128
> > #define NTB_LINK_DOWN_TIMEOUT 10
> >
> > static void ntb_transport_rxc_db(unsigned long data);
> > @@ -1015,6 +1082,10 @@ static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
> > count = ntb_spad_count(nt->ndev);
> > for (i = 0; i < count; i++)
> > ntb_spad_write(nt->ndev, i, 0);
> > +
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + ntb_edma_teardown_chans(&nt->edma);
> > +#endif
> > }
> >
> > static void ntb_transport_link_cleanup_work(struct work_struct *work)
> > @@ -1051,6 +1122,14 @@ static void ntb_transport_link_work(struct work_struct *work)
> >
> > /* send the local info, in the opposite order of the way we read it */
> >
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + rc = ntb_transport_edma_ep_init(nt);
> > + if (rc) {
> > + dev_err(&pdev->dev, "Failed to init EP: %d\n", rc);
> > + return;
> > + }
> > +#endif
> > +
> > if (nt->use_msi) {
> > rc = ntb_msi_setup_mws(ndev);
> > if (rc) {
> > @@ -1132,6 +1211,14 @@ static void ntb_transport_link_work(struct work_struct *work)
> >
> > nt->link_is_up = true;
> >
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + rc = ntb_transport_edma_rc_init(nt);
> > + if (rc) {
> > + dev_err(&pdev->dev, "Failed to init RC: %d\n", rc);
> > + goto out1;
> > + }
> > +#endif
> > +
> > for (i = 0; i < nt->qp_count; i++) {
> > struct ntb_transport_qp *qp = &nt->qp_vec[i];
> >
> > @@ -1277,6 +1364,8 @@ static const struct ntb_transport_backend_ops default_backend_ops = {
> > .debugfs_stats_show = ntb_transport_default_debugfs_stats_show,
> > };
> >
> > +static const struct ntb_transport_backend_ops edma_backend_ops;
> > +
> > static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> > {
> > struct ntb_transport_ctx *nt;
> > @@ -1311,7 +1400,23 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> >
> > nt->ndev = ndev;
> >
> > - nt->backend_ops = default_backend_ops;
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + if (use_remote_edma) {
> > + rc = ntb_transport_edma_init(nt, &mw_count);
> > + if (rc) {
> > + nt->mw_count = 0;
> > + goto err;
> > + }
> > + nt->backend_ops = edma_backend_ops;
> > +
> > + /*
> > + * On remote eDMA mode, we reserve a read channel for Host->EP
> > + * interruption.
> > + */
> > + use_msi = false;
> > + } else
> > +#endif
> > + nt->backend_ops = default_backend_ops;
> >
> > /*
> > * If we are using MSI, and have at least one extra memory window,
> > @@ -1402,6 +1507,10 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> > rc = ntb_transport_init_queue(nt, i);
> > if (rc)
> > goto err2;
> > +
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + ntb_transport_edma_init_queue(nt, i);
> > +#endif
> > }
> >
> > INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);
> > @@ -1433,6 +1542,9 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> > }
> > kfree(nt->mw_vec);
> > err:
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + ntb_transport_edma_uninit(nt);
> > +#endif
> > kfree(nt);
> > return rc;
> > }
> > @@ -2055,11 +2167,16 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
> >
> > nt->qp_bitmap_free &= ~qp_bit;
> >
> > + qp->qp_bit = qp_bit;
> > qp->cb_data = data;
> > qp->rx_handler = handlers->rx_handler;
> > qp->tx_handler = handlers->tx_handler;
> > qp->event_handler = handlers->event_handler;
> >
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + ntb_transport_edma_create_queue(nt, qp);
> > +#endif
> > +
> > dma_cap_zero(dma_mask);
> > dma_cap_set(DMA_MEMCPY, dma_mask);
> >
> > @@ -2105,6 +2222,9 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
> > goto err1;
> >
> > entry->qp = qp;
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + INIT_WORK(&entry->dma_work, ntb_transport_edma_rc_dma_work);
> > +#endif
> > ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> > &qp->rx_free_q);
> > }
> > @@ -2156,8 +2276,8 @@ EXPORT_SYMBOL_GPL(ntb_transport_create_queue);
> > */
> > void ntb_transport_free_queue(struct ntb_transport_qp *qp)
> > {
> > - struct pci_dev *pdev;
> > struct ntb_queue_entry *entry;
> > + struct pci_dev *pdev;
> > u64 qp_bit;
> >
> > if (!qp)
> > @@ -2208,6 +2328,10 @@ void ntb_transport_free_queue(struct ntb_transport_qp *qp)
> > tasklet_kill(&qp->rxc_db_work);
> >
> > cancel_delayed_work_sync(&qp->link_work);
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > + cancel_work_sync(&qp->read_work);
> > + cancel_work_sync(&qp->write_work);
> > +#endif
> >
> > qp->cb_data = NULL;
> > qp->rx_handler = NULL;
> > @@ -2346,6 +2470,1157 @@ int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
> > }
> > EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue);
> >
> > +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> > +/*
> > + * Remote eDMA mode implementation
> > + */
> > +struct ntb_edma_desc {
> > + u32 len;
> > + u32 flags;
> > + u64 addr; /* DMA address */
> > + u64 data;
> > +};
> > +
> > +struct ntb_edma_ring {
> > + struct ntb_edma_desc desc[NTB_EDMA_RING_ENTRIES];
> > + u32 head;
> > + u32 tail;
> > +};
> > +
> > +#define NTB_EDMA_DESC_OFF(i) ((size_t)(i) * sizeof(struct ntb_edma_desc))
> > +
> > +#define __NTB_EDMA_CHECK_INDEX(_i) \
> > +({ \
> > + unsigned long __i = (unsigned long)(_i); \
> > + WARN_ONCE(__i >= (unsigned long)NTB_EDMA_RING_ENTRIES, \
> > + "ntb_edma: index i=%lu >= ring_entries=%lu\n", \
> > + __i, (unsigned long)NTB_EDMA_RING_ENTRIES); \
> > + __i; \
> > +})
> > +
> > +#define NTB_EDMA_DESC_I(qp, i, n) \
> > +({ \
> > + typeof(qp) __qp = (qp); \
> > + unsigned long __i = __NTB_EDMA_CHECK_INDEX(i); \
> > + (struct ntb_edma_desc *) \
> > + ((char *)(__qp)->rx_buff + \
> > + (sizeof(struct ntb_edma_ring) * n) + \
> > + NTB_EDMA_DESC_OFF(__i)); \
> > +})
> > +
> > +#define NTB_EDMA_DESC_O(qp, i, n) \
> > +({ \
> > + typeof(qp) __qp = (qp); \
> > + unsigned long __i = __NTB_EDMA_CHECK_INDEX(i); \
> > + (struct ntb_edma_desc __iomem *) \
> > + ((char __iomem *)(__qp)->tx_mw + \
> > + (sizeof(struct ntb_edma_ring) * n) + \
> > + NTB_EDMA_DESC_OFF(__i)); \
> > +})
> > +
> > +#define NTB_EDMA_HEAD_I(qp, n) ((u32 *)((char *)qp->rx_buff + \
> > + (sizeof(struct ntb_edma_ring) * n) + \
> > + offsetof(struct ntb_edma_ring, head)))
> > +#define NTB_EDMA_HEAD_O(qp, n) ((u32 *)((char __iomem *)qp->tx_mw + \
> > + (sizeof(struct ntb_edma_ring) * n) + \
> > + offsetof(struct ntb_edma_ring, head)))
> > +#define NTB_EDMA_TAIL_I(qp, n) ((u32 *)((char *)qp->rx_buff + \
> > + (sizeof(struct ntb_edma_ring) * n) + \
> > + offsetof(struct ntb_edma_ring, tail)))
> > +#define NTB_EDMA_TAIL_O(qp, n) ((u32 *)((char __iomem *)qp->tx_mw + \
> > + (sizeof(struct ntb_edma_ring) * n) + \
> > + offsetof(struct ntb_edma_ring, tail)))
> > +
> > +/*
> > + * Macro naming rule:
> > + * NTB_DESC_RD_EP_I (as an example)
> > + * ^^ ^^ ^
> > + * : : `-- I(n) or O(ut). In = Read, Out = Write.
> > + * : `----- Who uses this macro.
> > + * `-------- DESC / HEAD / TAIL
> > + *
> > + * Read transfers (RC->EP):
> > + *
> > + * EP view (outbound, written via NTB):
> > + * - descs: NTB_DESC_RD_EP_O(qp, i) / NTB_DESC_RD_EP_I(qp, i)
> > + * [ len ][ flags ][ addr ][ data ]
> > + * [ len ][ flags ][ addr ][ data ]
> > + * :
> > + * [ len ][ flags ][ addr ][ data ]
> > + * - head: NTB_HEAD_RD_EP_O(qp)
> > + * - tail: NTB_TAIL_RD_EP_I(qp)
> > + *
> > + * RC view (inbound, local mapping):
> > + * - descs: NTB_DESC_RD_RC_I(qp, i) / NTB_DESC_RD_RC_O(qp, i)
> > + * [ len ][ flags ][ addr ][ data ]
> > + * [ len ][ flags ][ addr ][ data ]
> > + * :
> > + * [ len ][ flags ][ addr ][ data ]
> > + * - head: NTB_HEAD_RD_RC_I(qp)
> > + * - tail: NTB_TAIL_RD_RC_O(qp)
> > + *
> > + * Write transfers (EP -> RC) are analogous but use
> > + * NTB_DESC_WR_{EP_O,RC_I}(), NTB_HEAD_WR_{EP_O,RC_I}(),
> > + * and NTB_TAIL_WR_{EP_I,RC_O}().
> > + */
> > +#define NTB_DESC_RD_EP_I(qp, i) NTB_EDMA_DESC_I(qp, i, 0)
> > +#define NTB_DESC_RD_EP_O(qp, i) NTB_EDMA_DESC_O(qp, i, 0)
> > +#define NTB_DESC_WR_EP_I(qp, i) NTB_EDMA_DESC_I(qp, i, 1)
> > +#define NTB_DESC_WR_EP_O(qp, i) NTB_EDMA_DESC_O(qp, i, 1)
> > +#define NTB_DESC_RD_RC_I(qp, i) NTB_EDMA_DESC_I(qp, i, 0)
> > +#define NTB_DESC_RD_RC_O(qp, i) NTB_EDMA_DESC_O(qp, i, 0)
> > +#define NTB_DESC_WR_RC_I(qp, i) NTB_EDMA_DESC_I(qp, i, 1)
> > +#define NTB_DESC_WR_RC_O(qp, i) NTB_EDMA_DESC_O(qp, i, 1)
> > +
> > +#define NTB_HEAD_RD_EP_O(qp) NTB_EDMA_HEAD_O(qp, 0)
> > +#define NTB_HEAD_WR_EP_O(qp) NTB_EDMA_HEAD_O(qp, 1)
> > +#define NTB_HEAD_RD_RC_I(qp) NTB_EDMA_HEAD_I(qp, 0)
> > +#define NTB_HEAD_WR_RC_I(qp) NTB_EDMA_HEAD_I(qp, 1)
> > +
> > +#define NTB_TAIL_RD_EP_I(qp) NTB_EDMA_TAIL_I(qp, 0)
> > +#define NTB_TAIL_WR_EP_I(qp) NTB_EDMA_TAIL_I(qp, 1)
> > +#define NTB_TAIL_RD_RC_O(qp) NTB_EDMA_TAIL_O(qp, 0)
> > +#define NTB_TAIL_WR_RC_O(qp) NTB_EDMA_TAIL_O(qp, 1)
> > +
> > +static inline bool ntb_qp_edma_is_rc(struct ntb_transport_qp *qp)
> > +{
> > + return qp->transport->remote_edma_mode == REMOTE_EDMA_RC;
> > +}
> > +
> > +static inline bool ntb_qp_edma_is_ep(struct ntb_transport_qp *qp)
> > +{
> > + return qp->transport->remote_edma_mode == REMOTE_EDMA_EP;
> > +}
> > +
> > +static inline bool ntb_qp_edma_enabled(struct ntb_transport_qp *qp)
> > +{
> > + return ntb_qp_edma_is_rc(qp) || ntb_qp_edma_is_ep(qp);
> > +}
> > +
> > +static unsigned int ntb_transport_edma_tx_free_entry(struct ntb_transport_qp *qp)
> > +{
> > + unsigned int head, tail;
> > +
> > + if (ntb_qp_edma_is_ep(qp)) {
> > + scoped_guard(spinlock_irqsave, &qp->ep_tx_lock) {
> > + /* In this scope, only 'head' might proceed */
> > + tail = READ_ONCE(qp->wr_cons);
> > + head = READ_ONCE(qp->wr_prod);
> > + }
> > + return ntb_edma_ring_free_entry(head, tail);
> > + }
> > +
> > + scoped_guard(spinlock_irqsave, &qp->rc_lock) {
> > + /* In this scope, only 'head' might proceed */
> > + tail = READ_ONCE(qp->rd_issue);
> > + head = READ_ONCE(*NTB_HEAD_RD_RC_I(qp));
> > + }
> > + /*
> > + * On RC side, 'used' amount indicates how much EP side
> > + * has refilled, which are available for us to use for TX.
> > + */
> > + return ntb_edma_ring_used_entry(head, tail);
> > +}
> > +
> > +static void ntb_transport_edma_debugfs_stats_show(struct seq_file *s,
> > + struct ntb_transport_qp *qp)
> > +{
> > + seq_printf(s, "rx_bytes - \t%llu\n", qp->rx_bytes);
> > + seq_printf(s, "rx_pkts - \t%llu\n", qp->rx_pkts);
> > + seq_printf(s, "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
> > + seq_printf(s, "rx_buff - \t0x%p\n", qp->rx_buff);
> > + seq_printf(s, "rx_max_entry - \t%u\n", qp->rx_max_entry);
> > + seq_printf(s, "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry);
> > +
> > + seq_printf(s, "tx_bytes - \t%llu\n", qp->tx_bytes);
> > + seq_printf(s, "tx_pkts - \t%llu\n", qp->tx_pkts);
> > + seq_printf(s, "tx_ring_full - \t%llu\n", qp->tx_ring_full);
> > + seq_printf(s, "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
> > + seq_printf(s, "tx_mw - \t0x%p\n", qp->tx_mw);
> > + seq_printf(s, "tx_max_entry - \t%u\n", qp->tx_max_entry);
> > + seq_printf(s, "free tx - \t%u\n", ntb_transport_tx_free_entry(qp));
> > + seq_putc(s, '\n');
> > +
> > + seq_puts(s, "Using Remote eDMA - Yes\n");
> > + seq_printf(s, "QP Link - \t%s\n", qp->link_is_up ? "Up" : "Down");
> > +}
> > +
> > +static void ntb_transport_edma_uninit(struct ntb_transport_ctx *nt)
> > +{
> > + struct ntb_dev *ndev = nt->ndev;
> > +
> > + if (nt->remote_edma_mode == REMOTE_EDMA_EP && ndev && ndev->pdev)
> > + ntb_edma_teardown_isr(&ndev->pdev->dev);
> > +
> > + if (nt->wq)
> > + destroy_workqueue(nt->wq);
> > + nt->wq = NULL;
> > +}
> > +
> > +static int ntb_transport_edma_init(struct ntb_transport_ctx *nt,
> > + unsigned int *mw_count)
> > +{
> > + struct ntb_dev *ndev = nt->ndev;
> > +
> > + /*
> > + * We need at least one MW for the transport plus one MW reserved
> > + * for the remote eDMA window (see ntb_edma_setup_mws/peer).
> > + */
> > + if (*mw_count <= 1) {
> > + dev_err(&ndev->dev,
> > + "remote eDMA requires at least two MWS (have %u)\n",
> > + *mw_count);
> > + return -ENODEV;
> > + }
> > +
> > + nt->wq = alloc_workqueue("ntb-edma-wq", WQ_UNBOUND | WQ_SYSFS, 0);
> > + if (!nt->wq) {
> > + ntb_transport_edma_uninit(nt);
> > + return -ENOMEM;
> > + }
> > +
> > + /* Reserve the last peer MW exclusively for the eDMA window. */
> > + *mw_count -= 1;
> > +
> > + return 0;
> > +}
> > +
> > +static void ntb_transport_edma_db_work(struct work_struct *work)
> > +{
> > + struct ntb_transport_qp *qp =
> > + container_of(work, struct ntb_transport_qp, db_work);
> > +
> > + ntb_peer_db_set(qp->ndev, qp->qp_bit);
> > +}
> > +
> > +static void ntb_transport_edma_notify_peer(struct ntb_transport_qp *qp)
> > +{
> > + if (ntb_qp_edma_is_rc(qp))
> > + if (!ntb_edma_notify_peer(&qp->transport->edma, qp->qp_num))
> > + return;
> > +
> > + /*
> > + * Called from contexts that may be atomic. Since ntb_peer_db_set()
> > + * may sleep, delegate the actual doorbell write to a workqueue.
> > + */
> > + queue_work(system_highpri_wq, &qp->db_work);
> > +}
> > +
> > +static void ntb_transport_edma_isr(void *data, int qp_num)
> > +{
> > + struct ntb_transport_ctx *nt = data;
> > + struct ntb_transport_qp *qp;
> > +
> > + if (qp_num < 0 || qp_num >= nt->qp_count)
> > + return;
> > +
> > + qp = &nt->qp_vec[qp_num];
> > + if (WARN_ON(!qp))
> > + return;
> > +
> > + queue_work(nt->wq, &qp->read_work);
> > + queue_work(nt->wq, &qp->write_work);
> > +}
> > +
> > +static int ntb_transport_edma_rc_init(struct ntb_transport_ctx *nt)
> > +{
> > + struct ntb_dev *ndev = nt->ndev;
> > + struct pci_dev *pdev = ndev->pdev;
> > + int rc;
> > +
> > + if (!use_remote_edma || nt->remote_edma_mode != REMOTE_EDMA_UNKNOWN)
> > + return 0;
> > +
> > + rc = ntb_edma_setup_peer(ndev);
> > + if (rc) {
> > + dev_err(&pdev->dev, "Failed to enable remote eDMA: %d\n", rc);
> > + return rc;
> > + }
> > +
> > + rc = ntb_edma_setup_chans(get_dma_dev(ndev), &nt->edma);
> > + if (rc) {
> > + dev_err(&pdev->dev, "Failed to setup eDMA channels: %d\n", rc);
> > + return rc;
> > + }
> > +
> > + nt->remote_edma_mode = REMOTE_EDMA_RC;
> > + return 0;
> > +}
> > +
> > +static int ntb_transport_edma_ep_init(struct ntb_transport_ctx *nt)
> > +{
> > + struct ntb_dev *ndev = nt->ndev;
> > + struct pci_dev *pdev = ndev->pdev;
> > + struct pci_epc *epc;
> > + int rc;
> > +
> > + if (!use_remote_edma || nt->remote_edma_mode == REMOTE_EDMA_EP)
> > + return 0;
> > +
> > + /* Only EP side can return pci_epc */
> > + epc = ntb_get_pci_epc(ndev);
> > + if (!epc)
> > + return 0;
> > +
> > + rc = ntb_edma_setup_mws(ndev);
> > + if (rc) {
> > + dev_err(&pdev->dev,
> > + "Failed to set up memory window for eDMA: %d\n", rc);
> > + return rc;
> > + }
> > +
> > + rc = ntb_edma_setup_isr(&pdev->dev, &epc->dev, ntb_transport_edma_isr, nt);
> > + if (rc) {
> > + dev_err(&pdev->dev, "Failed to setup eDMA ISR (%d)\n", rc);
> > + return rc;
> > + }
> > +
> > + nt->remote_edma_mode = REMOTE_EDMA_EP;
> > + return 0;
> > +}
> > +
> > +static int ntb_transport_edma_setup_qp_mw(struct ntb_transport_ctx *nt,
> > + unsigned int qp_num)
> > +{
> > + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
> > + struct ntb_dev *ndev = nt->ndev;
> > + struct ntb_queue_entry *entry;
> > + struct ntb_transport_mw *mw;
> > + unsigned int mw_num, mw_count, qp_count;
> > + unsigned int qp_offset, rx_info_offset;
> > + unsigned int mw_size, mw_size_per_qp;
> > + unsigned int num_qps_mw;
> > + size_t edma_total;
> > + unsigned int i;
> > + int node;
> > +
> > + mw_count = nt->mw_count;
> > + qp_count = nt->qp_count;
> > +
> > + mw_num = QP_TO_MW(nt, qp_num);
> > + mw = &nt->mw_vec[mw_num];
> > +
> > + if (!mw->virt_addr)
> > + return -ENOMEM;
> > +
> > + if (mw_num < qp_count % mw_count)
> > + num_qps_mw = qp_count / mw_count + 1;
> > + else
> > + num_qps_mw = qp_count / mw_count;
> > +
> > + mw_size = min(nt->mw_vec[mw_num].phys_size, mw->xlat_size);
> > + if (max_mw_size && mw_size > max_mw_size)
> > + mw_size = max_mw_size;
> > +
> > + mw_size_per_qp = round_down((unsigned int)mw_size / num_qps_mw, SZ_64);
> > + qp_offset = mw_size_per_qp * (qp_num / mw_count);
> > + rx_info_offset = mw_size_per_qp - sizeof(struct ntb_rx_info);
> > +
> > + qp->tx_mw_size = mw_size_per_qp;
> > + qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;
> > + if (!qp->tx_mw)
> > + return -EINVAL;
> > + qp->tx_mw_phys = nt->mw_vec[mw_num].phys_addr + qp_offset;
> > + if (!qp->tx_mw_phys)
> > + return -EINVAL;
> > + qp->rx_info = qp->tx_mw + rx_info_offset;
> > + qp->rx_buff = mw->virt_addr + qp_offset;
> > + qp->remote_rx_info = qp->rx_buff + rx_info_offset;
> > +
> > + /* Due to housekeeping, there must be at least 2 buffs */
> > + qp->tx_max_frame = min(transport_mtu, mw_size_per_qp / 2);
> > + qp->rx_max_frame = min(transport_mtu, mw_size_per_qp / 2);
> > +
> > + /* In eDMA mode, decouple from MW sizing and force ring-sized entries */
> > + edma_total = 2 * sizeof(struct ntb_edma_ring);
> > + if (rx_info_offset < edma_total) {
> > + dev_err(&ndev->dev, "Ring space requires %luB (>=%uB)\n",
> > + edma_total, rx_info_offset);
> > + return -EINVAL;
> > + }
> > + qp->tx_max_entry = NTB_EDMA_RING_ENTRIES;
> > + qp->rx_max_entry = NTB_EDMA_RING_ENTRIES;
> > +
> > + /*
> > + * Checking to see if we have more entries than the default.
> > + * We should add additional entries if that is the case so we
> > + * can be in sync with the transport frames.
> > + */
> > + node = dev_to_node(&ndev->dev);
> > + for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) {
> > + entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
> > + if (!entry)
> > + return -ENOMEM;
> > +
> > + entry->qp = qp;
> > + INIT_WORK(&entry->dma_work, ntb_transport_edma_rc_dma_work);
> > + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> > + &qp->rx_free_q);
> > + qp->rx_alloc_entry++;
> > + }
> > +
> > + memset(qp->rx_buff, 0, edma_total);
> > +
> > + qp->rx_pkts = 0;
> > + qp->tx_pkts = 0;
> > +
> > + return 0;
> > +}
> > +
> > +static int ntb_transport_edma_ep_read_complete(struct ntb_transport_qp *qp)
> > +{
> > + struct device *dma_dev = get_dma_dev(qp->ndev);
> > + struct ntb_queue_entry *entry;
> > + struct ntb_edma_desc *in;
> > + unsigned int len;
> > + u32 idx;
> > +
> > + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_TAIL_RD_EP_I(qp)),
> > + qp->rd_cons) == 0)
> > + return 0;
> > +
> > + idx = ntb_edma_ring_idx(qp->rd_cons);
> > + in = NTB_DESC_RD_EP_I(qp, idx);
> > + if (!(in->flags & DESC_DONE_FLAG))
> > + return 0;
> > +
> > + in->flags = 0;
> > + len = in->len; /* might be smaller than entry->len */
> > +
> > + entry = (struct ntb_queue_entry *)(in->data);
> > + if (WARN_ON(!entry))
> > + return 0;
> > +
> > + if (in->flags & LINK_DOWN_FLAG) {
> > + ntb_qp_link_down(qp);
> > + qp->rd_cons++;
> > + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> > + return 1;
> > + }
> > +
> > + dma_unmap_single(dma_dev, entry->addr, entry->len, DMA_FROM_DEVICE);
> > +
> > + qp->rx_bytes += len;
> > + qp->rx_pkts++;
> > + qp->rd_cons++;
> > +
> > + if (qp->rx_handler && qp->client_ready)
> > + qp->rx_handler(qp, qp->cb_data, entry->cb_data, len);
> > +
> > + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> > + return 1;
> > +}
> > +
> > +static int ntb_transport_edma_ep_write_complete(struct ntb_transport_qp *qp)
> > +{
> > + struct ntb_queue_entry *entry;
> > + struct ntb_edma_desc *in;
> > + u32 idx;
> > +
> > + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_TAIL_WR_EP_I(qp)),
> > + qp->wr_cons) == 0)
> > + return 0;
> > +
> > + idx = ntb_edma_ring_idx(qp->wr_cons);
> > + in = NTB_DESC_WR_EP_I(qp, idx);
> > +
> > + entry = (struct ntb_queue_entry *)(in->data);
> > + if (WARN_ON(!entry))
> > + return 0;
> > +
> > + qp->wr_cons++;
> > +
> > + if (qp->tx_handler)
> > + qp->tx_handler(qp, qp->cb_data, entry->cb_data, entry->len);
> > +
> > + ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q);
> > + return 1;
> > +}
> > +
> > +static void ntb_transport_edma_ep_read_work(struct work_struct *work)
> > +{
> > + struct ntb_transport_qp *qp = container_of(
> > + work, struct ntb_transport_qp, read_work);
> > + unsigned int i;
> > +
> > + for (i = 0; i < NTB_EDMA_MAX_POLL; i++) {
> > + if (!ntb_transport_edma_ep_read_complete(qp))
> > + break;
> > + }
> > +
> > + if (ntb_transport_edma_ep_read_complete(qp))
> > + queue_work(qp->transport->wq, &qp->read_work);
> > +}
> > +
> > +static void ntb_transport_edma_ep_write_work(struct work_struct *work)
> > +{
> > + struct ntb_transport_qp *qp = container_of(
> > + work, struct ntb_transport_qp, write_work);
> > + unsigned int i;
> > +
> > + for (i = 0; i < NTB_EDMA_MAX_POLL; i++) {
> > + if (!ntb_transport_edma_ep_write_complete(qp))
> > + break;
> > + }
> > +
> > + if (ntb_transport_edma_ep_write_complete(qp))
> > + queue_work(qp->transport->wq, &qp->write_work);
> > +}
> > +
> > +static void ntb_transport_edma_rc_write_complete_work(struct work_struct *work)
> > +{
> > + struct ntb_transport_qp *qp = container_of(
> > + work, struct ntb_transport_qp, write_work);
> > + struct ntb_queue_entry *entry;
> > + struct ntb_edma_desc *in;
> > + unsigned int len;
> > + void *cb_data;
> > + u32 idx;
> > +
> > + while (ntb_edma_ring_used_entry(READ_ONCE(qp->wr_issue),
> > + qp->wr_cons) != 0) {
> > + /* Paired with smp_wmb() in ntb_transport_edma_rc_poll() */
> > + smp_rmb();
> > +
> > + idx = ntb_edma_ring_idx(qp->wr_cons);
> > + in = NTB_DESC_WR_RC_I(qp, idx);
> > + entry = (struct ntb_queue_entry *)READ_ONCE(in->data);
> > + if (!entry || !(entry->flags & DESC_DONE_FLAG))
> > + break;
> > +
> > + in->data = 0;
> > +
> > + cb_data = entry->cb_data;
> > + len = entry->len;
> > +
> > + iowrite32(++qp->wr_cons, NTB_TAIL_WR_RC_O(qp));
> > +
> > + if (unlikely(entry->flags & LINK_DOWN_FLAG)) {
> > + ntb_qp_link_down(qp);
> > + continue;
> > + }
> > +
> > + ntb_transport_edma_notify_peer(qp);
> > +
> > + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> > +
> > + if (qp->rx_handler && qp->client_ready)
> > + qp->rx_handler(qp, qp->cb_data, cb_data, len);
> > +
> > + /* stat updates */
> > + qp->rx_bytes += len;
> > + qp->rx_pkts++;
> > + }
> > +}
> > +
> > +static void ntb_transport_edma_rc_write_cb(void *data,
> > + const struct dmaengine_result *res)
> > +{
> > + struct ntb_queue_entry *entry = data;
> > + struct ntb_transport_qp *qp = entry->qp;
> > + struct ntb_transport_ctx *nt = qp->transport;
> > + enum dmaengine_tx_result dma_err = res->result;
> > + struct device *dma_dev = get_dma_dev(qp->ndev);
> > +
> > + switch (dma_err) {
> > + case DMA_TRANS_READ_FAILED:
> > + case DMA_TRANS_WRITE_FAILED:
> > + case DMA_TRANS_ABORTED:
> > + entry->errors++;
> > + entry->len = -EIO;
> > + break;
> > + case DMA_TRANS_NOERROR:
> > + default:
> > + break;
> > + }
> > + dma_unmap_sg(dma_dev, &entry->sgl, 1, DMA_FROM_DEVICE);
> > + sg_dma_address(&entry->sgl) = 0;
> > +
> > + entry->flags |= DESC_DONE_FLAG;
> > +
> > + queue_work(nt->wq, &qp->write_work);
> > +}
> > +
> > +static void ntb_transport_edma_rc_read_complete_work(struct work_struct *work)
> > +{
> > + struct ntb_transport_qp *qp = container_of(
> > + work, struct ntb_transport_qp, read_work);
> > + struct ntb_edma_desc *in, __iomem *out;
> > + struct ntb_queue_entry *entry;
> > + unsigned int len;
> > + void *cb_data;
> > + u32 idx;
> > +
> > + while (ntb_edma_ring_used_entry(READ_ONCE(qp->rd_issue),
> > + qp->rd_cons) != 0) {
> > + /* Paired with smp_wmb() in ntb_transport_edma_rc_tx_enqueue() */
> > + smp_rmb();
> > +
> > + idx = ntb_edma_ring_idx(qp->rd_cons);
> > + in = NTB_DESC_RD_RC_I(qp, idx);
> > + entry = (struct ntb_queue_entry *)in->data;
> > + if (!entry || !(entry->flags & DESC_DONE_FLAG))
> > + break;
> > +
> > + in->data = 0;
> > +
> > + cb_data = entry->cb_data;
> > + len = entry->len;
> > +
> > + out = NTB_DESC_RD_RC_O(qp, idx);
> > +
> > + WRITE_ONCE(qp->rd_cons, qp->rd_cons + 1);
> > +
> > + /*
> > + * No need to add barrier in-between to enforce ordering here.
> > + * The other side proceeds only after both flags and tail are
> > + * updated.
> > + */
> > + iowrite32(entry->flags, &out->flags);
> > + iowrite32(qp->rd_cons, NTB_TAIL_RD_RC_O(qp));
> > +
> > + ntb_transport_edma_notify_peer(qp);
> > +
> > + ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
> > + &qp->tx_free_q);
> > +
> > + if (qp->tx_handler)
> > + qp->tx_handler(qp, qp->cb_data, cb_data, len);
> > +
> > + /* stat updates */
> > + qp->tx_bytes += len;
> > + qp->tx_pkts++;
> > + }
> > +}
> > +
> > +static void ntb_transport_edma_rc_read_cb(void *data,
> > + const struct dmaengine_result *res)
> > +{
> > + struct ntb_queue_entry *entry = data;
> > + struct ntb_transport_qp *qp = entry->qp;
> > + struct ntb_transport_ctx *nt = qp->transport;
> > + struct device *dma_dev = get_dma_dev(qp->ndev);
> > + enum dmaengine_tx_result dma_err = res->result;
> > +
> > + switch (dma_err) {
> > + case DMA_TRANS_READ_FAILED:
> > + case DMA_TRANS_WRITE_FAILED:
> > + case DMA_TRANS_ABORTED:
> > + entry->errors++;
> > + entry->len = -EIO;
> > + break;
> > + case DMA_TRANS_NOERROR:
> > + default:
> > + break;
> > + }
> > + dma_unmap_sg(dma_dev, &entry->sgl, 1, DMA_TO_DEVICE);
> > + sg_dma_address(&entry->sgl) = 0;
> > +
> > + entry->flags |= DESC_DONE_FLAG;
> > +
> > + queue_work(nt->wq, &qp->read_work);
> > +}
> > +
> > +static int ntb_transport_edma_rc_write_start(struct device *d,
> > + struct dma_chan *chan, size_t len,
> > + dma_addr_t ep_src, void *rc_dst,
> > + struct ntb_queue_entry *entry)
> > +{
> > + struct scatterlist *sgl = &entry->sgl;
> > + struct dma_async_tx_descriptor *txd;
> > + struct dma_slave_config cfg;
> > + dma_cookie_t cookie;
> > + int nents, rc;
> > +
> > + if (!d)
> > + return -ENODEV;
> > +
> > + if (!chan)
> > + return -ENXIO;
> > +
> > + if (WARN_ON(!ep_src || !rc_dst))
> > + return -EINVAL;
> > +
> > + if (WARN_ON(sg_dma_address(sgl)))
> > + return -EINVAL;
> > +
> > + sg_init_one(sgl, rc_dst, len);
> > + nents = dma_map_sg(d, sgl, 1, DMA_FROM_DEVICE);
> > + if (nents <= 0)
> > + return -EIO;
> > +
> > + memset(&cfg, 0, sizeof(cfg));
> > + cfg.src_addr = ep_src;
> > + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> > + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> > + cfg.direction = DMA_DEV_TO_MEM;
> > + rc = dmaengine_slave_config(chan, &cfg);
> > + if (rc)
> > + goto out_unmap;
> > +
> > + txd = dmaengine_prep_slave_sg(chan, sgl, 1, DMA_DEV_TO_MEM,
> > + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> > + if (!txd) {
> > + rc = -EIO;
> > + goto out_unmap;
> > + }
> > +
> > + txd->callback_result = ntb_transport_edma_rc_write_cb;
> > + txd->callback_param = entry;
> > +
> > + cookie = dmaengine_submit(txd);
> > + if (dma_submit_error(cookie)) {
> > + rc = -EIO;
> > + goto out_unmap;
> > + }
> > + dma_async_issue_pending(chan);
> > + return 0;
> > +out_unmap:
> > + dma_unmap_sg(d, sgl, 1, DMA_FROM_DEVICE);
> > + return rc;
> > +}
> > +
> > +static int ntb_transport_edma_rc_read_start(struct device *d,
> > + struct dma_chan *chan, size_t len,
> > + void *rc_src, dma_addr_t ep_dst,
> > + struct ntb_queue_entry *entry)
> > +{
> > + struct scatterlist *sgl = &entry->sgl;
> > + struct dma_async_tx_descriptor *txd;
> > + struct dma_slave_config cfg;
> > + dma_cookie_t cookie;
> > + int nents, rc;
> > +
> > + if (!d)
> > + return -ENODEV;
> > +
> > + if (!chan)
> > + return -ENXIO;
> > +
> > + if (WARN_ON(!rc_src || !ep_dst))
> > + return -EINVAL;
> > +
> > + if (WARN_ON(sg_dma_address(sgl)))
> > + return -EINVAL;
> > +
> > + sg_init_one(sgl, rc_src, len);
> > + nents = dma_map_sg(d, sgl, 1, DMA_TO_DEVICE);
> > + if (nents <= 0)
> > + return -EIO;
> > +
> > + memset(&cfg, 0, sizeof(cfg));
> > + cfg.dst_addr = ep_dst;
> > + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> > + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> > + cfg.direction = DMA_MEM_TO_DEV;
> > + rc = dmaengine_slave_config(chan, &cfg);
> > + if (rc)
> > + goto out_unmap;
> > +
> > + txd = dmaengine_prep_slave_sg(chan, sgl, 1, DMA_MEM_TO_DEV,
> > + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> > + if (!txd) {
> > + rc = -EIO;
> > + goto out_unmap;
> > + }
> > +
> > + txd->callback_result = ntb_transport_edma_rc_read_cb;
> > + txd->callback_param = entry;
> > +
> > + cookie = dmaengine_submit(txd);
> > + if (dma_submit_error(cookie)) {
> > + rc = -EIO;
> > + goto out_unmap;
> > + }
> > + dma_async_issue_pending(chan);
> > + return 0;
> > +out_unmap:
> > + dma_unmap_sg(d, sgl, 1, DMA_TO_DEVICE);
> > + return rc;
> > +}
> > +
> > +static void ntb_transport_edma_rc_dma_work(struct work_struct *work)
> > +{
> > + struct ntb_queue_entry *entry = container_of(
> > + work, struct ntb_queue_entry, dma_work);
> > + struct ntb_transport_qp *qp = entry->qp;
> > + struct ntb_transport_ctx *nt = qp->transport;
> > + struct device *dma_dev = get_dma_dev(qp->ndev);
> > + struct dma_chan *chan;
> > + int rc;
> > +
> > + chan = ntb_edma_pick_chan(&nt->edma, REMOTE_EDMA_WRITE);
> > + rc = ntb_transport_edma_rc_write_start(dma_dev, chan, entry->len,
> > + entry->addr, entry->buf, entry);
> > + if (rc) {
> > + entry->errors++;
> > + entry->len = -EIO;
> > + entry->flags |= DESC_DONE_FLAG;
> > + queue_work(nt->wq, &qp->write_work);
> > + return;
> > + }
> > +}
> > +
> > +static void ntb_transport_edma_rc_poll(struct ntb_transport_qp *qp)
> > +{
> > + struct ntb_transport_ctx *nt = qp->transport;
> > + unsigned int budget = NTB_EDMA_MAX_POLL;
> > + struct ntb_queue_entry *entry;
> > + struct ntb_edma_desc *in;
> > + dma_addr_t ep_src;
> > + u32 len, idx;
> > +
> > + while (budget--) {
> > + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_HEAD_WR_RC_I(qp)),
> > + qp->wr_issue) == 0)
> > + break;
> > +
> > + idx = ntb_edma_ring_idx(qp->wr_issue);
> > + in = NTB_DESC_WR_RC_I(qp, idx);
> > +
> > + len = READ_ONCE(in->len);
> > + ep_src = (dma_addr_t)READ_ONCE(in->addr);
> > +
> > + /* Prepare 'entry' for write completion */
> > + entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q);
> > + if (!entry) {
> > + qp->rx_err_no_buf++;
> > + break;
> > + }
> > + if (WARN_ON(entry->flags & DESC_DONE_FLAG))
> > + entry->flags &= ~DESC_DONE_FLAG;
> > + entry->len = len; /* NB. entry->len can be <=0 */
> > + entry->addr = ep_src;
> > +
> > + /*
> > + * ntb_transport_edma_rc_write_complete_work() checks entry->flags
> > + * so it needs to be set before wr_issue++.
> > + */
> > + in->data = (uintptr_t)entry;
> > +
> > + /* Ensure in->data visible before wr_issue++ */
> > + smp_wmb();
> > +
> > + WRITE_ONCE(qp->wr_issue, qp->wr_issue + 1);
> > +
> > + if (!len) {
> > + entry->flags |= DESC_DONE_FLAG;
> > + queue_work(nt->wq, &qp->write_work);
> > + continue;
> > + }
> > +
> > + if (in->flags & LINK_DOWN_FLAG) {
> > + dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n");
> > + entry->flags |= DESC_DONE_FLAG | LINK_DOWN_FLAG;
> > + queue_work(nt->wq, &qp->write_work);
> > + continue;
> > + }
> > +
> > + queue_work(nt->wq, &entry->dma_work);
> > + }
> > +
> > + if (!budget)
> > + tasklet_schedule(&qp->rxc_db_work);
> > +}
> > +
> > +static int ntb_transport_edma_rc_tx_enqueue(struct ntb_transport_qp *qp,
> > + struct ntb_queue_entry *entry)
> > +{
> > + struct device *dma_dev = get_dma_dev(qp->ndev);
> > + struct ntb_transport_ctx *nt = qp->transport;
> > + struct ntb_edma_desc *in, __iomem *out;
> > + unsigned int len = entry->len;
> > + struct dma_chan *chan;
> > + u32 issue, idx, head;
> > + dma_addr_t ep_dst;
> > + int rc;
> > +
> > + WARN_ON_ONCE(entry->flags & DESC_DONE_FLAG);
> > +
> > + scoped_guard(spinlock_irqsave, &qp->rc_lock) {
> > + head = READ_ONCE(*NTB_HEAD_RD_RC_I(qp));
> > + issue = qp->rd_issue;
> > + if (ntb_edma_ring_used_entry(head, issue) == 0) {
> > + qp->tx_ring_full++;
> > + return -ENOSPC;
> > + }
> > +
> > + /*
> > + * ntb_transport_edma_rc_read_complete_work() checks entry->flags
> > + * so it needs to be set before rd_issue++.
> > + */
> > + idx = ntb_edma_ring_idx(issue);
> > + in = NTB_DESC_RD_RC_I(qp, idx);
> > + in->data = (uintptr_t)entry;
> > +
> > + /* Make in->data visible before rd_issue++ */
> > + smp_wmb();
> > +
> > + WRITE_ONCE(qp->rd_issue, qp->rd_issue + 1);
> > + }
> > +
> > + /* Publish the final transfer length to the EP side */
> > + out = NTB_DESC_RD_RC_O(qp, idx);
> > + iowrite32(len, &out->len);
> > + ioread32(&out->len);
> > +
> > + if (unlikely(!len)) {
> > + entry->flags |= DESC_DONE_FLAG;
> > + queue_work(nt->wq, &qp->read_work);
> > + return 0;
> > + }
> > +
> > + /* Paired with dma_wmb() in ntb_transport_edma_ep_rx_enqueue() */
> > + dma_rmb();
> > +
> > + /* kick remote eDMA read transfer */
> > + ep_dst = (dma_addr_t)in->addr;
> > + chan = ntb_edma_pick_chan(&nt->edma, REMOTE_EDMA_READ);
> > + rc = ntb_transport_edma_rc_read_start(dma_dev, chan, len,
> > + entry->buf, ep_dst, entry);
> > + if (rc) {
> > + entry->errors++;
> > + entry->len = -EIO;
> > + entry->flags |= DESC_DONE_FLAG;
> > + queue_work(nt->wq, &qp->read_work);
> > + }
> > + return 0;
> > +}
> > +
> > +static int ntb_transport_edma_ep_tx_enqueue(struct ntb_transport_qp *qp,
> > + struct ntb_queue_entry *entry)
> > +{
> > + struct device *dma_dev = get_dma_dev(qp->ndev);
> > + struct ntb_edma_desc *in, __iomem *out;
> > + unsigned int len = entry->len;
> > + dma_addr_t ep_src = 0;
> > + u32 idx;
> > + int rc;
> > +
> > + if (likely(len)) {
> > + ep_src = dma_map_single(dma_dev, entry->buf, len,
> > + DMA_TO_DEVICE);
> > + rc = dma_mapping_error(dma_dev, ep_src);
> > + if (rc)
> > + return rc;
> > + }
> > +
> > + scoped_guard(spinlock_irqsave, &qp->ep_tx_lock) {
> > + if (ntb_edma_ring_full(qp->wr_prod, qp->wr_cons)) {
> > + rc = -ENOSPC;
> > + qp->tx_ring_full++;
> > + goto out_unmap;
> > + }
> > +
> > + idx = ntb_edma_ring_idx(qp->wr_prod);
> > + in = NTB_DESC_WR_EP_I(qp, idx);
> > + out = NTB_DESC_WR_EP_O(qp, idx);
> > +
> > + WARN_ON(in->flags & DESC_DONE_FLAG);
> > + WARN_ON(entry->flags & DESC_DONE_FLAG);
> > + in->flags = 0;
> > + in->data = (uintptr_t)entry;
> > + entry->addr = ep_src;
> > +
> > + iowrite32(len, &out->len);
> > + iowrite32(entry->flags, &out->flags);
> > + iowrite64(ep_src, &out->addr);
> > + WRITE_ONCE(qp->wr_prod, qp->wr_prod + 1);
> > +
> > + dma_wmb();
> > + iowrite32(qp->wr_prod, NTB_HEAD_WR_EP_O(qp));
> > +
> > + qp->tx_bytes += len;
> > + qp->tx_pkts++;
> > + }
> > +
> > + ntb_transport_edma_notify_peer(qp);
> > +
> > + return 0;
> > +out_unmap:
> > + if (likely(len))
> > + dma_unmap_single(dma_dev, ep_src, len, DMA_TO_DEVICE);
> > + return rc;
> > +}
> > +
> > +static int ntb_transport_edma_tx_enqueue(struct ntb_transport_qp *qp,
> > + struct ntb_queue_entry *entry,
> > + void *cb, void *data, unsigned int len,
> > + unsigned int flags)
> > +{
> > + struct device *dma_dev;
> > +
> > + if (entry->addr) {
> > + /* Deferred unmap */
> > + dma_dev = get_dma_dev(qp->ndev);
> > + dma_unmap_single(dma_dev, entry->addr, entry->len, DMA_TO_DEVICE);
> > + }
> > +
> > + entry->cb_data = cb;
> > + entry->buf = data;
> > + entry->len = len;
> > + entry->flags = flags;
> > + entry->errors = 0;
> > + entry->addr = 0;
> > +
> > + WARN_ON_ONCE(!ntb_qp_edma_enabled(qp));
> > +
> > + if (ntb_qp_edma_is_ep(qp))
> > + return ntb_transport_edma_ep_tx_enqueue(qp, entry);
> > + else
> > + return ntb_transport_edma_rc_tx_enqueue(qp, entry);
> > +}
> > +
> > +static int ntb_transport_edma_ep_rx_enqueue(struct ntb_transport_qp *qp,
> > + struct ntb_queue_entry *entry)
> > +{
> > + struct device *dma_dev = get_dma_dev(qp->ndev);
> > + struct ntb_edma_desc *in, __iomem *out;
> > + unsigned int len = entry->len;
> > + void *data = entry->buf;
> > + dma_addr_t ep_dst;
> > + u32 idx;
> > + int rc;
> > +
> > + ep_dst = dma_map_single(dma_dev, data, len, DMA_FROM_DEVICE);
> > + rc = dma_mapping_error(dma_dev, ep_dst);
> > + if (rc)
> > + return rc;
> > +
> > + scoped_guard(spinlock_bh, &qp->ep_rx_lock) {
> > + if (ntb_edma_ring_full(READ_ONCE(qp->rd_prod),
> > + READ_ONCE(qp->rd_cons))) {
> > + rc = -ENOSPC;
> > + goto out_unmap;
> > + }
> > +
> > + idx = ntb_edma_ring_idx(qp->rd_prod);
> > + in = NTB_DESC_RD_EP_I(qp, idx);
> > + out = NTB_DESC_RD_EP_O(qp, idx);
> > +
> > + iowrite32(len, &out->len);
> > + iowrite64(ep_dst, &out->addr);
> > +
> > + WARN_ON(in->flags & DESC_DONE_FLAG);
> > + in->data = (uintptr_t)entry;
> > + entry->addr = ep_dst;
> > +
> > + /* Ensure len/addr are visible before the head update */
> > + dma_wmb();
> > +
> > + WRITE_ONCE(qp->rd_prod, qp->rd_prod + 1);
> > + iowrite32(qp->rd_prod, NTB_HEAD_RD_EP_O(qp));
> > + }
> > + return 0;
> > +out_unmap:
> > + dma_unmap_single(dma_dev, ep_dst, len, DMA_FROM_DEVICE);
> > + return rc;
> > +}
> > +
> > +static int ntb_transport_edma_rx_enqueue(struct ntb_transport_qp *qp,
> > + struct ntb_queue_entry *entry)
> > +{
> > + int rc;
> > +
> > + /* The behaviour is the same as the default backend for RC side */
> > + if (ntb_qp_edma_is_ep(qp)) {
> > + rc = ntb_transport_edma_ep_rx_enqueue(qp, entry);
> > + if (rc) {
> > + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> > + &qp->rx_free_q);
> > + return rc;
> > + }
> > + }
> > +
> > + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q);
> > +
> > + if (qp->active)
> > + tasklet_schedule(&qp->rxc_db_work);
> > +
> > + return 0;
> > +}
> > +
> > +static void ntb_transport_edma_rx_poll(struct ntb_transport_qp *qp)
> > +{
> > + struct ntb_transport_ctx *nt = qp->transport;
> > +
> > + if (ntb_qp_edma_is_rc(qp))
> > + ntb_transport_edma_rc_poll(qp);
> > + else if (ntb_qp_edma_is_ep(qp)) {
> > + /*
> > + * Make sure we poll the rings even if an eDMA interrupt is
> > + * cleared on the RC side earlier.
> > + */
> > + queue_work(nt->wq, &qp->read_work);
> > + queue_work(nt->wq, &qp->write_work);
> > + } else
> > + /* Unreachable */
> > + WARN_ON_ONCE(1);
> > +}
> > +
> > +static void ntb_transport_edma_read_work(struct work_struct *work)
> > +{
> > + struct ntb_transport_qp *qp = container_of(
> > + work, struct ntb_transport_qp, read_work);
> > +
> > + if (ntb_qp_edma_is_rc(qp))
> > + ntb_transport_edma_rc_read_complete_work(work);
> > + else if (ntb_qp_edma_is_ep(qp))
> > + ntb_transport_edma_ep_read_work(work);
> > + else
> > + /* Unreachable */
> > + WARN_ON_ONCE(1);
> > +}
> > +
> > +static void ntb_transport_edma_write_work(struct work_struct *work)
> > +{
> > + struct ntb_transport_qp *qp = container_of(
> > + work, struct ntb_transport_qp, write_work);
> > +
> > + if (ntb_qp_edma_is_rc(qp))
> > + ntb_transport_edma_rc_write_complete_work(work);
> > + else if (ntb_qp_edma_is_ep(qp))
> > + ntb_transport_edma_ep_write_work(work);
> > + else
> > + /* Unreachable */
> > + WARN_ON_ONCE(1);
> > +}
> > +
> > +static void ntb_transport_edma_init_queue(struct ntb_transport_ctx *nt,
> > + unsigned int qp_num)
> > +{
> > + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
> > +
> > + qp->wr_cons = 0;
> > + qp->rd_cons = 0;
> > + qp->wr_prod = 0;
> > + qp->rd_prod = 0;
> > + qp->wr_issue = 0;
> > + qp->rd_issue = 0;
> > +
> > + INIT_WORK(&qp->db_work, ntb_transport_edma_db_work);
> > + INIT_WORK(&qp->read_work, ntb_transport_edma_read_work);
> > + INIT_WORK(&qp->write_work, ntb_transport_edma_write_work);
> > +}
> > +
> > +static void ntb_transport_edma_create_queue(struct ntb_transport_ctx *nt,
> > + struct ntb_transport_qp *qp)
> > +{
> > + spin_lock_init(&qp->ep_tx_lock);
> > + spin_lock_init(&qp->ep_rx_lock);
> > + spin_lock_init(&qp->rc_lock);
> > +}
> > +
> > +static const struct ntb_transport_backend_ops edma_backend_ops = {
> > + .setup_qp_mw = ntb_transport_edma_setup_qp_mw,
> > + .tx_free_entry = ntb_transport_edma_tx_free_entry,
> > + .tx_enqueue = ntb_transport_edma_tx_enqueue,
> > + .rx_enqueue = ntb_transport_edma_rx_enqueue,
> > + .rx_poll = ntb_transport_edma_rx_poll,
> > + .debugfs_stats_show = ntb_transport_edma_debugfs_stats_show,
> > +};
> > +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> > +
> > /**
> > * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
> > * @qp: NTB transport layer queue to be enabled
>
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-12-02 6:59 ` Koichiro Den
@ 2025-12-02 14:53 ` Dave Jiang
2025-12-03 14:19 ` Koichiro Den
0 siblings, 1 reply; 19+ messages in thread
From: Dave Jiang @ 2025-12-02 14:53 UTC (permalink / raw)
To: Koichiro Den
Cc: ntb, linux-pci, dmaengine, linux-kernel, Frank.Li, mani,
kwilczynski, kishon, bhelgaas, corbet, vkoul, jdmason, allenbh,
Basavaraj.Natikar, Shyam-sundar.S-k, kurt.schwemmer, logang,
jingoohan1, lpieralisi, robh, jbrunet, fancer.lancer, arnd,
pstanner, elfring
On 12/1/25 11:59 PM, Koichiro Den wrote:
> On Mon, Dec 01, 2025 at 02:46:41PM -0700, Dave Jiang wrote:
>>
>>
>> On 11/29/25 9:03 AM, Koichiro Den wrote:
>>> Add a new transport backend that uses a remote DesignWare eDMA engine
>>> located on the NTB endpoint to move data between host and endpoint.
>>>
>>> In this mode:
>>>
>>> - The endpoint exposes a dedicated memory window that contains the
>>> eDMA register block followed by a small control structure (struct
>>> ntb_edma_info) and per-channel linked-list (LL) rings.
>>>
>>> - On the endpoint side, ntb_edma_setup_mws() allocates the control
>>> structure and LL rings in endpoint memory, then programs an inbound
>>> iATU region so that the host can access them via a peer MW.
>>>
>>> - On the host side, ntb_edma_setup_peer() ioremaps the peer MW, reads
>>> ntb_edma_info and configures a dw-edma DMA device to use the LL
>>> rings provided by the endpoint.
>>>
>>> - ntb_transport is extended with a new backend_ops implementation that
>>> routes TX and RX enqueue/poll operations through the remote eDMA
>>> rings while keeping the existing shared-memory backend intact.
>>>
>>> - The host signals the endpoint via a dedicated DMA read channel.
>>> 'use_msi' module option is ignored when 'use_remote_edma=1'.
>>>
>>> The new mode is guarded by a Kconfig option (NTB_TRANSPORT_EDMA) and a
>>> module parameter (use_remote_edma). When disabled, the existing
>>> ntb_transport behaviour is unchanged.
>>>
>>> Signed-off-by: Koichiro Den <den@valinux.co.jp>
>>> ---
>>> drivers/ntb/Kconfig | 11 +
>>> drivers/ntb/Makefile | 3 +
>>> drivers/ntb/ntb_edma.c | 628 ++++++++
>>> drivers/ntb/ntb_edma.h | 128 ++
>>
>> I briefly looked over the code. It feels like the EDMA bits should go in drivers/ntb/hw/ rather than drivers/ntb/ given it's pretty specific to the designware hardware. What sits in drivers/ntb should be generic APIs where a different vendor can utilize it and not having to adopt to designware hardware specifics. So maybe a bit more abstractions are needed?
>
> That makes sense, I'll reorganize things. Thank you for the suggestion.
Also, since a new transport is being introduced. Please update Documentation/driver-api/ntb.rst. While the current documentation doesn't provide adaquate API documentation for ntb_transport APIs, hopefully the new transport can do better going forward. :) Thank you!
DJ
>
>>
>>> .../{ntb_transport.c => ntb_transport_core.c} | 1281 ++++++++++++++++-
>>> 5 files changed, 2048 insertions(+), 3 deletions(-)
>>> create mode 100644 drivers/ntb/ntb_edma.c
>>> create mode 100644 drivers/ntb/ntb_edma.h
>>> rename drivers/ntb/{ntb_transport.c => ntb_transport_core.c} (65%)
>>>
>>> diff --git a/drivers/ntb/Kconfig b/drivers/ntb/Kconfig
>>> index df16c755b4da..db63f02bb116 100644
>>> --- a/drivers/ntb/Kconfig
>>> +++ b/drivers/ntb/Kconfig
>>> @@ -37,4 +37,15 @@ config NTB_TRANSPORT
>>>
>>> If unsure, say N.
>>>
>>> +config NTB_TRANSPORT_EDMA
>>> + bool "NTB Transport backed by remote eDMA"
>>> + depends on NTB_TRANSPORT
>>> + depends on PCI
>>> + select DMA_ENGINE
>>> + help
>>> + Enable a transport backend that uses a remote DesignWare eDMA engine
>>> + exposed through a dedicated NTB memory window. The host uses the
>>> + endpoint's eDMA engine to move data in both directions.
>>> + Say Y here if you intend to use the 'use_remote_edma' module parameter.
>>> +
>>> endif # NTB
>>> diff --git a/drivers/ntb/Makefile b/drivers/ntb/Makefile
>>> index 3a6fa181ff99..51f0e1e3aec7 100644
>>> --- a/drivers/ntb/Makefile
>>> +++ b/drivers/ntb/Makefile
>>> @@ -4,3 +4,6 @@ obj-$(CONFIG_NTB_TRANSPORT) += ntb_transport.o
>>>
>>> ntb-y := core.o
>>> ntb-$(CONFIG_NTB_MSI) += msi.o
>>> +
>>> +ntb_transport-y := ntb_transport_core.o
>>> +ntb_transport-$(CONFIG_NTB_TRANSPORT_EDMA) += ntb_edma.o
>>> diff --git a/drivers/ntb/ntb_edma.c b/drivers/ntb/ntb_edma.c
>>> new file mode 100644
>>> index 000000000000..cb35e0d56aa8
>>> --- /dev/null
>>> +++ b/drivers/ntb/ntb_edma.c
>>> @@ -0,0 +1,628 @@
>>> +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
>>> +
>>> +#include <linux/module.h>
>>> +#include <linux/device.h>
>>> +#include <linux/pci.h>
>>> +#include <linux/ntb.h>
>>> +#include <linux/io.h>
>>> +#include <linux/iommu.h>
>>> +#include <linux/dmaengine.h>
>>> +#include <linux/pci-epc.h>
>>> +#include <linux/dma/edma.h>
>>> +#include <linux/irq.h>
>>> +#include <linux/irqdomain.h>
>>> +#include <linux/of.h>
>>> +#include <linux/of_irq.h>
>>> +#include <dt-bindings/interrupt-controller/arm-gic.h>
>>> +
>>> +#include "ntb_edma.h"
>>> +
>>> +/*
>>> + * The interrupt register offsets below are taken from the DesignWare
>>> + * eDMA "unrolled" register map (EDMA_MF_EDMA_UNROLL). The remote eDMA
>>> + * backend currently only supports this layout.
>>> + */
>>> +#define DMA_WRITE_INT_STATUS_OFF 0x4c
>>> +#define DMA_WRITE_INT_MASK_OFF 0x54
>>> +#define DMA_WRITE_INT_CLEAR_OFF 0x58
>>> +#define DMA_READ_INT_STATUS_OFF 0xa0
>>> +#define DMA_READ_INT_MASK_OFF 0xa8
>>> +#define DMA_READ_INT_CLEAR_OFF 0xac
>>> +
>>> +#define NTB_EDMA_NOTIFY_MAX_QP 64
>>> +
>>> +static unsigned int edma_spi = 417; /* 0x1a1 */
>>> +module_param(edma_spi, uint, 0644);
>>> +MODULE_PARM_DESC(edma_spi, "SPI number used by remote eDMA interrupt (EP local)");
>>> +
>>> +static u64 edma_regs_phys = 0xe65d5000;
>>> +module_param(edma_regs_phys, ullong, 0644);
>>> +MODULE_PARM_DESC(edma_regs_phys, "Physical base address of local eDMA registers (EP)");
>>> +
>>> +static unsigned long edma_regs_size = 0x1200;
>>> +module_param(edma_regs_size, ulong, 0644);
>>> +MODULE_PARM_DESC(edma_regs_size, "Size of the local eDMA register space (EP)");
>>> +
>>> +struct ntb_edma_intr {
>>> + u32 db[NTB_EDMA_NOTIFY_MAX_QP];
>>> +};
>>> +
>>> +struct ntb_edma_ctx {
>>> + void *ll_wr_virt[EDMA_WR_CH_NUM];
>>> + dma_addr_t ll_wr_phys[EDMA_WR_CH_NUM];
>>> + void *ll_rd_virt[EDMA_RD_CH_NUM + 1];
>>> + dma_addr_t ll_rd_phys[EDMA_RD_CH_NUM + 1];
>>> +
>>> + struct ntb_edma_intr *intr_ep_virt;
>>> + dma_addr_t intr_ep_phys;
>>> + struct ntb_edma_intr *intr_rc_virt;
>>> + dma_addr_t intr_rc_phys;
>>> + u32 notify_qp_max;
>>> +
>>> + bool initialized;
>>> +};
>>> +
>>> +static struct ntb_edma_ctx edma_ctx;
>>> +
>>> +typedef void (*ntb_edma_interrupt_cb_t)(void *data, int qp_num);
>>> +
>>> +struct ntb_edma_interrupt {
>>> + int virq;
>>> + void __iomem *base;
>>> + ntb_edma_interrupt_cb_t cb;
>>> + void *data;
>>> +};
>>> +
>>> +static struct ntb_edma_interrupt ntb_edma_intr;
>>> +
>>> +static int ntb_edma_map_spi_to_virq(struct device *dev, unsigned int spi)
>>> +{
>>> + struct device_node *np = dev_of_node(dev);
>>> + struct device_node *parent;
>>> + struct irq_fwspec fwspec = { 0 };
>>> + int virq;
>>> +
>>> + parent = of_irq_find_parent(np);
>>> + if (!parent)
>>> + return -ENODEV;
>>> +
>>> + fwspec.fwnode = of_fwnode_handle(parent);
>>> + fwspec.param_count = 3;
>>> + fwspec.param[0] = GIC_SPI;
>>> + fwspec.param[1] = spi;
>>> + fwspec.param[2] = IRQ_TYPE_LEVEL_HIGH;
>>> +
>>> + virq = irq_create_fwspec_mapping(&fwspec);
>>> + of_node_put(parent);
>>> + return (virq > 0) ? virq : -EINVAL;
>>> +}
>>> +
>>> +static irqreturn_t ntb_edma_isr(int irq, void *data)
>>> +{
>>> + struct ntb_edma_interrupt *v = data;
>>> + u32 mask = BIT(EDMA_RD_CH_NUM);
>>> + u32 i, val;
>>> +
>>> + /*
>>> + * We do not ack interrupts here but instead we mask all local interrupt
>>> + * sources except the read channel used for notification. This reduces
>>> + * needless ISR invocations.
>>> + *
>>> + * In theory we could configure LIE=1/RIE=0 only for the notification
>>> + * transfer (keeping all other channels at LIE=1/RIE=1), but that would
>>> + * require intrusive changes to the dw-edma core.
>>> + *
>>> + * Note: The host side may have already cleared the read interrupt used
>>> + * for notification, so reading DMA_READ_INT_CLEAR_OFF is not a reliable
>>> + * way to detect it. As a result, we cannot reliably tell which specific
>>> + * channel triggered this interrupt. intr_ep_virt->db[i] teaches us
>>> + * instead.
>>> + */
>>> + iowrite32(~0x0, v->base + DMA_WRITE_INT_MASK_OFF);
>>> + iowrite32(~mask, v->base + DMA_READ_INT_MASK_OFF);
>>> +
>>> + if (!v->cb || !edma_ctx.intr_ep_virt)
>>> + return IRQ_HANDLED;
>>> +
>>> + for (i = 0; i < edma_ctx.notify_qp_max; i++) {
>>> + val = READ_ONCE(edma_ctx.intr_ep_virt->db[i]);
>>> + if (!val)
>>> + continue;
>>> +
>>> + WRITE_ONCE(edma_ctx.intr_ep_virt->db[i], 0);
>>> + v->cb(v->data, i);
>>> + }
>>> +
>>> + return IRQ_HANDLED;
>>> +}
>>> +
>>> +int ntb_edma_setup_isr(struct device *dev, struct device *epc_dev,
>>> + ntb_edma_interrupt_cb_t cb, void *data)
>>> +{
>>> + struct ntb_edma_interrupt *v = &ntb_edma_intr;
>>> + int virq = ntb_edma_map_spi_to_virq(epc_dev->parent, edma_spi);
>>> + int ret;
>>> +
>>> + if (virq < 0) {
>>> + dev_err(dev, "failed to get virq (%d)\n", virq);
>>> + return virq;
>>> + }
>>> +
>>> + v->virq = virq;
>>> + v->cb = cb;
>>> + v->data = data;
>>> + if (edma_regs_phys && !v->base)
>>> + v->base = devm_ioremap(dev, edma_regs_phys, edma_regs_size);
>>> + if (!v->base) {
>>> + dev_err(dev, "failed to setup v->base\n");
>>> + return -1;
>>> + }
>>> + ret = devm_request_irq(dev, v->virq, ntb_edma_isr, 0, "ntb-edma", v);
>>> + if (ret)
>>> + return ret;
>>> +
>>> + if (v->base) {
>>> + iowrite32(0x0, v->base + DMA_WRITE_INT_MASK_OFF);
>>> + iowrite32(0x0, v->base + DMA_READ_INT_MASK_OFF);
>>> + }
>>> + return 0;
>>> +}
>>> +
>>> +void ntb_edma_teardown_isr(struct device *dev)
>>> +{
>>> + struct ntb_edma_interrupt *v = &ntb_edma_intr;
>>> +
>>> + /* Mask all write/read interrupts so we don't get called again. */
>>> + if (v->base) {
>>> + iowrite32(~0x0, v->base + DMA_WRITE_INT_MASK_OFF);
>>> + iowrite32(~0x0, v->base + DMA_READ_INT_MASK_OFF);
>>> + }
>>> +
>>> + if (v->virq > 0)
>>> + devm_free_irq(dev, v->virq, v);
>>> +
>>> + if (v->base)
>>> + devm_iounmap(dev, v->base);
>>> +
>>> + v->virq = 0;
>>> + v->cb = NULL;
>>> + v->data = NULL;
>>> +}
>>> +
>>> +int ntb_edma_setup_mws(struct ntb_dev *ndev)
>>> +{
>>> + const size_t info_bytes = PAGE_SIZE;
>>> + resource_size_t size_max, offset;
>>> + dma_addr_t intr_phys, info_phys;
>>> + u32 wr_done = 0, rd_done = 0;
>>> + struct ntb_edma_intr *intr;
>>> + struct ntb_edma_info *info;
>>> + int peer_mw, mw_index, rc;
>>> + struct iommu_domain *dom;
>>> + bool reg_mapped = false;
>>> + size_t ll_bytes, size;
>>> + struct pci_epc *epc;
>>> + struct device *dev;
>>> + unsigned long iova;
>>> + phys_addr_t phys;
>>> + u64 need;
>>> + u32 i;
>>> +
>>> + /* +1 is for interruption */
>>> + ll_bytes = (EDMA_WR_CH_NUM + EDMA_RD_CH_NUM + 1) * DMA_LLP_MEM_SIZE;
>>> + need = EDMA_REG_SIZE + info_bytes + ll_bytes;
>>> +
>>> + epc = ntb_get_pci_epc(ndev);
>>> + if (!epc)
>>> + return -ENODEV;
>>> + dev = epc->dev.parent;
>>> +
>>> + if (edma_ctx.initialized)
>>> + return 0;
>>> +
>>> + info = dma_alloc_coherent(dev, info_bytes, &info_phys, GFP_KERNEL);
>>> + if (!info)
>>> + return -ENOMEM;
>>> +
>>> + memset(info, 0, info_bytes);
>>> + info->magic = NTB_EDMA_INFO_MAGIC;
>>> + info->wr_cnt = EDMA_WR_CH_NUM;
>>> + info->rd_cnt = EDMA_RD_CH_NUM + 1; /* +1 for interruption */
>>> + info->regs_phys = edma_regs_phys;
>>> + info->ll_stride = DMA_LLP_MEM_SIZE;
>>> +
>>> + for (i = 0; i < EDMA_WR_CH_NUM; i++) {
>>> + edma_ctx.ll_wr_virt[i] = dma_alloc_attrs(dev, DMA_LLP_MEM_SIZE,
>>> + &edma_ctx.ll_wr_phys[i],
>>> + GFP_KERNEL,
>>> + DMA_ATTR_FORCE_CONTIGUOUS);
>>> + if (!edma_ctx.ll_wr_virt[i]) {
>>> + rc = -ENOMEM;
>>> + goto err_free_ll;
>>> + }
>>> + wr_done++;
>>> + info->ll_wr_phys[i] = edma_ctx.ll_wr_phys[i];
>>> + }
>>> + for (i = 0; i < EDMA_RD_CH_NUM + 1; i++) {
>>> + edma_ctx.ll_rd_virt[i] = dma_alloc_attrs(dev, DMA_LLP_MEM_SIZE,
>>> + &edma_ctx.ll_rd_phys[i],
>>> + GFP_KERNEL,
>>> + DMA_ATTR_FORCE_CONTIGUOUS);
>>> + if (!edma_ctx.ll_rd_virt[i]) {
>>> + rc = -ENOMEM;
>>> + goto err_free_ll;
>>> + }
>>> + rd_done++;
>>> + info->ll_rd_phys[i] = edma_ctx.ll_rd_phys[i];
>>> + }
>>> +
>>> + /* For interruption */
>>> + edma_ctx.notify_qp_max = NTB_EDMA_NOTIFY_MAX_QP;
>>> + intr = dma_alloc_coherent(dev, sizeof(*intr), &intr_phys, GFP_KERNEL);
>>> + if (!intr) {
>>> + rc = -ENOMEM;
>>> + goto err_free_ll;
>>> + }
>>> + memset(intr, 0, sizeof(*intr));
>>> + edma_ctx.intr_ep_virt = intr;
>>> + edma_ctx.intr_ep_phys = intr_phys;
>>> + info->intr_dar_base = intr_phys;
>>> +
>>> + peer_mw = ntb_peer_mw_count(ndev);
>>> + if (peer_mw <= 0) {
>>> + rc = -ENODEV;
>>> + goto err_free_ll;
>>> + }
>>> +
>>> + mw_index = peer_mw - 1; /* last MW */
>>> +
>>> + rc = ntb_mw_get_align(ndev, 0, mw_index, 0, NULL, &size_max,
>>> + &offset);
>>> + if (rc)
>>> + goto err_free_ll;
>>> +
>>> + if (size_max < need) {
>>> + rc = -ENOSPC;
>>> + goto err_free_ll;
>>> + }
>>> +
>>> + /* Map register space (direct) */
>>> + dom = iommu_get_domain_for_dev(dev);
>>> + if (dom) {
>>> + phys = edma_regs_phys & PAGE_MASK;
>>> + size = PAGE_ALIGN(EDMA_REG_SIZE + edma_regs_phys - phys);
>>> + iova = phys;
>>> +
>>> + rc = iommu_map(dom, iova, phys, EDMA_REG_SIZE,
>>> + IOMMU_READ | IOMMU_WRITE | IOMMU_MMIO, GFP_KERNEL);
>>> + if (rc)
>>> + dev_err(&ndev->dev, "failed to create direct mapping for eDMA reg space\n");
>>> + reg_mapped = true;
>>> + }
>>> +
>>> + rc = ntb_mw_set_trans(ndev, 0, mw_index, edma_regs_phys, EDMA_REG_SIZE, offset);
>>> + if (rc)
>>> + goto err_unmap_reg;
>>> +
>>> + offset += EDMA_REG_SIZE;
>>> +
>>> + /* Map ntb_edma_info */
>>> + rc = ntb_mw_set_trans(ndev, 0, mw_index, info_phys, info_bytes, offset);
>>> + if (rc)
>>> + goto err_clear_trans;
>>> + offset += info_bytes;
>>> +
>>> + /* Map LL location */
>>> + for (i = 0; i < EDMA_WR_CH_NUM; i++) {
>>> + rc = ntb_mw_set_trans(ndev, 0, mw_index, edma_ctx.ll_wr_phys[i],
>>> + DMA_LLP_MEM_SIZE, offset);
>>> + if (rc)
>>> + goto err_clear_trans;
>>> + offset += DMA_LLP_MEM_SIZE;
>>> + }
>>> + for (i = 0; i < EDMA_RD_CH_NUM + 1; i++) {
>>> + rc = ntb_mw_set_trans(ndev, 0, mw_index, edma_ctx.ll_rd_phys[i],
>>> + DMA_LLP_MEM_SIZE, offset);
>>> + if (rc)
>>> + goto err_clear_trans;
>>> + offset += DMA_LLP_MEM_SIZE;
>>> + }
>>> + edma_ctx.initialized = true;
>>> +
>>> + return 0;
>>> +
>>> +err_clear_trans:
>>> + /*
>>> + * Tear down the NTB translation window used for the eDMA MW.
>>> + * There is no sub-range clear API for ntb_mw_set_trans(), so we
>>> + * unconditionally drop the whole mapping on error.
>>> + */
>>> + ntb_mw_clear_trans(ndev, 0, mw_index);
>>> +
>>> +err_unmap_reg:
>>> + if (reg_mapped)
>>> + iommu_unmap(dom, iova, size);
>>> +err_free_ll:
>>> + while (rd_done--)
>>> + dma_free_attrs(dev, DMA_LLP_MEM_SIZE,
>>> + edma_ctx.ll_rd_virt[rd_done],
>>> + edma_ctx.ll_rd_phys[rd_done],
>>> + DMA_ATTR_FORCE_CONTIGUOUS);
>>> + while (wr_done--)
>>> + dma_free_attrs(dev, DMA_LLP_MEM_SIZE,
>>> + edma_ctx.ll_wr_virt[wr_done],
>>> + edma_ctx.ll_wr_phys[wr_done],
>>> + DMA_ATTR_FORCE_CONTIGUOUS);
>>> + if (edma_ctx.intr_ep_virt)
>>> + dma_free_coherent(dev, sizeof(struct ntb_edma_intr),
>>> + edma_ctx.intr_ep_virt,
>>> + edma_ctx.intr_ep_phys);
>>> + dma_free_coherent(dev, info_bytes, info, info_phys);
>>> + return rc;
>>> +}
>>> +
>>> +static int ntb_edma_irq_vector(struct device *dev, unsigned int nr)
>>> +{
>>> + struct pci_dev *pdev = to_pci_dev(dev);
>>> + int ret, nvec;
>>> +
>>> + nvec = pci_msi_vec_count(pdev);
>>> + for (; nr < nvec; nr++) {
>>> + ret = pci_irq_vector(pdev, nr);
>>> + if (!irq_has_action(ret))
>>> + return ret;
>>> + }
>>> + return 0;
>>> +}
>>> +
>>> +static const struct dw_edma_plat_ops ntb_edma_ops = {
>>> + .irq_vector = ntb_edma_irq_vector,
>>> +};
>>> +
>>> +int ntb_edma_setup_peer(struct ntb_dev *ndev)
>>> +{
>>> + struct ntb_edma_info *info;
>>> + unsigned int wr_cnt, rd_cnt;
>>> + struct dw_edma_chip *chip;
>>> + void __iomem *edma_virt;
>>> + phys_addr_t edma_phys;
>>> + resource_size_t mw_size;
>>> + u64 off = EDMA_REG_SIZE;
>>> + int peer_mw, mw_index;
>>> + unsigned int i;
>>> + int ret;
>>> +
>>> + peer_mw = ntb_peer_mw_count(ndev);
>>> + if (peer_mw <= 0)
>>> + return -ENODEV;
>>> +
>>> + mw_index = peer_mw - 1; /* last MW */
>>> +
>>> + ret = ntb_peer_mw_get_addr(ndev, mw_index, &edma_phys,
>>> + &mw_size);
>>> + if (ret)
>>> + return -1;
>>> +
>>> + edma_virt = ioremap(edma_phys, mw_size);
>>> +
>>> + chip = devm_kzalloc(&ndev->dev, sizeof(*chip), GFP_KERNEL);
>>> + if (!chip) {
>>> + ret = -ENOMEM;
>>> + return ret;
>>> + }
>>> +
>>> + chip->dev = &ndev->pdev->dev;
>>> + chip->nr_irqs = 4;
>>> + chip->ops = &ntb_edma_ops;
>>> + chip->flags = 0;
>>> + chip->reg_base = edma_virt;
>>> + chip->mf = EDMA_MF_EDMA_UNROLL;
>>> +
>>> + info = edma_virt + off;
>>> + if (info->magic != NTB_EDMA_INFO_MAGIC)
>>> + return -EINVAL;
>>> + wr_cnt = info->wr_cnt;
>>> + rd_cnt = info->rd_cnt;
>>> + chip->ll_wr_cnt = wr_cnt;
>>> + chip->ll_rd_cnt = rd_cnt;
>>> + off += PAGE_SIZE;
>>> +
>>> + edma_ctx.notify_qp_max = NTB_EDMA_NOTIFY_MAX_QP;
>>> + edma_ctx.intr_ep_phys = info->intr_dar_base;
>>> + if (edma_ctx.intr_ep_phys) {
>>> + edma_ctx.intr_rc_virt =
>>> + dma_alloc_coherent(&ndev->pdev->dev,
>>> + sizeof(struct ntb_edma_intr),
>>> + &edma_ctx.intr_rc_phys,
>>> + GFP_KERNEL);
>>> + if (!edma_ctx.intr_rc_virt)
>>> + return -ENOMEM;
>>> + memset(edma_ctx.intr_rc_virt, 0,
>>> + sizeof(struct ntb_edma_intr));
>>> + }
>>> +
>>> + for (i = 0; i < wr_cnt; i++) {
>>> + chip->ll_region_wr[i].vaddr.io = edma_virt + off;
>>> + chip->ll_region_wr[i].paddr = info->ll_wr_phys[i];
>>> + chip->ll_region_wr[i].sz = DMA_LLP_MEM_SIZE;
>>> + off += DMA_LLP_MEM_SIZE;
>>> + }
>>> + for (i = 0; i < rd_cnt; i++) {
>>> + chip->ll_region_rd[i].vaddr.io = edma_virt + off;
>>> + chip->ll_region_rd[i].paddr = info->ll_rd_phys[i];
>>> + chip->ll_region_rd[i].sz = DMA_LLP_MEM_SIZE;
>>> + off += DMA_LLP_MEM_SIZE;
>>> + }
>>> +
>>> + if (!pci_dev_msi_enabled(ndev->pdev))
>>> + return -ENXIO;
>>> +
>>> + ret = dw_edma_probe(chip);
>>> + if (ret) {
>>> + dev_err(&ndev->dev, "dw_edma_probe failed: %d\n", ret);
>>> + return ret;
>>> + }
>>> +
>>> + return 0;
>>> +}
>>> +
>>> +struct ntb_edma_filter {
>>> + struct device *dma_dev;
>>> + u32 direction;
>>> +};
>>> +
>>> +static bool ntb_edma_filter_fn(struct dma_chan *chan, void *arg)
>>> +{
>>> + struct ntb_edma_filter *filter = arg;
>>> + u32 dir = filter->direction;
>>> + struct dma_slave_caps caps;
>>> + int ret;
>>> +
>>> + if (chan->device->dev != filter->dma_dev)
>>> + return false;
>>> +
>>> + ret = dma_get_slave_caps(chan, &caps);
>>> + if (ret < 0)
>>> + return false;
>>> +
>>> + return !!(caps.directions & dir);
>>> +}
>>> +
>>> +void ntb_edma_teardown_chans(struct ntb_edma_chans *edma)
>>> +{
>>> + unsigned int i;
>>> +
>>> + for (i = 0; i < edma->num_wr_chan; i++)
>>> + dma_release_channel(edma->wr_chan[i]);
>>> +
>>> + for (i = 0; i < edma->num_rd_chan; i++)
>>> + dma_release_channel(edma->rd_chan[i]);
>>> +
>>> + if (edma->intr_chan)
>>> + dma_release_channel(edma->intr_chan);
>>> +}
>>> +
>>> +int ntb_edma_setup_chans(struct device *dma_dev, struct ntb_edma_chans *edma)
>>> +{
>>> + struct ntb_edma_filter filter;
>>> + dma_cap_mask_t dma_mask;
>>> + unsigned int i;
>>> +
>>> + dma_cap_zero(dma_mask);
>>> + dma_cap_set(DMA_SLAVE, dma_mask);
>>> +
>>> + memset(edma, 0, sizeof(*edma));
>>> + edma->dev = dma_dev;
>>> +
>>> + filter.dma_dev = dma_dev;
>>> + filter.direction = BIT(DMA_DEV_TO_MEM);
>>> + for (i = 0; i < EDMA_WR_CH_NUM; i++) {
>>> + edma->wr_chan[i] = dma_request_channel(dma_mask,
>>> + ntb_edma_filter_fn,
>>> + &filter);
>>> + if (!edma->wr_chan[i])
>>> + break;
>>> + edma->num_wr_chan++;
>>> + }
>>> +
>>> + filter.direction = BIT(DMA_MEM_TO_DEV);
>>> + for (i = 0; i < EDMA_RD_CH_NUM; i++) {
>>> + edma->rd_chan[i] = dma_request_channel(dma_mask,
>>> + ntb_edma_filter_fn,
>>> + &filter);
>>> + if (!edma->rd_chan[i])
>>> + break;
>>> + edma->num_rd_chan++;
>>> + }
>>> +
>>> + edma->intr_chan = dma_request_channel(dma_mask, ntb_edma_filter_fn,
>>> + &filter);
>>> + if (!edma->intr_chan)
>>> + dev_warn(dma_dev,
>>> + "Remote eDMA notify channel could not be allocated\n");
>>> +
>>> + if (!edma->num_wr_chan || !edma->num_rd_chan) {
>>> + dev_warn(dma_dev, "Remote eDMA channels failed to initialize\n");
>>> + ntb_edma_teardown_chans(edma);
>>> + return -ENODEV;
>>> + }
>>> + return 0;
>>> +}
>>> +
>>> +struct dma_chan *ntb_edma_pick_chan(struct ntb_edma_chans *edma,
>>> + remote_edma_dir_t dir)
>>> +{
>>> + unsigned int n, cur, idx;
>>> + struct dma_chan **chans;
>>> + atomic_t *cur_chan;
>>> +
>>> + if (dir == REMOTE_EDMA_WRITE) {
>>> + n = edma->num_wr_chan;
>>> + chans = edma->wr_chan;
>>> + cur_chan = &edma->cur_wr_chan;
>>> + } else {
>>> + n = edma->num_rd_chan;
>>> + chans = edma->rd_chan;
>>> + cur_chan = &edma->cur_rd_chan;
>>> + }
>>> + if (WARN_ON_ONCE(!n))
>>> + return NULL;
>>> +
>>> + /* Simple round-robin */
>>> + cur = (unsigned int)atomic_inc_return(cur_chan) - 1;
>>> + idx = cur % n;
>>> + return chans[idx];
>>> +}
>>> +
>>> +int ntb_edma_notify_peer(struct ntb_edma_chans *edma, int qp_num)
>>> +{
>>> + struct dma_async_tx_descriptor *txd;
>>> + struct dma_slave_config cfg;
>>> + struct scatterlist sgl;
>>> + dma_cookie_t cookie;
>>> + struct device *dev;
>>> +
>>> + if (!edma || !edma->intr_chan)
>>> + return -ENXIO;
>>> +
>>> + if (qp_num < 0 || qp_num >= edma_ctx.notify_qp_max)
>>> + return -EINVAL;
>>> +
>>> + if (!edma_ctx.intr_rc_virt || !edma_ctx.intr_ep_phys)
>>> + return -EINVAL;
>>> +
>>> + dev = edma->dev;
>>> + if (!dev)
>>> + return -ENODEV;
>>> +
>>> + WRITE_ONCE(edma_ctx.intr_rc_virt->db[qp_num], 1);
>>> +
>>> + /* Ensure store is visible before kicking the DMA transfer */
>>> + wmb();
>>> +
>>> + sg_init_table(&sgl, 1);
>>> + sg_dma_address(&sgl) = edma_ctx.intr_rc_phys + qp_num * sizeof(u32);
>>> + sg_dma_len(&sgl) = sizeof(u32);
>>> +
>>> + memset(&cfg, 0, sizeof(cfg));
>>> + cfg.dst_addr = edma_ctx.intr_ep_phys + qp_num * sizeof(u32);
>>> + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
>>> + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
>>> + cfg.direction = DMA_MEM_TO_DEV;
>>> +
>>> + if (dmaengine_slave_config(edma->intr_chan, &cfg))
>>> + return -EINVAL;
>>> +
>>> + txd = dmaengine_prep_slave_sg(edma->intr_chan, &sgl, 1,
>>> + DMA_MEM_TO_DEV,
>>> + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
>>> + if (!txd)
>>> + return -ENOSPC;
>>> +
>>> + cookie = dmaengine_submit(txd);
>>> + if (dma_submit_error(cookie))
>>> + return -ENOSPC;
>>> +
>>> + dma_async_issue_pending(edma->intr_chan);
>>> + return 0;
>>> +}
>>> diff --git a/drivers/ntb/ntb_edma.h b/drivers/ntb/ntb_edma.h
>>> new file mode 100644
>>> index 000000000000..da0451827edb
>>> --- /dev/null
>>> +++ b/drivers/ntb/ntb_edma.h
>>> @@ -0,0 +1,128 @@
>>> +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
>>> +#ifndef _NTB_EDMA_H_
>>> +#define _NTB_EDMA_H_
>>> +
>>> +#include <linux/completion.h>
>>> +#include <linux/device.h>
>>> +#include <linux/interrupt.h>
>>> +
>>> +#define EDMA_REG_SIZE SZ_64K
>>> +#define DMA_LLP_MEM_SIZE SZ_4K
>>> +#define EDMA_WR_CH_NUM 4
>>> +#define EDMA_RD_CH_NUM 4
>>> +#define NTB_EDMA_MAX_CH 8
>>> +
>>> +#define NTB_EDMA_INFO_MAGIC 0x45444D41 /* "EDMA" */
>>> +#define NTB_EDMA_INFO_OFF EDMA_REG_SIZE
>>> +
>>> +#define NTB_EDMA_RING_ORDER 7
>>> +#define NTB_EDMA_RING_ENTRIES (1U << NTB_EDMA_RING_ORDER)
>>> +#define NTB_EDMA_RING_MASK (NTB_EDMA_RING_ENTRIES - 1)
>>> +
>>> +typedef void (*ntb_edma_interrupt_cb_t)(void *data, int qp_num);
>>> +
>>> +/*
>>> + * REMOTE_EDMA_EP:
>>> + * Endpoint owns the eDMA engine and pushes descriptors into a shared MW.
>>> + *
>>> + * REMOTE_EDMA_RC:
>>> + * Root Complex controls the endpoint eDMA through the shared MW and
>>> + * drives reads/writes on behalf of the host.
>>> + */
>>> +typedef enum {
>>> + REMOTE_EDMA_UNKNOWN,
>>> + REMOTE_EDMA_EP,
>>> + REMOTE_EDMA_RC,
>>> +} remote_edma_mode_t;
>>> +
>>> +typedef enum {
>>> + REMOTE_EDMA_WRITE,
>>> + REMOTE_EDMA_READ,
>>> +} remote_edma_dir_t;
>>> +
>>> +/*
>>> + * Layout of remote eDMA MW (EP local address space, RC sees via peer MW):
>>> + *
>>> + * 0 .. EDMA_REG_SIZE-1 : DesignWare eDMA registers
>>> + * EDMA_REG_SIZE .. +PAGE_SIZE : struct ntb_edma_info (EP writes, RC reads)
>>> + * +PAGE_SIZE .. : LL ring buffers (EP allocates phys addresses,
>>> + * RC configures via dw_edma)
>>> + *
>>> + * ntb_edma_setup_mws() on EP:
>>> + * - allocates ntb_edma_info and LLs in EP memory
>>> + * - programs inbound iATU so that RC peer MW[n] points at this block
>>> + *
>>> + * ntb_edma_setup_peer() on RC:
>>> + * - ioremaps peer MW[n]
>>> + * - reads ntb_edma_info
>>> + * - sets up dw_edma_chip ll_region_* from that info
>>> + */
>>> +struct ntb_edma_info {
>>> + u32 magic;
>>> + u16 wr_cnt;
>>> + u16 rd_cnt;
>>> + u64 regs_phys;
>>> + u32 ll_stride;
>>> + u32 rsvd;
>>> + u64 ll_wr_phys[NTB_EDMA_MAX_CH];
>>> + u64 ll_rd_phys[NTB_EDMA_MAX_CH];
>>> +
>>> + u64 intr_dar_base;
>>> +} __packed;
>>> +
>>> +struct ll_dma_addrs {
>>> + dma_addr_t wr[EDMA_WR_CH_NUM];
>>> + dma_addr_t rd[EDMA_RD_CH_NUM];
>>> +};
>>> +
>>> +struct ntb_edma_chans {
>>> + struct device *dev;
>>> +
>>> + struct dma_chan *wr_chan[EDMA_WR_CH_NUM];
>>> + struct dma_chan *rd_chan[EDMA_RD_CH_NUM];
>>> + struct dma_chan *intr_chan;
>>> +
>>> + unsigned int num_wr_chan;
>>> + unsigned int num_rd_chan;
>>> + atomic_t cur_wr_chan;
>>> + atomic_t cur_rd_chan;
>>> +};
>>> +
>>> +static __always_inline u32 ntb_edma_ring_idx(u32 v)
>>> +{
>>> + return v & NTB_EDMA_RING_MASK;
>>> +}
>>> +
>>> +static __always_inline u32 ntb_edma_ring_used_entry(u32 head, u32 tail)
>>> +{
>>> + if (head >= tail) {
>>> + WARN_ON_ONCE((head - tail) > (NTB_EDMA_RING_ENTRIES - 1));
>>> + return head - tail;
>>> + }
>>> +
>>> + WARN_ON_ONCE((U32_MAX - tail + head + 1) > (NTB_EDMA_RING_ENTRIES - 1));
>>> + return U32_MAX - tail + head + 1;
>>> +}
>>> +
>>> +static __always_inline u32 ntb_edma_ring_free_entry(u32 head, u32 tail)
>>> +{
>>> + return NTB_EDMA_RING_ENTRIES - ntb_edma_ring_used_entry(head, tail) - 1;
>>> +}
>>> +
>>> +static __always_inline bool ntb_edma_ring_full(u32 head, u32 tail)
>>> +{
>>> + return ntb_edma_ring_free_entry(head, tail) == 0;
>>> +}
>>> +
>>> +int ntb_edma_setup_isr(struct device *dev, struct device *epc_dev,
>>> + ntb_edma_interrupt_cb_t cb, void *data);
>>> +void ntb_edma_teardown_isr(struct device *dev);
>>> +int ntb_edma_setup_mws(struct ntb_dev *ndev);
>>> +int ntb_edma_setup_peer(struct ntb_dev *ndev);
>>> +int ntb_edma_setup_chans(struct device *dma_dev, struct ntb_edma_chans *edma);
>>> +struct dma_chan *ntb_edma_pick_chan(struct ntb_edma_chans *edma,
>>> + remote_edma_dir_t dir);
>>> +void ntb_edma_teardown_chans(struct ntb_edma_chans *edma);
>>> +int ntb_edma_notify_peer(struct ntb_edma_chans *edma, int qp_num);
>>> +
>>> +#endif
>>> diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport_core.c
>>> similarity index 65%
>>> rename from drivers/ntb/ntb_transport.c
>>> rename to drivers/ntb/ntb_transport_core.c
>>> index 907db6c93d4d..48d48921978d 100644
>>> --- a/drivers/ntb/ntb_transport.c
>>> +++ b/drivers/ntb/ntb_transport_core.c
>>> @@ -47,6 +47,9 @@
>>> * Contact Information:
>>> * Jon Mason <jon.mason@intel.com>
>>> */
>>> +#include <linux/atomic.h>
>>> +#include <linux/bug.h>
>>> +#include <linux/compiler.h>
>>> #include <linux/debugfs.h>
>>> #include <linux/delay.h>
>>> #include <linux/dmaengine.h>
>>> @@ -71,6 +74,8 @@
>>> #define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB"
>>> #define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2)
>>>
>>> +#define NTB_EDMA_MAX_POLL 32
>>> +
>>> MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
>>> MODULE_VERSION(NTB_TRANSPORT_VER);
>>> MODULE_LICENSE("Dual BSD/GPL");
>>> @@ -102,6 +107,13 @@ module_param(use_msi, bool, 0644);
>>> MODULE_PARM_DESC(use_msi, "Use MSI interrupts instead of doorbells");
>>> #endif
>>>
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>
>> This is a comment throughout this patch. Doing ifdefs inside C source is pretty frowed upon in the kernel. The preferred way is to only have ifdefs in the header files. So please give this a bit more consideration and see if it can be done differently to address this.
>
> I agree, there is no good reason to keep those remaining ifdefs at all.
> I'll clean it up. Thanks for pointing this out.
>
>>
>>> +#include "ntb_edma.h"
>>> +static bool use_remote_edma;
>>> +module_param(use_remote_edma, bool, 0644);
>>> +MODULE_PARM_DESC(use_remote_edma, "Use remote eDMA mode (when enabled, use_msi is ignored)");
>>> +#endif
>>> +
>>> static struct dentry *nt_debugfs_dir;
>>>
>>> /* Only two-ports NTB devices are supported */
>>> @@ -125,6 +137,14 @@ struct ntb_queue_entry {
>>> struct ntb_payload_header __iomem *tx_hdr;
>>> struct ntb_payload_header *rx_hdr;
>>> };
>>> +
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>> + dma_addr_t addr;
>>> +
>>> + /* Used by RC side only */
>>> + struct scatterlist sgl;
>>> + struct work_struct dma_work;
>>> +#endif
>>> };
>>>
>>> struct ntb_rx_info {
>>> @@ -202,6 +222,33 @@ struct ntb_transport_qp {
>>> int msi_irq;
>>> struct ntb_msi_desc msi_desc;
>>> struct ntb_msi_desc peer_msi_desc;
>>> +
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>> + /*
>>> + * For ensuring peer notification in non-atomic context.
>>> + * ntb_peer_db_set might sleep or schedule.
>>> + */
>>> + struct work_struct db_work;
>>> +
>>> + /*
>>> + * wr: remote eDMA write transfer (EP -> RC direction)
>>> + * rd: remote eDMA read transfer (RC -> EP direction)
>>> + */
>>> + u32 wr_cons;
>>> + u32 rd_cons;
>>> + u32 wr_prod;
>>> + u32 rd_prod;
>>> + u32 wr_issue;
>>> + u32 rd_issue;
>>> +
>>> + spinlock_t ep_tx_lock;
>>> + spinlock_t ep_rx_lock;
>>> + spinlock_t rc_lock;
>>> +
>>> + /* Completion work for read/write transfers. */
>>> + struct work_struct read_work;
>>> + struct work_struct write_work;
>>> +#endif
>>
>> For something like this, maybe it needs its own struct instead of an ifdef chunk. Perhaps 'ntb_rx_info' can serve as a core data struct with EDMA having 'ntb_rx_info_edma' and embed 'ntb_rx_info'.
>
> Thanks again for the suggestion. I'll reorganize things.
>
> Koichiro
>
>>
>> DJ
>>
>>> };
>>>
>>> struct ntb_transport_mw {
>>> @@ -249,6 +296,13 @@ struct ntb_transport_ctx {
>>>
>>> /* Make sure workq of link event be executed serially */
>>> struct mutex link_event_lock;
>>> +
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>> + remote_edma_mode_t remote_edma_mode;
>>> + struct device *dma_dev;
>>> + struct workqueue_struct *wq;
>>> + struct ntb_edma_chans edma;
>>> +#endif
>>> };
>>>
>>> enum {
>>> @@ -262,6 +316,19 @@ struct ntb_payload_header {
>>> unsigned int flags;
>>> };
>>>
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>> +static void ntb_transport_edma_uninit(struct ntb_transport_ctx *nt);
>>> +static int ntb_transport_edma_init(struct ntb_transport_ctx *nt,
>>> + unsigned int *mw_count);
>>> +static void ntb_transport_edma_init_queue(struct ntb_transport_ctx *nt,
>>> + unsigned int qp_num);
>>> +static void ntb_transport_edma_create_queue(struct ntb_transport_ctx *nt,
>>> + struct ntb_transport_qp *qp);
>>> +static int ntb_transport_edma_rc_init(struct ntb_transport_ctx *nt);
>>> +static int ntb_transport_edma_ep_init(struct ntb_transport_ctx *nt);
>>> +static void ntb_transport_edma_rc_dma_work(struct work_struct *work);
>>> +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
>>> +
>>> /*
>>> * Return the device that should be used for DMA mapping.
>>> *
>>> @@ -298,7 +365,7 @@ enum {
>>> container_of((__drv), struct ntb_transport_client, driver)
>>>
>>> #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count)
>>> -#define NTB_QP_DEF_NUM_ENTRIES 100
>>> +#define NTB_QP_DEF_NUM_ENTRIES 128
>>> #define NTB_LINK_DOWN_TIMEOUT 10
>>>
>>> static void ntb_transport_rxc_db(unsigned long data);
>>> @@ -1015,6 +1082,10 @@ static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
>>> count = ntb_spad_count(nt->ndev);
>>> for (i = 0; i < count; i++)
>>> ntb_spad_write(nt->ndev, i, 0);
>>> +
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>> + ntb_edma_teardown_chans(&nt->edma);
>>> +#endif
>>> }
>>>
>>> static void ntb_transport_link_cleanup_work(struct work_struct *work)
>>> @@ -1051,6 +1122,14 @@ static void ntb_transport_link_work(struct work_struct *work)
>>>
>>> /* send the local info, in the opposite order of the way we read it */
>>>
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>> + rc = ntb_transport_edma_ep_init(nt);
>>> + if (rc) {
>>> + dev_err(&pdev->dev, "Failed to init EP: %d\n", rc);
>>> + return;
>>> + }
>>> +#endif
>>> +
>>> if (nt->use_msi) {
>>> rc = ntb_msi_setup_mws(ndev);
>>> if (rc) {
>>> @@ -1132,6 +1211,14 @@ static void ntb_transport_link_work(struct work_struct *work)
>>>
>>> nt->link_is_up = true;
>>>
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>> + rc = ntb_transport_edma_rc_init(nt);
>>> + if (rc) {
>>> + dev_err(&pdev->dev, "Failed to init RC: %d\n", rc);
>>> + goto out1;
>>> + }
>>> +#endif
>>> +
>>> for (i = 0; i < nt->qp_count; i++) {
>>> struct ntb_transport_qp *qp = &nt->qp_vec[i];
>>>
>>> @@ -1277,6 +1364,8 @@ static const struct ntb_transport_backend_ops default_backend_ops = {
>>> .debugfs_stats_show = ntb_transport_default_debugfs_stats_show,
>>> };
>>>
>>> +static const struct ntb_transport_backend_ops edma_backend_ops;
>>> +
>>> static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
>>> {
>>> struct ntb_transport_ctx *nt;
>>> @@ -1311,7 +1400,23 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
>>>
>>> nt->ndev = ndev;
>>>
>>> - nt->backend_ops = default_backend_ops;
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>> + if (use_remote_edma) {
>>> + rc = ntb_transport_edma_init(nt, &mw_count);
>>> + if (rc) {
>>> + nt->mw_count = 0;
>>> + goto err;
>>> + }
>>> + nt->backend_ops = edma_backend_ops;
>>> +
>>> + /*
>>> + * On remote eDMA mode, we reserve a read channel for Host->EP
>>> + * interruption.
>>> + */
>>> + use_msi = false;
>>> + } else
>>> +#endif
>>> + nt->backend_ops = default_backend_ops;
>>>
>>> /*
>>> * If we are using MSI, and have at least one extra memory window,
>>> @@ -1402,6 +1507,10 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
>>> rc = ntb_transport_init_queue(nt, i);
>>> if (rc)
>>> goto err2;
>>> +
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>> + ntb_transport_edma_init_queue(nt, i);
>>> +#endif
>>> }
>>>
>>> INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);
>>> @@ -1433,6 +1542,9 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
>>> }
>>> kfree(nt->mw_vec);
>>> err:
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>> + ntb_transport_edma_uninit(nt);
>>> +#endif
>>> kfree(nt);
>>> return rc;
>>> }
>>> @@ -2055,11 +2167,16 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
>>>
>>> nt->qp_bitmap_free &= ~qp_bit;
>>>
>>> + qp->qp_bit = qp_bit;
>>> qp->cb_data = data;
>>> qp->rx_handler = handlers->rx_handler;
>>> qp->tx_handler = handlers->tx_handler;
>>> qp->event_handler = handlers->event_handler;
>>>
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>> + ntb_transport_edma_create_queue(nt, qp);
>>> +#endif
>>> +
>>> dma_cap_zero(dma_mask);
>>> dma_cap_set(DMA_MEMCPY, dma_mask);
>>>
>>> @@ -2105,6 +2222,9 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
>>> goto err1;
>>>
>>> entry->qp = qp;
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>> + INIT_WORK(&entry->dma_work, ntb_transport_edma_rc_dma_work);
>>> +#endif
>>> ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
>>> &qp->rx_free_q);
>>> }
>>> @@ -2156,8 +2276,8 @@ EXPORT_SYMBOL_GPL(ntb_transport_create_queue);
>>> */
>>> void ntb_transport_free_queue(struct ntb_transport_qp *qp)
>>> {
>>> - struct pci_dev *pdev;
>>> struct ntb_queue_entry *entry;
>>> + struct pci_dev *pdev;
>>> u64 qp_bit;
>>>
>>> if (!qp)
>>> @@ -2208,6 +2328,10 @@ void ntb_transport_free_queue(struct ntb_transport_qp *qp)
>>> tasklet_kill(&qp->rxc_db_work);
>>>
>>> cancel_delayed_work_sync(&qp->link_work);
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>> + cancel_work_sync(&qp->read_work);
>>> + cancel_work_sync(&qp->write_work);
>>> +#endif
>>>
>>> qp->cb_data = NULL;
>>> qp->rx_handler = NULL;
>>> @@ -2346,6 +2470,1157 @@ int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
>>> }
>>> EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue);
>>>
>>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
>>> +/*
>>> + * Remote eDMA mode implementation
>>> + */
>>> +struct ntb_edma_desc {
>>> + u32 len;
>>> + u32 flags;
>>> + u64 addr; /* DMA address */
>>> + u64 data;
>>> +};
>>> +
>>> +struct ntb_edma_ring {
>>> + struct ntb_edma_desc desc[NTB_EDMA_RING_ENTRIES];
>>> + u32 head;
>>> + u32 tail;
>>> +};
>>> +
>>> +#define NTB_EDMA_DESC_OFF(i) ((size_t)(i) * sizeof(struct ntb_edma_desc))
>>> +
>>> +#define __NTB_EDMA_CHECK_INDEX(_i) \
>>> +({ \
>>> + unsigned long __i = (unsigned long)(_i); \
>>> + WARN_ONCE(__i >= (unsigned long)NTB_EDMA_RING_ENTRIES, \
>>> + "ntb_edma: index i=%lu >= ring_entries=%lu\n", \
>>> + __i, (unsigned long)NTB_EDMA_RING_ENTRIES); \
>>> + __i; \
>>> +})
>>> +
>>> +#define NTB_EDMA_DESC_I(qp, i, n) \
>>> +({ \
>>> + typeof(qp) __qp = (qp); \
>>> + unsigned long __i = __NTB_EDMA_CHECK_INDEX(i); \
>>> + (struct ntb_edma_desc *) \
>>> + ((char *)(__qp)->rx_buff + \
>>> + (sizeof(struct ntb_edma_ring) * n) + \
>>> + NTB_EDMA_DESC_OFF(__i)); \
>>> +})
>>> +
>>> +#define NTB_EDMA_DESC_O(qp, i, n) \
>>> +({ \
>>> + typeof(qp) __qp = (qp); \
>>> + unsigned long __i = __NTB_EDMA_CHECK_INDEX(i); \
>>> + (struct ntb_edma_desc __iomem *) \
>>> + ((char __iomem *)(__qp)->tx_mw + \
>>> + (sizeof(struct ntb_edma_ring) * n) + \
>>> + NTB_EDMA_DESC_OFF(__i)); \
>>> +})
>>> +
>>> +#define NTB_EDMA_HEAD_I(qp, n) ((u32 *)((char *)qp->rx_buff + \
>>> + (sizeof(struct ntb_edma_ring) * n) + \
>>> + offsetof(struct ntb_edma_ring, head)))
>>> +#define NTB_EDMA_HEAD_O(qp, n) ((u32 *)((char __iomem *)qp->tx_mw + \
>>> + (sizeof(struct ntb_edma_ring) * n) + \
>>> + offsetof(struct ntb_edma_ring, head)))
>>> +#define NTB_EDMA_TAIL_I(qp, n) ((u32 *)((char *)qp->rx_buff + \
>>> + (sizeof(struct ntb_edma_ring) * n) + \
>>> + offsetof(struct ntb_edma_ring, tail)))
>>> +#define NTB_EDMA_TAIL_O(qp, n) ((u32 *)((char __iomem *)qp->tx_mw + \
>>> + (sizeof(struct ntb_edma_ring) * n) + \
>>> + offsetof(struct ntb_edma_ring, tail)))
>>> +
>>> +/*
>>> + * Macro naming rule:
>>> + * NTB_DESC_RD_EP_I (as an example)
>>> + * ^^ ^^ ^
>>> + * : : `-- I(n) or O(ut). In = Read, Out = Write.
>>> + * : `----- Who uses this macro.
>>> + * `-------- DESC / HEAD / TAIL
>>> + *
>>> + * Read transfers (RC->EP):
>>> + *
>>> + * EP view (outbound, written via NTB):
>>> + * - descs: NTB_DESC_RD_EP_O(qp, i) / NTB_DESC_RD_EP_I(qp, i)
>>> + * [ len ][ flags ][ addr ][ data ]
>>> + * [ len ][ flags ][ addr ][ data ]
>>> + * :
>>> + * [ len ][ flags ][ addr ][ data ]
>>> + * - head: NTB_HEAD_RD_EP_O(qp)
>>> + * - tail: NTB_TAIL_RD_EP_I(qp)
>>> + *
>>> + * RC view (inbound, local mapping):
>>> + * - descs: NTB_DESC_RD_RC_I(qp, i) / NTB_DESC_RD_RC_O(qp, i)
>>> + * [ len ][ flags ][ addr ][ data ]
>>> + * [ len ][ flags ][ addr ][ data ]
>>> + * :
>>> + * [ len ][ flags ][ addr ][ data ]
>>> + * - head: NTB_HEAD_RD_RC_I(qp)
>>> + * - tail: NTB_TAIL_RD_RC_O(qp)
>>> + *
>>> + * Write transfers (EP -> RC) are analogous but use
>>> + * NTB_DESC_WR_{EP_O,RC_I}(), NTB_HEAD_WR_{EP_O,RC_I}(),
>>> + * and NTB_TAIL_WR_{EP_I,RC_O}().
>>> + */
>>> +#define NTB_DESC_RD_EP_I(qp, i) NTB_EDMA_DESC_I(qp, i, 0)
>>> +#define NTB_DESC_RD_EP_O(qp, i) NTB_EDMA_DESC_O(qp, i, 0)
>>> +#define NTB_DESC_WR_EP_I(qp, i) NTB_EDMA_DESC_I(qp, i, 1)
>>> +#define NTB_DESC_WR_EP_O(qp, i) NTB_EDMA_DESC_O(qp, i, 1)
>>> +#define NTB_DESC_RD_RC_I(qp, i) NTB_EDMA_DESC_I(qp, i, 0)
>>> +#define NTB_DESC_RD_RC_O(qp, i) NTB_EDMA_DESC_O(qp, i, 0)
>>> +#define NTB_DESC_WR_RC_I(qp, i) NTB_EDMA_DESC_I(qp, i, 1)
>>> +#define NTB_DESC_WR_RC_O(qp, i) NTB_EDMA_DESC_O(qp, i, 1)
>>> +
>>> +#define NTB_HEAD_RD_EP_O(qp) NTB_EDMA_HEAD_O(qp, 0)
>>> +#define NTB_HEAD_WR_EP_O(qp) NTB_EDMA_HEAD_O(qp, 1)
>>> +#define NTB_HEAD_RD_RC_I(qp) NTB_EDMA_HEAD_I(qp, 0)
>>> +#define NTB_HEAD_WR_RC_I(qp) NTB_EDMA_HEAD_I(qp, 1)
>>> +
>>> +#define NTB_TAIL_RD_EP_I(qp) NTB_EDMA_TAIL_I(qp, 0)
>>> +#define NTB_TAIL_WR_EP_I(qp) NTB_EDMA_TAIL_I(qp, 1)
>>> +#define NTB_TAIL_RD_RC_O(qp) NTB_EDMA_TAIL_O(qp, 0)
>>> +#define NTB_TAIL_WR_RC_O(qp) NTB_EDMA_TAIL_O(qp, 1)
>>> +
>>> +static inline bool ntb_qp_edma_is_rc(struct ntb_transport_qp *qp)
>>> +{
>>> + return qp->transport->remote_edma_mode == REMOTE_EDMA_RC;
>>> +}
>>> +
>>> +static inline bool ntb_qp_edma_is_ep(struct ntb_transport_qp *qp)
>>> +{
>>> + return qp->transport->remote_edma_mode == REMOTE_EDMA_EP;
>>> +}
>>> +
>>> +static inline bool ntb_qp_edma_enabled(struct ntb_transport_qp *qp)
>>> +{
>>> + return ntb_qp_edma_is_rc(qp) || ntb_qp_edma_is_ep(qp);
>>> +}
>>> +
>>> +static unsigned int ntb_transport_edma_tx_free_entry(struct ntb_transport_qp *qp)
>>> +{
>>> + unsigned int head, tail;
>>> +
>>> + if (ntb_qp_edma_is_ep(qp)) {
>>> + scoped_guard(spinlock_irqsave, &qp->ep_tx_lock) {
>>> + /* In this scope, only 'head' might proceed */
>>> + tail = READ_ONCE(qp->wr_cons);
>>> + head = READ_ONCE(qp->wr_prod);
>>> + }
>>> + return ntb_edma_ring_free_entry(head, tail);
>>> + }
>>> +
>>> + scoped_guard(spinlock_irqsave, &qp->rc_lock) {
>>> + /* In this scope, only 'head' might proceed */
>>> + tail = READ_ONCE(qp->rd_issue);
>>> + head = READ_ONCE(*NTB_HEAD_RD_RC_I(qp));
>>> + }
>>> + /*
>>> + * On RC side, 'used' amount indicates how much EP side
>>> + * has refilled, which are available for us to use for TX.
>>> + */
>>> + return ntb_edma_ring_used_entry(head, tail);
>>> +}
>>> +
>>> +static void ntb_transport_edma_debugfs_stats_show(struct seq_file *s,
>>> + struct ntb_transport_qp *qp)
>>> +{
>>> + seq_printf(s, "rx_bytes - \t%llu\n", qp->rx_bytes);
>>> + seq_printf(s, "rx_pkts - \t%llu\n", qp->rx_pkts);
>>> + seq_printf(s, "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
>>> + seq_printf(s, "rx_buff - \t0x%p\n", qp->rx_buff);
>>> + seq_printf(s, "rx_max_entry - \t%u\n", qp->rx_max_entry);
>>> + seq_printf(s, "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry);
>>> +
>>> + seq_printf(s, "tx_bytes - \t%llu\n", qp->tx_bytes);
>>> + seq_printf(s, "tx_pkts - \t%llu\n", qp->tx_pkts);
>>> + seq_printf(s, "tx_ring_full - \t%llu\n", qp->tx_ring_full);
>>> + seq_printf(s, "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
>>> + seq_printf(s, "tx_mw - \t0x%p\n", qp->tx_mw);
>>> + seq_printf(s, "tx_max_entry - \t%u\n", qp->tx_max_entry);
>>> + seq_printf(s, "free tx - \t%u\n", ntb_transport_tx_free_entry(qp));
>>> + seq_putc(s, '\n');
>>> +
>>> + seq_puts(s, "Using Remote eDMA - Yes\n");
>>> + seq_printf(s, "QP Link - \t%s\n", qp->link_is_up ? "Up" : "Down");
>>> +}
>>> +
>>> +static void ntb_transport_edma_uninit(struct ntb_transport_ctx *nt)
>>> +{
>>> + struct ntb_dev *ndev = nt->ndev;
>>> +
>>> + if (nt->remote_edma_mode == REMOTE_EDMA_EP && ndev && ndev->pdev)
>>> + ntb_edma_teardown_isr(&ndev->pdev->dev);
>>> +
>>> + if (nt->wq)
>>> + destroy_workqueue(nt->wq);
>>> + nt->wq = NULL;
>>> +}
>>> +
>>> +static int ntb_transport_edma_init(struct ntb_transport_ctx *nt,
>>> + unsigned int *mw_count)
>>> +{
>>> + struct ntb_dev *ndev = nt->ndev;
>>> +
>>> + /*
>>> + * We need at least one MW for the transport plus one MW reserved
>>> + * for the remote eDMA window (see ntb_edma_setup_mws/peer).
>>> + */
>>> + if (*mw_count <= 1) {
>>> + dev_err(&ndev->dev,
>>> + "remote eDMA requires at least two MWS (have %u)\n",
>>> + *mw_count);
>>> + return -ENODEV;
>>> + }
>>> +
>>> + nt->wq = alloc_workqueue("ntb-edma-wq", WQ_UNBOUND | WQ_SYSFS, 0);
>>> + if (!nt->wq) {
>>> + ntb_transport_edma_uninit(nt);
>>> + return -ENOMEM;
>>> + }
>>> +
>>> + /* Reserve the last peer MW exclusively for the eDMA window. */
>>> + *mw_count -= 1;
>>> +
>>> + return 0;
>>> +}
>>> +
>>> +static void ntb_transport_edma_db_work(struct work_struct *work)
>>> +{
>>> + struct ntb_transport_qp *qp =
>>> + container_of(work, struct ntb_transport_qp, db_work);
>>> +
>>> + ntb_peer_db_set(qp->ndev, qp->qp_bit);
>>> +}
>>> +
>>> +static void ntb_transport_edma_notify_peer(struct ntb_transport_qp *qp)
>>> +{
>>> + if (ntb_qp_edma_is_rc(qp))
>>> + if (!ntb_edma_notify_peer(&qp->transport->edma, qp->qp_num))
>>> + return;
>>> +
>>> + /*
>>> + * Called from contexts that may be atomic. Since ntb_peer_db_set()
>>> + * may sleep, delegate the actual doorbell write to a workqueue.
>>> + */
>>> + queue_work(system_highpri_wq, &qp->db_work);
>>> +}
>>> +
>>> +static void ntb_transport_edma_isr(void *data, int qp_num)
>>> +{
>>> + struct ntb_transport_ctx *nt = data;
>>> + struct ntb_transport_qp *qp;
>>> +
>>> + if (qp_num < 0 || qp_num >= nt->qp_count)
>>> + return;
>>> +
>>> + qp = &nt->qp_vec[qp_num];
>>> + if (WARN_ON(!qp))
>>> + return;
>>> +
>>> + queue_work(nt->wq, &qp->read_work);
>>> + queue_work(nt->wq, &qp->write_work);
>>> +}
>>> +
>>> +static int ntb_transport_edma_rc_init(struct ntb_transport_ctx *nt)
>>> +{
>>> + struct ntb_dev *ndev = nt->ndev;
>>> + struct pci_dev *pdev = ndev->pdev;
>>> + int rc;
>>> +
>>> + if (!use_remote_edma || nt->remote_edma_mode != REMOTE_EDMA_UNKNOWN)
>>> + return 0;
>>> +
>>> + rc = ntb_edma_setup_peer(ndev);
>>> + if (rc) {
>>> + dev_err(&pdev->dev, "Failed to enable remote eDMA: %d\n", rc);
>>> + return rc;
>>> + }
>>> +
>>> + rc = ntb_edma_setup_chans(get_dma_dev(ndev), &nt->edma);
>>> + if (rc) {
>>> + dev_err(&pdev->dev, "Failed to setup eDMA channels: %d\n", rc);
>>> + return rc;
>>> + }
>>> +
>>> + nt->remote_edma_mode = REMOTE_EDMA_RC;
>>> + return 0;
>>> +}
>>> +
>>> +static int ntb_transport_edma_ep_init(struct ntb_transport_ctx *nt)
>>> +{
>>> + struct ntb_dev *ndev = nt->ndev;
>>> + struct pci_dev *pdev = ndev->pdev;
>>> + struct pci_epc *epc;
>>> + int rc;
>>> +
>>> + if (!use_remote_edma || nt->remote_edma_mode == REMOTE_EDMA_EP)
>>> + return 0;
>>> +
>>> + /* Only EP side can return pci_epc */
>>> + epc = ntb_get_pci_epc(ndev);
>>> + if (!epc)
>>> + return 0;
>>> +
>>> + rc = ntb_edma_setup_mws(ndev);
>>> + if (rc) {
>>> + dev_err(&pdev->dev,
>>> + "Failed to set up memory window for eDMA: %d\n", rc);
>>> + return rc;
>>> + }
>>> +
>>> + rc = ntb_edma_setup_isr(&pdev->dev, &epc->dev, ntb_transport_edma_isr, nt);
>>> + if (rc) {
>>> + dev_err(&pdev->dev, "Failed to setup eDMA ISR (%d)\n", rc);
>>> + return rc;
>>> + }
>>> +
>>> + nt->remote_edma_mode = REMOTE_EDMA_EP;
>>> + return 0;
>>> +}
>>> +
>>> +static int ntb_transport_edma_setup_qp_mw(struct ntb_transport_ctx *nt,
>>> + unsigned int qp_num)
>>> +{
>>> + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
>>> + struct ntb_dev *ndev = nt->ndev;
>>> + struct ntb_queue_entry *entry;
>>> + struct ntb_transport_mw *mw;
>>> + unsigned int mw_num, mw_count, qp_count;
>>> + unsigned int qp_offset, rx_info_offset;
>>> + unsigned int mw_size, mw_size_per_qp;
>>> + unsigned int num_qps_mw;
>>> + size_t edma_total;
>>> + unsigned int i;
>>> + int node;
>>> +
>>> + mw_count = nt->mw_count;
>>> + qp_count = nt->qp_count;
>>> +
>>> + mw_num = QP_TO_MW(nt, qp_num);
>>> + mw = &nt->mw_vec[mw_num];
>>> +
>>> + if (!mw->virt_addr)
>>> + return -ENOMEM;
>>> +
>>> + if (mw_num < qp_count % mw_count)
>>> + num_qps_mw = qp_count / mw_count + 1;
>>> + else
>>> + num_qps_mw = qp_count / mw_count;
>>> +
>>> + mw_size = min(nt->mw_vec[mw_num].phys_size, mw->xlat_size);
>>> + if (max_mw_size && mw_size > max_mw_size)
>>> + mw_size = max_mw_size;
>>> +
>>> + mw_size_per_qp = round_down((unsigned int)mw_size / num_qps_mw, SZ_64);
>>> + qp_offset = mw_size_per_qp * (qp_num / mw_count);
>>> + rx_info_offset = mw_size_per_qp - sizeof(struct ntb_rx_info);
>>> +
>>> + qp->tx_mw_size = mw_size_per_qp;
>>> + qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;
>>> + if (!qp->tx_mw)
>>> + return -EINVAL;
>>> + qp->tx_mw_phys = nt->mw_vec[mw_num].phys_addr + qp_offset;
>>> + if (!qp->tx_mw_phys)
>>> + return -EINVAL;
>>> + qp->rx_info = qp->tx_mw + rx_info_offset;
>>> + qp->rx_buff = mw->virt_addr + qp_offset;
>>> + qp->remote_rx_info = qp->rx_buff + rx_info_offset;
>>> +
>>> + /* Due to housekeeping, there must be at least 2 buffs */
>>> + qp->tx_max_frame = min(transport_mtu, mw_size_per_qp / 2);
>>> + qp->rx_max_frame = min(transport_mtu, mw_size_per_qp / 2);
>>> +
>>> + /* In eDMA mode, decouple from MW sizing and force ring-sized entries */
>>> + edma_total = 2 * sizeof(struct ntb_edma_ring);
>>> + if (rx_info_offset < edma_total) {
>>> + dev_err(&ndev->dev, "Ring space requires %luB (>=%uB)\n",
>>> + edma_total, rx_info_offset);
>>> + return -EINVAL;
>>> + }
>>> + qp->tx_max_entry = NTB_EDMA_RING_ENTRIES;
>>> + qp->rx_max_entry = NTB_EDMA_RING_ENTRIES;
>>> +
>>> + /*
>>> + * Checking to see if we have more entries than the default.
>>> + * We should add additional entries if that is the case so we
>>> + * can be in sync with the transport frames.
>>> + */
>>> + node = dev_to_node(&ndev->dev);
>>> + for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) {
>>> + entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
>>> + if (!entry)
>>> + return -ENOMEM;
>>> +
>>> + entry->qp = qp;
>>> + INIT_WORK(&entry->dma_work, ntb_transport_edma_rc_dma_work);
>>> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
>>> + &qp->rx_free_q);
>>> + qp->rx_alloc_entry++;
>>> + }
>>> +
>>> + memset(qp->rx_buff, 0, edma_total);
>>> +
>>> + qp->rx_pkts = 0;
>>> + qp->tx_pkts = 0;
>>> +
>>> + return 0;
>>> +}
>>> +
>>> +static int ntb_transport_edma_ep_read_complete(struct ntb_transport_qp *qp)
>>> +{
>>> + struct device *dma_dev = get_dma_dev(qp->ndev);
>>> + struct ntb_queue_entry *entry;
>>> + struct ntb_edma_desc *in;
>>> + unsigned int len;
>>> + u32 idx;
>>> +
>>> + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_TAIL_RD_EP_I(qp)),
>>> + qp->rd_cons) == 0)
>>> + return 0;
>>> +
>>> + idx = ntb_edma_ring_idx(qp->rd_cons);
>>> + in = NTB_DESC_RD_EP_I(qp, idx);
>>> + if (!(in->flags & DESC_DONE_FLAG))
>>> + return 0;
>>> +
>>> + in->flags = 0;
>>> + len = in->len; /* might be smaller than entry->len */
>>> +
>>> + entry = (struct ntb_queue_entry *)(in->data);
>>> + if (WARN_ON(!entry))
>>> + return 0;
>>> +
>>> + if (in->flags & LINK_DOWN_FLAG) {
>>> + ntb_qp_link_down(qp);
>>> + qp->rd_cons++;
>>> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
>>> + return 1;
>>> + }
>>> +
>>> + dma_unmap_single(dma_dev, entry->addr, entry->len, DMA_FROM_DEVICE);
>>> +
>>> + qp->rx_bytes += len;
>>> + qp->rx_pkts++;
>>> + qp->rd_cons++;
>>> +
>>> + if (qp->rx_handler && qp->client_ready)
>>> + qp->rx_handler(qp, qp->cb_data, entry->cb_data, len);
>>> +
>>> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
>>> + return 1;
>>> +}
>>> +
>>> +static int ntb_transport_edma_ep_write_complete(struct ntb_transport_qp *qp)
>>> +{
>>> + struct ntb_queue_entry *entry;
>>> + struct ntb_edma_desc *in;
>>> + u32 idx;
>>> +
>>> + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_TAIL_WR_EP_I(qp)),
>>> + qp->wr_cons) == 0)
>>> + return 0;
>>> +
>>> + idx = ntb_edma_ring_idx(qp->wr_cons);
>>> + in = NTB_DESC_WR_EP_I(qp, idx);
>>> +
>>> + entry = (struct ntb_queue_entry *)(in->data);
>>> + if (WARN_ON(!entry))
>>> + return 0;
>>> +
>>> + qp->wr_cons++;
>>> +
>>> + if (qp->tx_handler)
>>> + qp->tx_handler(qp, qp->cb_data, entry->cb_data, entry->len);
>>> +
>>> + ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q);
>>> + return 1;
>>> +}
>>> +
>>> +static void ntb_transport_edma_ep_read_work(struct work_struct *work)
>>> +{
>>> + struct ntb_transport_qp *qp = container_of(
>>> + work, struct ntb_transport_qp, read_work);
>>> + unsigned int i;
>>> +
>>> + for (i = 0; i < NTB_EDMA_MAX_POLL; i++) {
>>> + if (!ntb_transport_edma_ep_read_complete(qp))
>>> + break;
>>> + }
>>> +
>>> + if (ntb_transport_edma_ep_read_complete(qp))
>>> + queue_work(qp->transport->wq, &qp->read_work);
>>> +}
>>> +
>>> +static void ntb_transport_edma_ep_write_work(struct work_struct *work)
>>> +{
>>> + struct ntb_transport_qp *qp = container_of(
>>> + work, struct ntb_transport_qp, write_work);
>>> + unsigned int i;
>>> +
>>> + for (i = 0; i < NTB_EDMA_MAX_POLL; i++) {
>>> + if (!ntb_transport_edma_ep_write_complete(qp))
>>> + break;
>>> + }
>>> +
>>> + if (ntb_transport_edma_ep_write_complete(qp))
>>> + queue_work(qp->transport->wq, &qp->write_work);
>>> +}
>>> +
>>> +static void ntb_transport_edma_rc_write_complete_work(struct work_struct *work)
>>> +{
>>> + struct ntb_transport_qp *qp = container_of(
>>> + work, struct ntb_transport_qp, write_work);
>>> + struct ntb_queue_entry *entry;
>>> + struct ntb_edma_desc *in;
>>> + unsigned int len;
>>> + void *cb_data;
>>> + u32 idx;
>>> +
>>> + while (ntb_edma_ring_used_entry(READ_ONCE(qp->wr_issue),
>>> + qp->wr_cons) != 0) {
>>> + /* Paired with smp_wmb() in ntb_transport_edma_rc_poll() */
>>> + smp_rmb();
>>> +
>>> + idx = ntb_edma_ring_idx(qp->wr_cons);
>>> + in = NTB_DESC_WR_RC_I(qp, idx);
>>> + entry = (struct ntb_queue_entry *)READ_ONCE(in->data);
>>> + if (!entry || !(entry->flags & DESC_DONE_FLAG))
>>> + break;
>>> +
>>> + in->data = 0;
>>> +
>>> + cb_data = entry->cb_data;
>>> + len = entry->len;
>>> +
>>> + iowrite32(++qp->wr_cons, NTB_TAIL_WR_RC_O(qp));
>>> +
>>> + if (unlikely(entry->flags & LINK_DOWN_FLAG)) {
>>> + ntb_qp_link_down(qp);
>>> + continue;
>>> + }
>>> +
>>> + ntb_transport_edma_notify_peer(qp);
>>> +
>>> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
>>> +
>>> + if (qp->rx_handler && qp->client_ready)
>>> + qp->rx_handler(qp, qp->cb_data, cb_data, len);
>>> +
>>> + /* stat updates */
>>> + qp->rx_bytes += len;
>>> + qp->rx_pkts++;
>>> + }
>>> +}
>>> +
>>> +static void ntb_transport_edma_rc_write_cb(void *data,
>>> + const struct dmaengine_result *res)
>>> +{
>>> + struct ntb_queue_entry *entry = data;
>>> + struct ntb_transport_qp *qp = entry->qp;
>>> + struct ntb_transport_ctx *nt = qp->transport;
>>> + enum dmaengine_tx_result dma_err = res->result;
>>> + struct device *dma_dev = get_dma_dev(qp->ndev);
>>> +
>>> + switch (dma_err) {
>>> + case DMA_TRANS_READ_FAILED:
>>> + case DMA_TRANS_WRITE_FAILED:
>>> + case DMA_TRANS_ABORTED:
>>> + entry->errors++;
>>> + entry->len = -EIO;
>>> + break;
>>> + case DMA_TRANS_NOERROR:
>>> + default:
>>> + break;
>>> + }
>>> + dma_unmap_sg(dma_dev, &entry->sgl, 1, DMA_FROM_DEVICE);
>>> + sg_dma_address(&entry->sgl) = 0;
>>> +
>>> + entry->flags |= DESC_DONE_FLAG;
>>> +
>>> + queue_work(nt->wq, &qp->write_work);
>>> +}
>>> +
>>> +static void ntb_transport_edma_rc_read_complete_work(struct work_struct *work)
>>> +{
>>> + struct ntb_transport_qp *qp = container_of(
>>> + work, struct ntb_transport_qp, read_work);
>>> + struct ntb_edma_desc *in, __iomem *out;
>>> + struct ntb_queue_entry *entry;
>>> + unsigned int len;
>>> + void *cb_data;
>>> + u32 idx;
>>> +
>>> + while (ntb_edma_ring_used_entry(READ_ONCE(qp->rd_issue),
>>> + qp->rd_cons) != 0) {
>>> + /* Paired with smp_wmb() in ntb_transport_edma_rc_tx_enqueue() */
>>> + smp_rmb();
>>> +
>>> + idx = ntb_edma_ring_idx(qp->rd_cons);
>>> + in = NTB_DESC_RD_RC_I(qp, idx);
>>> + entry = (struct ntb_queue_entry *)in->data;
>>> + if (!entry || !(entry->flags & DESC_DONE_FLAG))
>>> + break;
>>> +
>>> + in->data = 0;
>>> +
>>> + cb_data = entry->cb_data;
>>> + len = entry->len;
>>> +
>>> + out = NTB_DESC_RD_RC_O(qp, idx);
>>> +
>>> + WRITE_ONCE(qp->rd_cons, qp->rd_cons + 1);
>>> +
>>> + /*
>>> + * No need to add barrier in-between to enforce ordering here.
>>> + * The other side proceeds only after both flags and tail are
>>> + * updated.
>>> + */
>>> + iowrite32(entry->flags, &out->flags);
>>> + iowrite32(qp->rd_cons, NTB_TAIL_RD_RC_O(qp));
>>> +
>>> + ntb_transport_edma_notify_peer(qp);
>>> +
>>> + ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
>>> + &qp->tx_free_q);
>>> +
>>> + if (qp->tx_handler)
>>> + qp->tx_handler(qp, qp->cb_data, cb_data, len);
>>> +
>>> + /* stat updates */
>>> + qp->tx_bytes += len;
>>> + qp->tx_pkts++;
>>> + }
>>> +}
>>> +
>>> +static void ntb_transport_edma_rc_read_cb(void *data,
>>> + const struct dmaengine_result *res)
>>> +{
>>> + struct ntb_queue_entry *entry = data;
>>> + struct ntb_transport_qp *qp = entry->qp;
>>> + struct ntb_transport_ctx *nt = qp->transport;
>>> + struct device *dma_dev = get_dma_dev(qp->ndev);
>>> + enum dmaengine_tx_result dma_err = res->result;
>>> +
>>> + switch (dma_err) {
>>> + case DMA_TRANS_READ_FAILED:
>>> + case DMA_TRANS_WRITE_FAILED:
>>> + case DMA_TRANS_ABORTED:
>>> + entry->errors++;
>>> + entry->len = -EIO;
>>> + break;
>>> + case DMA_TRANS_NOERROR:
>>> + default:
>>> + break;
>>> + }
>>> + dma_unmap_sg(dma_dev, &entry->sgl, 1, DMA_TO_DEVICE);
>>> + sg_dma_address(&entry->sgl) = 0;
>>> +
>>> + entry->flags |= DESC_DONE_FLAG;
>>> +
>>> + queue_work(nt->wq, &qp->read_work);
>>> +}
>>> +
>>> +static int ntb_transport_edma_rc_write_start(struct device *d,
>>> + struct dma_chan *chan, size_t len,
>>> + dma_addr_t ep_src, void *rc_dst,
>>> + struct ntb_queue_entry *entry)
>>> +{
>>> + struct scatterlist *sgl = &entry->sgl;
>>> + struct dma_async_tx_descriptor *txd;
>>> + struct dma_slave_config cfg;
>>> + dma_cookie_t cookie;
>>> + int nents, rc;
>>> +
>>> + if (!d)
>>> + return -ENODEV;
>>> +
>>> + if (!chan)
>>> + return -ENXIO;
>>> +
>>> + if (WARN_ON(!ep_src || !rc_dst))
>>> + return -EINVAL;
>>> +
>>> + if (WARN_ON(sg_dma_address(sgl)))
>>> + return -EINVAL;
>>> +
>>> + sg_init_one(sgl, rc_dst, len);
>>> + nents = dma_map_sg(d, sgl, 1, DMA_FROM_DEVICE);
>>> + if (nents <= 0)
>>> + return -EIO;
>>> +
>>> + memset(&cfg, 0, sizeof(cfg));
>>> + cfg.src_addr = ep_src;
>>> + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
>>> + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
>>> + cfg.direction = DMA_DEV_TO_MEM;
>>> + rc = dmaengine_slave_config(chan, &cfg);
>>> + if (rc)
>>> + goto out_unmap;
>>> +
>>> + txd = dmaengine_prep_slave_sg(chan, sgl, 1, DMA_DEV_TO_MEM,
>>> + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
>>> + if (!txd) {
>>> + rc = -EIO;
>>> + goto out_unmap;
>>> + }
>>> +
>>> + txd->callback_result = ntb_transport_edma_rc_write_cb;
>>> + txd->callback_param = entry;
>>> +
>>> + cookie = dmaengine_submit(txd);
>>> + if (dma_submit_error(cookie)) {
>>> + rc = -EIO;
>>> + goto out_unmap;
>>> + }
>>> + dma_async_issue_pending(chan);
>>> + return 0;
>>> +out_unmap:
>>> + dma_unmap_sg(d, sgl, 1, DMA_FROM_DEVICE);
>>> + return rc;
>>> +}
>>> +
>>> +static int ntb_transport_edma_rc_read_start(struct device *d,
>>> + struct dma_chan *chan, size_t len,
>>> + void *rc_src, dma_addr_t ep_dst,
>>> + struct ntb_queue_entry *entry)
>>> +{
>>> + struct scatterlist *sgl = &entry->sgl;
>>> + struct dma_async_tx_descriptor *txd;
>>> + struct dma_slave_config cfg;
>>> + dma_cookie_t cookie;
>>> + int nents, rc;
>>> +
>>> + if (!d)
>>> + return -ENODEV;
>>> +
>>> + if (!chan)
>>> + return -ENXIO;
>>> +
>>> + if (WARN_ON(!rc_src || !ep_dst))
>>> + return -EINVAL;
>>> +
>>> + if (WARN_ON(sg_dma_address(sgl)))
>>> + return -EINVAL;
>>> +
>>> + sg_init_one(sgl, rc_src, len);
>>> + nents = dma_map_sg(d, sgl, 1, DMA_TO_DEVICE);
>>> + if (nents <= 0)
>>> + return -EIO;
>>> +
>>> + memset(&cfg, 0, sizeof(cfg));
>>> + cfg.dst_addr = ep_dst;
>>> + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
>>> + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
>>> + cfg.direction = DMA_MEM_TO_DEV;
>>> + rc = dmaengine_slave_config(chan, &cfg);
>>> + if (rc)
>>> + goto out_unmap;
>>> +
>>> + txd = dmaengine_prep_slave_sg(chan, sgl, 1, DMA_MEM_TO_DEV,
>>> + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
>>> + if (!txd) {
>>> + rc = -EIO;
>>> + goto out_unmap;
>>> + }
>>> +
>>> + txd->callback_result = ntb_transport_edma_rc_read_cb;
>>> + txd->callback_param = entry;
>>> +
>>> + cookie = dmaengine_submit(txd);
>>> + if (dma_submit_error(cookie)) {
>>> + rc = -EIO;
>>> + goto out_unmap;
>>> + }
>>> + dma_async_issue_pending(chan);
>>> + return 0;
>>> +out_unmap:
>>> + dma_unmap_sg(d, sgl, 1, DMA_TO_DEVICE);
>>> + return rc;
>>> +}
>>> +
>>> +static void ntb_transport_edma_rc_dma_work(struct work_struct *work)
>>> +{
>>> + struct ntb_queue_entry *entry = container_of(
>>> + work, struct ntb_queue_entry, dma_work);
>>> + struct ntb_transport_qp *qp = entry->qp;
>>> + struct ntb_transport_ctx *nt = qp->transport;
>>> + struct device *dma_dev = get_dma_dev(qp->ndev);
>>> + struct dma_chan *chan;
>>> + int rc;
>>> +
>>> + chan = ntb_edma_pick_chan(&nt->edma, REMOTE_EDMA_WRITE);
>>> + rc = ntb_transport_edma_rc_write_start(dma_dev, chan, entry->len,
>>> + entry->addr, entry->buf, entry);
>>> + if (rc) {
>>> + entry->errors++;
>>> + entry->len = -EIO;
>>> + entry->flags |= DESC_DONE_FLAG;
>>> + queue_work(nt->wq, &qp->write_work);
>>> + return;
>>> + }
>>> +}
>>> +
>>> +static void ntb_transport_edma_rc_poll(struct ntb_transport_qp *qp)
>>> +{
>>> + struct ntb_transport_ctx *nt = qp->transport;
>>> + unsigned int budget = NTB_EDMA_MAX_POLL;
>>> + struct ntb_queue_entry *entry;
>>> + struct ntb_edma_desc *in;
>>> + dma_addr_t ep_src;
>>> + u32 len, idx;
>>> +
>>> + while (budget--) {
>>> + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_HEAD_WR_RC_I(qp)),
>>> + qp->wr_issue) == 0)
>>> + break;
>>> +
>>> + idx = ntb_edma_ring_idx(qp->wr_issue);
>>> + in = NTB_DESC_WR_RC_I(qp, idx);
>>> +
>>> + len = READ_ONCE(in->len);
>>> + ep_src = (dma_addr_t)READ_ONCE(in->addr);
>>> +
>>> + /* Prepare 'entry' for write completion */
>>> + entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q);
>>> + if (!entry) {
>>> + qp->rx_err_no_buf++;
>>> + break;
>>> + }
>>> + if (WARN_ON(entry->flags & DESC_DONE_FLAG))
>>> + entry->flags &= ~DESC_DONE_FLAG;
>>> + entry->len = len; /* NB. entry->len can be <=0 */
>>> + entry->addr = ep_src;
>>> +
>>> + /*
>>> + * ntb_transport_edma_rc_write_complete_work() checks entry->flags
>>> + * so it needs to be set before wr_issue++.
>>> + */
>>> + in->data = (uintptr_t)entry;
>>> +
>>> + /* Ensure in->data visible before wr_issue++ */
>>> + smp_wmb();
>>> +
>>> + WRITE_ONCE(qp->wr_issue, qp->wr_issue + 1);
>>> +
>>> + if (!len) {
>>> + entry->flags |= DESC_DONE_FLAG;
>>> + queue_work(nt->wq, &qp->write_work);
>>> + continue;
>>> + }
>>> +
>>> + if (in->flags & LINK_DOWN_FLAG) {
>>> + dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n");
>>> + entry->flags |= DESC_DONE_FLAG | LINK_DOWN_FLAG;
>>> + queue_work(nt->wq, &qp->write_work);
>>> + continue;
>>> + }
>>> +
>>> + queue_work(nt->wq, &entry->dma_work);
>>> + }
>>> +
>>> + if (!budget)
>>> + tasklet_schedule(&qp->rxc_db_work);
>>> +}
>>> +
>>> +static int ntb_transport_edma_rc_tx_enqueue(struct ntb_transport_qp *qp,
>>> + struct ntb_queue_entry *entry)
>>> +{
>>> + struct device *dma_dev = get_dma_dev(qp->ndev);
>>> + struct ntb_transport_ctx *nt = qp->transport;
>>> + struct ntb_edma_desc *in, __iomem *out;
>>> + unsigned int len = entry->len;
>>> + struct dma_chan *chan;
>>> + u32 issue, idx, head;
>>> + dma_addr_t ep_dst;
>>> + int rc;
>>> +
>>> + WARN_ON_ONCE(entry->flags & DESC_DONE_FLAG);
>>> +
>>> + scoped_guard(spinlock_irqsave, &qp->rc_lock) {
>>> + head = READ_ONCE(*NTB_HEAD_RD_RC_I(qp));
>>> + issue = qp->rd_issue;
>>> + if (ntb_edma_ring_used_entry(head, issue) == 0) {
>>> + qp->tx_ring_full++;
>>> + return -ENOSPC;
>>> + }
>>> +
>>> + /*
>>> + * ntb_transport_edma_rc_read_complete_work() checks entry->flags
>>> + * so it needs to be set before rd_issue++.
>>> + */
>>> + idx = ntb_edma_ring_idx(issue);
>>> + in = NTB_DESC_RD_RC_I(qp, idx);
>>> + in->data = (uintptr_t)entry;
>>> +
>>> + /* Make in->data visible before rd_issue++ */
>>> + smp_wmb();
>>> +
>>> + WRITE_ONCE(qp->rd_issue, qp->rd_issue + 1);
>>> + }
>>> +
>>> + /* Publish the final transfer length to the EP side */
>>> + out = NTB_DESC_RD_RC_O(qp, idx);
>>> + iowrite32(len, &out->len);
>>> + ioread32(&out->len);
>>> +
>>> + if (unlikely(!len)) {
>>> + entry->flags |= DESC_DONE_FLAG;
>>> + queue_work(nt->wq, &qp->read_work);
>>> + return 0;
>>> + }
>>> +
>>> + /* Paired with dma_wmb() in ntb_transport_edma_ep_rx_enqueue() */
>>> + dma_rmb();
>>> +
>>> + /* kick remote eDMA read transfer */
>>> + ep_dst = (dma_addr_t)in->addr;
>>> + chan = ntb_edma_pick_chan(&nt->edma, REMOTE_EDMA_READ);
>>> + rc = ntb_transport_edma_rc_read_start(dma_dev, chan, len,
>>> + entry->buf, ep_dst, entry);
>>> + if (rc) {
>>> + entry->errors++;
>>> + entry->len = -EIO;
>>> + entry->flags |= DESC_DONE_FLAG;
>>> + queue_work(nt->wq, &qp->read_work);
>>> + }
>>> + return 0;
>>> +}
>>> +
>>> +static int ntb_transport_edma_ep_tx_enqueue(struct ntb_transport_qp *qp,
>>> + struct ntb_queue_entry *entry)
>>> +{
>>> + struct device *dma_dev = get_dma_dev(qp->ndev);
>>> + struct ntb_edma_desc *in, __iomem *out;
>>> + unsigned int len = entry->len;
>>> + dma_addr_t ep_src = 0;
>>> + u32 idx;
>>> + int rc;
>>> +
>>> + if (likely(len)) {
>>> + ep_src = dma_map_single(dma_dev, entry->buf, len,
>>> + DMA_TO_DEVICE);
>>> + rc = dma_mapping_error(dma_dev, ep_src);
>>> + if (rc)
>>> + return rc;
>>> + }
>>> +
>>> + scoped_guard(spinlock_irqsave, &qp->ep_tx_lock) {
>>> + if (ntb_edma_ring_full(qp->wr_prod, qp->wr_cons)) {
>>> + rc = -ENOSPC;
>>> + qp->tx_ring_full++;
>>> + goto out_unmap;
>>> + }
>>> +
>>> + idx = ntb_edma_ring_idx(qp->wr_prod);
>>> + in = NTB_DESC_WR_EP_I(qp, idx);
>>> + out = NTB_DESC_WR_EP_O(qp, idx);
>>> +
>>> + WARN_ON(in->flags & DESC_DONE_FLAG);
>>> + WARN_ON(entry->flags & DESC_DONE_FLAG);
>>> + in->flags = 0;
>>> + in->data = (uintptr_t)entry;
>>> + entry->addr = ep_src;
>>> +
>>> + iowrite32(len, &out->len);
>>> + iowrite32(entry->flags, &out->flags);
>>> + iowrite64(ep_src, &out->addr);
>>> + WRITE_ONCE(qp->wr_prod, qp->wr_prod + 1);
>>> +
>>> + dma_wmb();
>>> + iowrite32(qp->wr_prod, NTB_HEAD_WR_EP_O(qp));
>>> +
>>> + qp->tx_bytes += len;
>>> + qp->tx_pkts++;
>>> + }
>>> +
>>> + ntb_transport_edma_notify_peer(qp);
>>> +
>>> + return 0;
>>> +out_unmap:
>>> + if (likely(len))
>>> + dma_unmap_single(dma_dev, ep_src, len, DMA_TO_DEVICE);
>>> + return rc;
>>> +}
>>> +
>>> +static int ntb_transport_edma_tx_enqueue(struct ntb_transport_qp *qp,
>>> + struct ntb_queue_entry *entry,
>>> + void *cb, void *data, unsigned int len,
>>> + unsigned int flags)
>>> +{
>>> + struct device *dma_dev;
>>> +
>>> + if (entry->addr) {
>>> + /* Deferred unmap */
>>> + dma_dev = get_dma_dev(qp->ndev);
>>> + dma_unmap_single(dma_dev, entry->addr, entry->len, DMA_TO_DEVICE);
>>> + }
>>> +
>>> + entry->cb_data = cb;
>>> + entry->buf = data;
>>> + entry->len = len;
>>> + entry->flags = flags;
>>> + entry->errors = 0;
>>> + entry->addr = 0;
>>> +
>>> + WARN_ON_ONCE(!ntb_qp_edma_enabled(qp));
>>> +
>>> + if (ntb_qp_edma_is_ep(qp))
>>> + return ntb_transport_edma_ep_tx_enqueue(qp, entry);
>>> + else
>>> + return ntb_transport_edma_rc_tx_enqueue(qp, entry);
>>> +}
>>> +
>>> +static int ntb_transport_edma_ep_rx_enqueue(struct ntb_transport_qp *qp,
>>> + struct ntb_queue_entry *entry)
>>> +{
>>> + struct device *dma_dev = get_dma_dev(qp->ndev);
>>> + struct ntb_edma_desc *in, __iomem *out;
>>> + unsigned int len = entry->len;
>>> + void *data = entry->buf;
>>> + dma_addr_t ep_dst;
>>> + u32 idx;
>>> + int rc;
>>> +
>>> + ep_dst = dma_map_single(dma_dev, data, len, DMA_FROM_DEVICE);
>>> + rc = dma_mapping_error(dma_dev, ep_dst);
>>> + if (rc)
>>> + return rc;
>>> +
>>> + scoped_guard(spinlock_bh, &qp->ep_rx_lock) {
>>> + if (ntb_edma_ring_full(READ_ONCE(qp->rd_prod),
>>> + READ_ONCE(qp->rd_cons))) {
>>> + rc = -ENOSPC;
>>> + goto out_unmap;
>>> + }
>>> +
>>> + idx = ntb_edma_ring_idx(qp->rd_prod);
>>> + in = NTB_DESC_RD_EP_I(qp, idx);
>>> + out = NTB_DESC_RD_EP_O(qp, idx);
>>> +
>>> + iowrite32(len, &out->len);
>>> + iowrite64(ep_dst, &out->addr);
>>> +
>>> + WARN_ON(in->flags & DESC_DONE_FLAG);
>>> + in->data = (uintptr_t)entry;
>>> + entry->addr = ep_dst;
>>> +
>>> + /* Ensure len/addr are visible before the head update */
>>> + dma_wmb();
>>> +
>>> + WRITE_ONCE(qp->rd_prod, qp->rd_prod + 1);
>>> + iowrite32(qp->rd_prod, NTB_HEAD_RD_EP_O(qp));
>>> + }
>>> + return 0;
>>> +out_unmap:
>>> + dma_unmap_single(dma_dev, ep_dst, len, DMA_FROM_DEVICE);
>>> + return rc;
>>> +}
>>> +
>>> +static int ntb_transport_edma_rx_enqueue(struct ntb_transport_qp *qp,
>>> + struct ntb_queue_entry *entry)
>>> +{
>>> + int rc;
>>> +
>>> + /* The behaviour is the same as the default backend for RC side */
>>> + if (ntb_qp_edma_is_ep(qp)) {
>>> + rc = ntb_transport_edma_ep_rx_enqueue(qp, entry);
>>> + if (rc) {
>>> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
>>> + &qp->rx_free_q);
>>> + return rc;
>>> + }
>>> + }
>>> +
>>> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q);
>>> +
>>> + if (qp->active)
>>> + tasklet_schedule(&qp->rxc_db_work);
>>> +
>>> + return 0;
>>> +}
>>> +
>>> +static void ntb_transport_edma_rx_poll(struct ntb_transport_qp *qp)
>>> +{
>>> + struct ntb_transport_ctx *nt = qp->transport;
>>> +
>>> + if (ntb_qp_edma_is_rc(qp))
>>> + ntb_transport_edma_rc_poll(qp);
>>> + else if (ntb_qp_edma_is_ep(qp)) {
>>> + /*
>>> + * Make sure we poll the rings even if an eDMA interrupt is
>>> + * cleared on the RC side earlier.
>>> + */
>>> + queue_work(nt->wq, &qp->read_work);
>>> + queue_work(nt->wq, &qp->write_work);
>>> + } else
>>> + /* Unreachable */
>>> + WARN_ON_ONCE(1);
>>> +}
>>> +
>>> +static void ntb_transport_edma_read_work(struct work_struct *work)
>>> +{
>>> + struct ntb_transport_qp *qp = container_of(
>>> + work, struct ntb_transport_qp, read_work);
>>> +
>>> + if (ntb_qp_edma_is_rc(qp))
>>> + ntb_transport_edma_rc_read_complete_work(work);
>>> + else if (ntb_qp_edma_is_ep(qp))
>>> + ntb_transport_edma_ep_read_work(work);
>>> + else
>>> + /* Unreachable */
>>> + WARN_ON_ONCE(1);
>>> +}
>>> +
>>> +static void ntb_transport_edma_write_work(struct work_struct *work)
>>> +{
>>> + struct ntb_transport_qp *qp = container_of(
>>> + work, struct ntb_transport_qp, write_work);
>>> +
>>> + if (ntb_qp_edma_is_rc(qp))
>>> + ntb_transport_edma_rc_write_complete_work(work);
>>> + else if (ntb_qp_edma_is_ep(qp))
>>> + ntb_transport_edma_ep_write_work(work);
>>> + else
>>> + /* Unreachable */
>>> + WARN_ON_ONCE(1);
>>> +}
>>> +
>>> +static void ntb_transport_edma_init_queue(struct ntb_transport_ctx *nt,
>>> + unsigned int qp_num)
>>> +{
>>> + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
>>> +
>>> + qp->wr_cons = 0;
>>> + qp->rd_cons = 0;
>>> + qp->wr_prod = 0;
>>> + qp->rd_prod = 0;
>>> + qp->wr_issue = 0;
>>> + qp->rd_issue = 0;
>>> +
>>> + INIT_WORK(&qp->db_work, ntb_transport_edma_db_work);
>>> + INIT_WORK(&qp->read_work, ntb_transport_edma_read_work);
>>> + INIT_WORK(&qp->write_work, ntb_transport_edma_write_work);
>>> +}
>>> +
>>> +static void ntb_transport_edma_create_queue(struct ntb_transport_ctx *nt,
>>> + struct ntb_transport_qp *qp)
>>> +{
>>> + spin_lock_init(&qp->ep_tx_lock);
>>> + spin_lock_init(&qp->ep_rx_lock);
>>> + spin_lock_init(&qp->rc_lock);
>>> +}
>>> +
>>> +static const struct ntb_transport_backend_ops edma_backend_ops = {
>>> + .setup_qp_mw = ntb_transport_edma_setup_qp_mw,
>>> + .tx_free_entry = ntb_transport_edma_tx_free_entry,
>>> + .tx_enqueue = ntb_transport_edma_tx_enqueue,
>>> + .rx_enqueue = ntb_transport_edma_rx_enqueue,
>>> + .rx_poll = ntb_transport_edma_rx_poll,
>>> + .debugfs_stats_show = ntb_transport_edma_debugfs_stats_show,
>>> +};
>>> +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
>>> +
>>> /**
>>> * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
>>> * @qp: NTB transport layer queue to be enabled
>>
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [RFC PATCH v2 20/27] NTB: ntb_transport: Introduce remote eDMA backed transport mode
2025-12-02 14:53 ` Dave Jiang
@ 2025-12-03 14:19 ` Koichiro Den
0 siblings, 0 replies; 19+ messages in thread
From: Koichiro Den @ 2025-12-03 14:19 UTC (permalink / raw)
To: Dave Jiang
Cc: ntb, linux-pci, dmaengine, linux-kernel, Frank.Li, mani,
kwilczynski, kishon, bhelgaas, corbet, vkoul, jdmason, allenbh,
Basavaraj.Natikar, Shyam-sundar.S-k, kurt.schwemmer, logang,
jingoohan1, lpieralisi, robh, jbrunet, fancer.lancer, arnd,
pstanner, elfring
On Tue, Dec 02, 2025 at 07:53:59AM -0700, Dave Jiang wrote:
>
>
> On 12/1/25 11:59 PM, Koichiro Den wrote:
> > On Mon, Dec 01, 2025 at 02:46:41PM -0700, Dave Jiang wrote:
> >>
> >>
> >> On 11/29/25 9:03 AM, Koichiro Den wrote:
> >>> Add a new transport backend that uses a remote DesignWare eDMA engine
> >>> located on the NTB endpoint to move data between host and endpoint.
> >>>
> >>> In this mode:
> >>>
> >>> - The endpoint exposes a dedicated memory window that contains the
> >>> eDMA register block followed by a small control structure (struct
> >>> ntb_edma_info) and per-channel linked-list (LL) rings.
> >>>
> >>> - On the endpoint side, ntb_edma_setup_mws() allocates the control
> >>> structure and LL rings in endpoint memory, then programs an inbound
> >>> iATU region so that the host can access them via a peer MW.
> >>>
> >>> - On the host side, ntb_edma_setup_peer() ioremaps the peer MW, reads
> >>> ntb_edma_info and configures a dw-edma DMA device to use the LL
> >>> rings provided by the endpoint.
> >>>
> >>> - ntb_transport is extended with a new backend_ops implementation that
> >>> routes TX and RX enqueue/poll operations through the remote eDMA
> >>> rings while keeping the existing shared-memory backend intact.
> >>>
> >>> - The host signals the endpoint via a dedicated DMA read channel.
> >>> 'use_msi' module option is ignored when 'use_remote_edma=1'.
> >>>
> >>> The new mode is guarded by a Kconfig option (NTB_TRANSPORT_EDMA) and a
> >>> module parameter (use_remote_edma). When disabled, the existing
> >>> ntb_transport behaviour is unchanged.
> >>>
> >>> Signed-off-by: Koichiro Den <den@valinux.co.jp>
> >>> ---
> >>> drivers/ntb/Kconfig | 11 +
> >>> drivers/ntb/Makefile | 3 +
> >>> drivers/ntb/ntb_edma.c | 628 ++++++++
> >>> drivers/ntb/ntb_edma.h | 128 ++
> >>
> >> I briefly looked over the code. It feels like the EDMA bits should go in drivers/ntb/hw/ rather than drivers/ntb/ given it's pretty specific to the designware hardware. What sits in drivers/ntb should be generic APIs where a different vendor can utilize it and not having to adopt to designware hardware specifics. So maybe a bit more abstractions are needed?
> >
> > That makes sense, I'll reorganize things. Thank you for the suggestion.
>
> Also, since a new transport is being introduced. Please update Documentation/driver-api/ntb.rst. While the current documentation doesn't provide adaquate API documentation for ntb_transport APIs, hopefully the new transport can do better going forward. :) Thank you!
Thanks for the reminder. I'll update ntb.rst in the next revision (perhaps
as part of a split-out series).
Thank you for the review,
Koichiro
>
> DJ
>
> >
> >>
> >>> .../{ntb_transport.c => ntb_transport_core.c} | 1281 ++++++++++++++++-
> >>> 5 files changed, 2048 insertions(+), 3 deletions(-)
> >>> create mode 100644 drivers/ntb/ntb_edma.c
> >>> create mode 100644 drivers/ntb/ntb_edma.h
> >>> rename drivers/ntb/{ntb_transport.c => ntb_transport_core.c} (65%)
> >>>
> >>> diff --git a/drivers/ntb/Kconfig b/drivers/ntb/Kconfig
> >>> index df16c755b4da..db63f02bb116 100644
> >>> --- a/drivers/ntb/Kconfig
> >>> +++ b/drivers/ntb/Kconfig
> >>> @@ -37,4 +37,15 @@ config NTB_TRANSPORT
> >>>
> >>> If unsure, say N.
> >>>
> >>> +config NTB_TRANSPORT_EDMA
> >>> + bool "NTB Transport backed by remote eDMA"
> >>> + depends on NTB_TRANSPORT
> >>> + depends on PCI
> >>> + select DMA_ENGINE
> >>> + help
> >>> + Enable a transport backend that uses a remote DesignWare eDMA engine
> >>> + exposed through a dedicated NTB memory window. The host uses the
> >>> + endpoint's eDMA engine to move data in both directions.
> >>> + Say Y here if you intend to use the 'use_remote_edma' module parameter.
> >>> +
> >>> endif # NTB
> >>> diff --git a/drivers/ntb/Makefile b/drivers/ntb/Makefile
> >>> index 3a6fa181ff99..51f0e1e3aec7 100644
> >>> --- a/drivers/ntb/Makefile
> >>> +++ b/drivers/ntb/Makefile
> >>> @@ -4,3 +4,6 @@ obj-$(CONFIG_NTB_TRANSPORT) += ntb_transport.o
> >>>
> >>> ntb-y := core.o
> >>> ntb-$(CONFIG_NTB_MSI) += msi.o
> >>> +
> >>> +ntb_transport-y := ntb_transport_core.o
> >>> +ntb_transport-$(CONFIG_NTB_TRANSPORT_EDMA) += ntb_edma.o
> >>> diff --git a/drivers/ntb/ntb_edma.c b/drivers/ntb/ntb_edma.c
> >>> new file mode 100644
> >>> index 000000000000..cb35e0d56aa8
> >>> --- /dev/null
> >>> +++ b/drivers/ntb/ntb_edma.c
> >>> @@ -0,0 +1,628 @@
> >>> +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
> >>> +
> >>> +#include <linux/module.h>
> >>> +#include <linux/device.h>
> >>> +#include <linux/pci.h>
> >>> +#include <linux/ntb.h>
> >>> +#include <linux/io.h>
> >>> +#include <linux/iommu.h>
> >>> +#include <linux/dmaengine.h>
> >>> +#include <linux/pci-epc.h>
> >>> +#include <linux/dma/edma.h>
> >>> +#include <linux/irq.h>
> >>> +#include <linux/irqdomain.h>
> >>> +#include <linux/of.h>
> >>> +#include <linux/of_irq.h>
> >>> +#include <dt-bindings/interrupt-controller/arm-gic.h>
> >>> +
> >>> +#include "ntb_edma.h"
> >>> +
> >>> +/*
> >>> + * The interrupt register offsets below are taken from the DesignWare
> >>> + * eDMA "unrolled" register map (EDMA_MF_EDMA_UNROLL). The remote eDMA
> >>> + * backend currently only supports this layout.
> >>> + */
> >>> +#define DMA_WRITE_INT_STATUS_OFF 0x4c
> >>> +#define DMA_WRITE_INT_MASK_OFF 0x54
> >>> +#define DMA_WRITE_INT_CLEAR_OFF 0x58
> >>> +#define DMA_READ_INT_STATUS_OFF 0xa0
> >>> +#define DMA_READ_INT_MASK_OFF 0xa8
> >>> +#define DMA_READ_INT_CLEAR_OFF 0xac
> >>> +
> >>> +#define NTB_EDMA_NOTIFY_MAX_QP 64
> >>> +
> >>> +static unsigned int edma_spi = 417; /* 0x1a1 */
> >>> +module_param(edma_spi, uint, 0644);
> >>> +MODULE_PARM_DESC(edma_spi, "SPI number used by remote eDMA interrupt (EP local)");
> >>> +
> >>> +static u64 edma_regs_phys = 0xe65d5000;
> >>> +module_param(edma_regs_phys, ullong, 0644);
> >>> +MODULE_PARM_DESC(edma_regs_phys, "Physical base address of local eDMA registers (EP)");
> >>> +
> >>> +static unsigned long edma_regs_size = 0x1200;
> >>> +module_param(edma_regs_size, ulong, 0644);
> >>> +MODULE_PARM_DESC(edma_regs_size, "Size of the local eDMA register space (EP)");
> >>> +
> >>> +struct ntb_edma_intr {
> >>> + u32 db[NTB_EDMA_NOTIFY_MAX_QP];
> >>> +};
> >>> +
> >>> +struct ntb_edma_ctx {
> >>> + void *ll_wr_virt[EDMA_WR_CH_NUM];
> >>> + dma_addr_t ll_wr_phys[EDMA_WR_CH_NUM];
> >>> + void *ll_rd_virt[EDMA_RD_CH_NUM + 1];
> >>> + dma_addr_t ll_rd_phys[EDMA_RD_CH_NUM + 1];
> >>> +
> >>> + struct ntb_edma_intr *intr_ep_virt;
> >>> + dma_addr_t intr_ep_phys;
> >>> + struct ntb_edma_intr *intr_rc_virt;
> >>> + dma_addr_t intr_rc_phys;
> >>> + u32 notify_qp_max;
> >>> +
> >>> + bool initialized;
> >>> +};
> >>> +
> >>> +static struct ntb_edma_ctx edma_ctx;
> >>> +
> >>> +typedef void (*ntb_edma_interrupt_cb_t)(void *data, int qp_num);
> >>> +
> >>> +struct ntb_edma_interrupt {
> >>> + int virq;
> >>> + void __iomem *base;
> >>> + ntb_edma_interrupt_cb_t cb;
> >>> + void *data;
> >>> +};
> >>> +
> >>> +static struct ntb_edma_interrupt ntb_edma_intr;
> >>> +
> >>> +static int ntb_edma_map_spi_to_virq(struct device *dev, unsigned int spi)
> >>> +{
> >>> + struct device_node *np = dev_of_node(dev);
> >>> + struct device_node *parent;
> >>> + struct irq_fwspec fwspec = { 0 };
> >>> + int virq;
> >>> +
> >>> + parent = of_irq_find_parent(np);
> >>> + if (!parent)
> >>> + return -ENODEV;
> >>> +
> >>> + fwspec.fwnode = of_fwnode_handle(parent);
> >>> + fwspec.param_count = 3;
> >>> + fwspec.param[0] = GIC_SPI;
> >>> + fwspec.param[1] = spi;
> >>> + fwspec.param[2] = IRQ_TYPE_LEVEL_HIGH;
> >>> +
> >>> + virq = irq_create_fwspec_mapping(&fwspec);
> >>> + of_node_put(parent);
> >>> + return (virq > 0) ? virq : -EINVAL;
> >>> +}
> >>> +
> >>> +static irqreturn_t ntb_edma_isr(int irq, void *data)
> >>> +{
> >>> + struct ntb_edma_interrupt *v = data;
> >>> + u32 mask = BIT(EDMA_RD_CH_NUM);
> >>> + u32 i, val;
> >>> +
> >>> + /*
> >>> + * We do not ack interrupts here but instead we mask all local interrupt
> >>> + * sources except the read channel used for notification. This reduces
> >>> + * needless ISR invocations.
> >>> + *
> >>> + * In theory we could configure LIE=1/RIE=0 only for the notification
> >>> + * transfer (keeping all other channels at LIE=1/RIE=1), but that would
> >>> + * require intrusive changes to the dw-edma core.
> >>> + *
> >>> + * Note: The host side may have already cleared the read interrupt used
> >>> + * for notification, so reading DMA_READ_INT_CLEAR_OFF is not a reliable
> >>> + * way to detect it. As a result, we cannot reliably tell which specific
> >>> + * channel triggered this interrupt. intr_ep_virt->db[i] teaches us
> >>> + * instead.
> >>> + */
> >>> + iowrite32(~0x0, v->base + DMA_WRITE_INT_MASK_OFF);
> >>> + iowrite32(~mask, v->base + DMA_READ_INT_MASK_OFF);
> >>> +
> >>> + if (!v->cb || !edma_ctx.intr_ep_virt)
> >>> + return IRQ_HANDLED;
> >>> +
> >>> + for (i = 0; i < edma_ctx.notify_qp_max; i++) {
> >>> + val = READ_ONCE(edma_ctx.intr_ep_virt->db[i]);
> >>> + if (!val)
> >>> + continue;
> >>> +
> >>> + WRITE_ONCE(edma_ctx.intr_ep_virt->db[i], 0);
> >>> + v->cb(v->data, i);
> >>> + }
> >>> +
> >>> + return IRQ_HANDLED;
> >>> +}
> >>> +
> >>> +int ntb_edma_setup_isr(struct device *dev, struct device *epc_dev,
> >>> + ntb_edma_interrupt_cb_t cb, void *data)
> >>> +{
> >>> + struct ntb_edma_interrupt *v = &ntb_edma_intr;
> >>> + int virq = ntb_edma_map_spi_to_virq(epc_dev->parent, edma_spi);
> >>> + int ret;
> >>> +
> >>> + if (virq < 0) {
> >>> + dev_err(dev, "failed to get virq (%d)\n", virq);
> >>> + return virq;
> >>> + }
> >>> +
> >>> + v->virq = virq;
> >>> + v->cb = cb;
> >>> + v->data = data;
> >>> + if (edma_regs_phys && !v->base)
> >>> + v->base = devm_ioremap(dev, edma_regs_phys, edma_regs_size);
> >>> + if (!v->base) {
> >>> + dev_err(dev, "failed to setup v->base\n");
> >>> + return -1;
> >>> + }
> >>> + ret = devm_request_irq(dev, v->virq, ntb_edma_isr, 0, "ntb-edma", v);
> >>> + if (ret)
> >>> + return ret;
> >>> +
> >>> + if (v->base) {
> >>> + iowrite32(0x0, v->base + DMA_WRITE_INT_MASK_OFF);
> >>> + iowrite32(0x0, v->base + DMA_READ_INT_MASK_OFF);
> >>> + }
> >>> + return 0;
> >>> +}
> >>> +
> >>> +void ntb_edma_teardown_isr(struct device *dev)
> >>> +{
> >>> + struct ntb_edma_interrupt *v = &ntb_edma_intr;
> >>> +
> >>> + /* Mask all write/read interrupts so we don't get called again. */
> >>> + if (v->base) {
> >>> + iowrite32(~0x0, v->base + DMA_WRITE_INT_MASK_OFF);
> >>> + iowrite32(~0x0, v->base + DMA_READ_INT_MASK_OFF);
> >>> + }
> >>> +
> >>> + if (v->virq > 0)
> >>> + devm_free_irq(dev, v->virq, v);
> >>> +
> >>> + if (v->base)
> >>> + devm_iounmap(dev, v->base);
> >>> +
> >>> + v->virq = 0;
> >>> + v->cb = NULL;
> >>> + v->data = NULL;
> >>> +}
> >>> +
> >>> +int ntb_edma_setup_mws(struct ntb_dev *ndev)
> >>> +{
> >>> + const size_t info_bytes = PAGE_SIZE;
> >>> + resource_size_t size_max, offset;
> >>> + dma_addr_t intr_phys, info_phys;
> >>> + u32 wr_done = 0, rd_done = 0;
> >>> + struct ntb_edma_intr *intr;
> >>> + struct ntb_edma_info *info;
> >>> + int peer_mw, mw_index, rc;
> >>> + struct iommu_domain *dom;
> >>> + bool reg_mapped = false;
> >>> + size_t ll_bytes, size;
> >>> + struct pci_epc *epc;
> >>> + struct device *dev;
> >>> + unsigned long iova;
> >>> + phys_addr_t phys;
> >>> + u64 need;
> >>> + u32 i;
> >>> +
> >>> + /* +1 is for interruption */
> >>> + ll_bytes = (EDMA_WR_CH_NUM + EDMA_RD_CH_NUM + 1) * DMA_LLP_MEM_SIZE;
> >>> + need = EDMA_REG_SIZE + info_bytes + ll_bytes;
> >>> +
> >>> + epc = ntb_get_pci_epc(ndev);
> >>> + if (!epc)
> >>> + return -ENODEV;
> >>> + dev = epc->dev.parent;
> >>> +
> >>> + if (edma_ctx.initialized)
> >>> + return 0;
> >>> +
> >>> + info = dma_alloc_coherent(dev, info_bytes, &info_phys, GFP_KERNEL);
> >>> + if (!info)
> >>> + return -ENOMEM;
> >>> +
> >>> + memset(info, 0, info_bytes);
> >>> + info->magic = NTB_EDMA_INFO_MAGIC;
> >>> + info->wr_cnt = EDMA_WR_CH_NUM;
> >>> + info->rd_cnt = EDMA_RD_CH_NUM + 1; /* +1 for interruption */
> >>> + info->regs_phys = edma_regs_phys;
> >>> + info->ll_stride = DMA_LLP_MEM_SIZE;
> >>> +
> >>> + for (i = 0; i < EDMA_WR_CH_NUM; i++) {
> >>> + edma_ctx.ll_wr_virt[i] = dma_alloc_attrs(dev, DMA_LLP_MEM_SIZE,
> >>> + &edma_ctx.ll_wr_phys[i],
> >>> + GFP_KERNEL,
> >>> + DMA_ATTR_FORCE_CONTIGUOUS);
> >>> + if (!edma_ctx.ll_wr_virt[i]) {
> >>> + rc = -ENOMEM;
> >>> + goto err_free_ll;
> >>> + }
> >>> + wr_done++;
> >>> + info->ll_wr_phys[i] = edma_ctx.ll_wr_phys[i];
> >>> + }
> >>> + for (i = 0; i < EDMA_RD_CH_NUM + 1; i++) {
> >>> + edma_ctx.ll_rd_virt[i] = dma_alloc_attrs(dev, DMA_LLP_MEM_SIZE,
> >>> + &edma_ctx.ll_rd_phys[i],
> >>> + GFP_KERNEL,
> >>> + DMA_ATTR_FORCE_CONTIGUOUS);
> >>> + if (!edma_ctx.ll_rd_virt[i]) {
> >>> + rc = -ENOMEM;
> >>> + goto err_free_ll;
> >>> + }
> >>> + rd_done++;
> >>> + info->ll_rd_phys[i] = edma_ctx.ll_rd_phys[i];
> >>> + }
> >>> +
> >>> + /* For interruption */
> >>> + edma_ctx.notify_qp_max = NTB_EDMA_NOTIFY_MAX_QP;
> >>> + intr = dma_alloc_coherent(dev, sizeof(*intr), &intr_phys, GFP_KERNEL);
> >>> + if (!intr) {
> >>> + rc = -ENOMEM;
> >>> + goto err_free_ll;
> >>> + }
> >>> + memset(intr, 0, sizeof(*intr));
> >>> + edma_ctx.intr_ep_virt = intr;
> >>> + edma_ctx.intr_ep_phys = intr_phys;
> >>> + info->intr_dar_base = intr_phys;
> >>> +
> >>> + peer_mw = ntb_peer_mw_count(ndev);
> >>> + if (peer_mw <= 0) {
> >>> + rc = -ENODEV;
> >>> + goto err_free_ll;
> >>> + }
> >>> +
> >>> + mw_index = peer_mw - 1; /* last MW */
> >>> +
> >>> + rc = ntb_mw_get_align(ndev, 0, mw_index, 0, NULL, &size_max,
> >>> + &offset);
> >>> + if (rc)
> >>> + goto err_free_ll;
> >>> +
> >>> + if (size_max < need) {
> >>> + rc = -ENOSPC;
> >>> + goto err_free_ll;
> >>> + }
> >>> +
> >>> + /* Map register space (direct) */
> >>> + dom = iommu_get_domain_for_dev(dev);
> >>> + if (dom) {
> >>> + phys = edma_regs_phys & PAGE_MASK;
> >>> + size = PAGE_ALIGN(EDMA_REG_SIZE + edma_regs_phys - phys);
> >>> + iova = phys;
> >>> +
> >>> + rc = iommu_map(dom, iova, phys, EDMA_REG_SIZE,
> >>> + IOMMU_READ | IOMMU_WRITE | IOMMU_MMIO, GFP_KERNEL);
> >>> + if (rc)
> >>> + dev_err(&ndev->dev, "failed to create direct mapping for eDMA reg space\n");
> >>> + reg_mapped = true;
> >>> + }
> >>> +
> >>> + rc = ntb_mw_set_trans(ndev, 0, mw_index, edma_regs_phys, EDMA_REG_SIZE, offset);
> >>> + if (rc)
> >>> + goto err_unmap_reg;
> >>> +
> >>> + offset += EDMA_REG_SIZE;
> >>> +
> >>> + /* Map ntb_edma_info */
> >>> + rc = ntb_mw_set_trans(ndev, 0, mw_index, info_phys, info_bytes, offset);
> >>> + if (rc)
> >>> + goto err_clear_trans;
> >>> + offset += info_bytes;
> >>> +
> >>> + /* Map LL location */
> >>> + for (i = 0; i < EDMA_WR_CH_NUM; i++) {
> >>> + rc = ntb_mw_set_trans(ndev, 0, mw_index, edma_ctx.ll_wr_phys[i],
> >>> + DMA_LLP_MEM_SIZE, offset);
> >>> + if (rc)
> >>> + goto err_clear_trans;
> >>> + offset += DMA_LLP_MEM_SIZE;
> >>> + }
> >>> + for (i = 0; i < EDMA_RD_CH_NUM + 1; i++) {
> >>> + rc = ntb_mw_set_trans(ndev, 0, mw_index, edma_ctx.ll_rd_phys[i],
> >>> + DMA_LLP_MEM_SIZE, offset);
> >>> + if (rc)
> >>> + goto err_clear_trans;
> >>> + offset += DMA_LLP_MEM_SIZE;
> >>> + }
> >>> + edma_ctx.initialized = true;
> >>> +
> >>> + return 0;
> >>> +
> >>> +err_clear_trans:
> >>> + /*
> >>> + * Tear down the NTB translation window used for the eDMA MW.
> >>> + * There is no sub-range clear API for ntb_mw_set_trans(), so we
> >>> + * unconditionally drop the whole mapping on error.
> >>> + */
> >>> + ntb_mw_clear_trans(ndev, 0, mw_index);
> >>> +
> >>> +err_unmap_reg:
> >>> + if (reg_mapped)
> >>> + iommu_unmap(dom, iova, size);
> >>> +err_free_ll:
> >>> + while (rd_done--)
> >>> + dma_free_attrs(dev, DMA_LLP_MEM_SIZE,
> >>> + edma_ctx.ll_rd_virt[rd_done],
> >>> + edma_ctx.ll_rd_phys[rd_done],
> >>> + DMA_ATTR_FORCE_CONTIGUOUS);
> >>> + while (wr_done--)
> >>> + dma_free_attrs(dev, DMA_LLP_MEM_SIZE,
> >>> + edma_ctx.ll_wr_virt[wr_done],
> >>> + edma_ctx.ll_wr_phys[wr_done],
> >>> + DMA_ATTR_FORCE_CONTIGUOUS);
> >>> + if (edma_ctx.intr_ep_virt)
> >>> + dma_free_coherent(dev, sizeof(struct ntb_edma_intr),
> >>> + edma_ctx.intr_ep_virt,
> >>> + edma_ctx.intr_ep_phys);
> >>> + dma_free_coherent(dev, info_bytes, info, info_phys);
> >>> + return rc;
> >>> +}
> >>> +
> >>> +static int ntb_edma_irq_vector(struct device *dev, unsigned int nr)
> >>> +{
> >>> + struct pci_dev *pdev = to_pci_dev(dev);
> >>> + int ret, nvec;
> >>> +
> >>> + nvec = pci_msi_vec_count(pdev);
> >>> + for (; nr < nvec; nr++) {
> >>> + ret = pci_irq_vector(pdev, nr);
> >>> + if (!irq_has_action(ret))
> >>> + return ret;
> >>> + }
> >>> + return 0;
> >>> +}
> >>> +
> >>> +static const struct dw_edma_plat_ops ntb_edma_ops = {
> >>> + .irq_vector = ntb_edma_irq_vector,
> >>> +};
> >>> +
> >>> +int ntb_edma_setup_peer(struct ntb_dev *ndev)
> >>> +{
> >>> + struct ntb_edma_info *info;
> >>> + unsigned int wr_cnt, rd_cnt;
> >>> + struct dw_edma_chip *chip;
> >>> + void __iomem *edma_virt;
> >>> + phys_addr_t edma_phys;
> >>> + resource_size_t mw_size;
> >>> + u64 off = EDMA_REG_SIZE;
> >>> + int peer_mw, mw_index;
> >>> + unsigned int i;
> >>> + int ret;
> >>> +
> >>> + peer_mw = ntb_peer_mw_count(ndev);
> >>> + if (peer_mw <= 0)
> >>> + return -ENODEV;
> >>> +
> >>> + mw_index = peer_mw - 1; /* last MW */
> >>> +
> >>> + ret = ntb_peer_mw_get_addr(ndev, mw_index, &edma_phys,
> >>> + &mw_size);
> >>> + if (ret)
> >>> + return -1;
> >>> +
> >>> + edma_virt = ioremap(edma_phys, mw_size);
> >>> +
> >>> + chip = devm_kzalloc(&ndev->dev, sizeof(*chip), GFP_KERNEL);
> >>> + if (!chip) {
> >>> + ret = -ENOMEM;
> >>> + return ret;
> >>> + }
> >>> +
> >>> + chip->dev = &ndev->pdev->dev;
> >>> + chip->nr_irqs = 4;
> >>> + chip->ops = &ntb_edma_ops;
> >>> + chip->flags = 0;
> >>> + chip->reg_base = edma_virt;
> >>> + chip->mf = EDMA_MF_EDMA_UNROLL;
> >>> +
> >>> + info = edma_virt + off;
> >>> + if (info->magic != NTB_EDMA_INFO_MAGIC)
> >>> + return -EINVAL;
> >>> + wr_cnt = info->wr_cnt;
> >>> + rd_cnt = info->rd_cnt;
> >>> + chip->ll_wr_cnt = wr_cnt;
> >>> + chip->ll_rd_cnt = rd_cnt;
> >>> + off += PAGE_SIZE;
> >>> +
> >>> + edma_ctx.notify_qp_max = NTB_EDMA_NOTIFY_MAX_QP;
> >>> + edma_ctx.intr_ep_phys = info->intr_dar_base;
> >>> + if (edma_ctx.intr_ep_phys) {
> >>> + edma_ctx.intr_rc_virt =
> >>> + dma_alloc_coherent(&ndev->pdev->dev,
> >>> + sizeof(struct ntb_edma_intr),
> >>> + &edma_ctx.intr_rc_phys,
> >>> + GFP_KERNEL);
> >>> + if (!edma_ctx.intr_rc_virt)
> >>> + return -ENOMEM;
> >>> + memset(edma_ctx.intr_rc_virt, 0,
> >>> + sizeof(struct ntb_edma_intr));
> >>> + }
> >>> +
> >>> + for (i = 0; i < wr_cnt; i++) {
> >>> + chip->ll_region_wr[i].vaddr.io = edma_virt + off;
> >>> + chip->ll_region_wr[i].paddr = info->ll_wr_phys[i];
> >>> + chip->ll_region_wr[i].sz = DMA_LLP_MEM_SIZE;
> >>> + off += DMA_LLP_MEM_SIZE;
> >>> + }
> >>> + for (i = 0; i < rd_cnt; i++) {
> >>> + chip->ll_region_rd[i].vaddr.io = edma_virt + off;
> >>> + chip->ll_region_rd[i].paddr = info->ll_rd_phys[i];
> >>> + chip->ll_region_rd[i].sz = DMA_LLP_MEM_SIZE;
> >>> + off += DMA_LLP_MEM_SIZE;
> >>> + }
> >>> +
> >>> + if (!pci_dev_msi_enabled(ndev->pdev))
> >>> + return -ENXIO;
> >>> +
> >>> + ret = dw_edma_probe(chip);
> >>> + if (ret) {
> >>> + dev_err(&ndev->dev, "dw_edma_probe failed: %d\n", ret);
> >>> + return ret;
> >>> + }
> >>> +
> >>> + return 0;
> >>> +}
> >>> +
> >>> +struct ntb_edma_filter {
> >>> + struct device *dma_dev;
> >>> + u32 direction;
> >>> +};
> >>> +
> >>> +static bool ntb_edma_filter_fn(struct dma_chan *chan, void *arg)
> >>> +{
> >>> + struct ntb_edma_filter *filter = arg;
> >>> + u32 dir = filter->direction;
> >>> + struct dma_slave_caps caps;
> >>> + int ret;
> >>> +
> >>> + if (chan->device->dev != filter->dma_dev)
> >>> + return false;
> >>> +
> >>> + ret = dma_get_slave_caps(chan, &caps);
> >>> + if (ret < 0)
> >>> + return false;
> >>> +
> >>> + return !!(caps.directions & dir);
> >>> +}
> >>> +
> >>> +void ntb_edma_teardown_chans(struct ntb_edma_chans *edma)
> >>> +{
> >>> + unsigned int i;
> >>> +
> >>> + for (i = 0; i < edma->num_wr_chan; i++)
> >>> + dma_release_channel(edma->wr_chan[i]);
> >>> +
> >>> + for (i = 0; i < edma->num_rd_chan; i++)
> >>> + dma_release_channel(edma->rd_chan[i]);
> >>> +
> >>> + if (edma->intr_chan)
> >>> + dma_release_channel(edma->intr_chan);
> >>> +}
> >>> +
> >>> +int ntb_edma_setup_chans(struct device *dma_dev, struct ntb_edma_chans *edma)
> >>> +{
> >>> + struct ntb_edma_filter filter;
> >>> + dma_cap_mask_t dma_mask;
> >>> + unsigned int i;
> >>> +
> >>> + dma_cap_zero(dma_mask);
> >>> + dma_cap_set(DMA_SLAVE, dma_mask);
> >>> +
> >>> + memset(edma, 0, sizeof(*edma));
> >>> + edma->dev = dma_dev;
> >>> +
> >>> + filter.dma_dev = dma_dev;
> >>> + filter.direction = BIT(DMA_DEV_TO_MEM);
> >>> + for (i = 0; i < EDMA_WR_CH_NUM; i++) {
> >>> + edma->wr_chan[i] = dma_request_channel(dma_mask,
> >>> + ntb_edma_filter_fn,
> >>> + &filter);
> >>> + if (!edma->wr_chan[i])
> >>> + break;
> >>> + edma->num_wr_chan++;
> >>> + }
> >>> +
> >>> + filter.direction = BIT(DMA_MEM_TO_DEV);
> >>> + for (i = 0; i < EDMA_RD_CH_NUM; i++) {
> >>> + edma->rd_chan[i] = dma_request_channel(dma_mask,
> >>> + ntb_edma_filter_fn,
> >>> + &filter);
> >>> + if (!edma->rd_chan[i])
> >>> + break;
> >>> + edma->num_rd_chan++;
> >>> + }
> >>> +
> >>> + edma->intr_chan = dma_request_channel(dma_mask, ntb_edma_filter_fn,
> >>> + &filter);
> >>> + if (!edma->intr_chan)
> >>> + dev_warn(dma_dev,
> >>> + "Remote eDMA notify channel could not be allocated\n");
> >>> +
> >>> + if (!edma->num_wr_chan || !edma->num_rd_chan) {
> >>> + dev_warn(dma_dev, "Remote eDMA channels failed to initialize\n");
> >>> + ntb_edma_teardown_chans(edma);
> >>> + return -ENODEV;
> >>> + }
> >>> + return 0;
> >>> +}
> >>> +
> >>> +struct dma_chan *ntb_edma_pick_chan(struct ntb_edma_chans *edma,
> >>> + remote_edma_dir_t dir)
> >>> +{
> >>> + unsigned int n, cur, idx;
> >>> + struct dma_chan **chans;
> >>> + atomic_t *cur_chan;
> >>> +
> >>> + if (dir == REMOTE_EDMA_WRITE) {
> >>> + n = edma->num_wr_chan;
> >>> + chans = edma->wr_chan;
> >>> + cur_chan = &edma->cur_wr_chan;
> >>> + } else {
> >>> + n = edma->num_rd_chan;
> >>> + chans = edma->rd_chan;
> >>> + cur_chan = &edma->cur_rd_chan;
> >>> + }
> >>> + if (WARN_ON_ONCE(!n))
> >>> + return NULL;
> >>> +
> >>> + /* Simple round-robin */
> >>> + cur = (unsigned int)atomic_inc_return(cur_chan) - 1;
> >>> + idx = cur % n;
> >>> + return chans[idx];
> >>> +}
> >>> +
> >>> +int ntb_edma_notify_peer(struct ntb_edma_chans *edma, int qp_num)
> >>> +{
> >>> + struct dma_async_tx_descriptor *txd;
> >>> + struct dma_slave_config cfg;
> >>> + struct scatterlist sgl;
> >>> + dma_cookie_t cookie;
> >>> + struct device *dev;
> >>> +
> >>> + if (!edma || !edma->intr_chan)
> >>> + return -ENXIO;
> >>> +
> >>> + if (qp_num < 0 || qp_num >= edma_ctx.notify_qp_max)
> >>> + return -EINVAL;
> >>> +
> >>> + if (!edma_ctx.intr_rc_virt || !edma_ctx.intr_ep_phys)
> >>> + return -EINVAL;
> >>> +
> >>> + dev = edma->dev;
> >>> + if (!dev)
> >>> + return -ENODEV;
> >>> +
> >>> + WRITE_ONCE(edma_ctx.intr_rc_virt->db[qp_num], 1);
> >>> +
> >>> + /* Ensure store is visible before kicking the DMA transfer */
> >>> + wmb();
> >>> +
> >>> + sg_init_table(&sgl, 1);
> >>> + sg_dma_address(&sgl) = edma_ctx.intr_rc_phys + qp_num * sizeof(u32);
> >>> + sg_dma_len(&sgl) = sizeof(u32);
> >>> +
> >>> + memset(&cfg, 0, sizeof(cfg));
> >>> + cfg.dst_addr = edma_ctx.intr_ep_phys + qp_num * sizeof(u32);
> >>> + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> >>> + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> >>> + cfg.direction = DMA_MEM_TO_DEV;
> >>> +
> >>> + if (dmaengine_slave_config(edma->intr_chan, &cfg))
> >>> + return -EINVAL;
> >>> +
> >>> + txd = dmaengine_prep_slave_sg(edma->intr_chan, &sgl, 1,
> >>> + DMA_MEM_TO_DEV,
> >>> + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> >>> + if (!txd)
> >>> + return -ENOSPC;
> >>> +
> >>> + cookie = dmaengine_submit(txd);
> >>> + if (dma_submit_error(cookie))
> >>> + return -ENOSPC;
> >>> +
> >>> + dma_async_issue_pending(edma->intr_chan);
> >>> + return 0;
> >>> +}
> >>> diff --git a/drivers/ntb/ntb_edma.h b/drivers/ntb/ntb_edma.h
> >>> new file mode 100644
> >>> index 000000000000..da0451827edb
> >>> --- /dev/null
> >>> +++ b/drivers/ntb/ntb_edma.h
> >>> @@ -0,0 +1,128 @@
> >>> +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
> >>> +#ifndef _NTB_EDMA_H_
> >>> +#define _NTB_EDMA_H_
> >>> +
> >>> +#include <linux/completion.h>
> >>> +#include <linux/device.h>
> >>> +#include <linux/interrupt.h>
> >>> +
> >>> +#define EDMA_REG_SIZE SZ_64K
> >>> +#define DMA_LLP_MEM_SIZE SZ_4K
> >>> +#define EDMA_WR_CH_NUM 4
> >>> +#define EDMA_RD_CH_NUM 4
> >>> +#define NTB_EDMA_MAX_CH 8
> >>> +
> >>> +#define NTB_EDMA_INFO_MAGIC 0x45444D41 /* "EDMA" */
> >>> +#define NTB_EDMA_INFO_OFF EDMA_REG_SIZE
> >>> +
> >>> +#define NTB_EDMA_RING_ORDER 7
> >>> +#define NTB_EDMA_RING_ENTRIES (1U << NTB_EDMA_RING_ORDER)
> >>> +#define NTB_EDMA_RING_MASK (NTB_EDMA_RING_ENTRIES - 1)
> >>> +
> >>> +typedef void (*ntb_edma_interrupt_cb_t)(void *data, int qp_num);
> >>> +
> >>> +/*
> >>> + * REMOTE_EDMA_EP:
> >>> + * Endpoint owns the eDMA engine and pushes descriptors into a shared MW.
> >>> + *
> >>> + * REMOTE_EDMA_RC:
> >>> + * Root Complex controls the endpoint eDMA through the shared MW and
> >>> + * drives reads/writes on behalf of the host.
> >>> + */
> >>> +typedef enum {
> >>> + REMOTE_EDMA_UNKNOWN,
> >>> + REMOTE_EDMA_EP,
> >>> + REMOTE_EDMA_RC,
> >>> +} remote_edma_mode_t;
> >>> +
> >>> +typedef enum {
> >>> + REMOTE_EDMA_WRITE,
> >>> + REMOTE_EDMA_READ,
> >>> +} remote_edma_dir_t;
> >>> +
> >>> +/*
> >>> + * Layout of remote eDMA MW (EP local address space, RC sees via peer MW):
> >>> + *
> >>> + * 0 .. EDMA_REG_SIZE-1 : DesignWare eDMA registers
> >>> + * EDMA_REG_SIZE .. +PAGE_SIZE : struct ntb_edma_info (EP writes, RC reads)
> >>> + * +PAGE_SIZE .. : LL ring buffers (EP allocates phys addresses,
> >>> + * RC configures via dw_edma)
> >>> + *
> >>> + * ntb_edma_setup_mws() on EP:
> >>> + * - allocates ntb_edma_info and LLs in EP memory
> >>> + * - programs inbound iATU so that RC peer MW[n] points at this block
> >>> + *
> >>> + * ntb_edma_setup_peer() on RC:
> >>> + * - ioremaps peer MW[n]
> >>> + * - reads ntb_edma_info
> >>> + * - sets up dw_edma_chip ll_region_* from that info
> >>> + */
> >>> +struct ntb_edma_info {
> >>> + u32 magic;
> >>> + u16 wr_cnt;
> >>> + u16 rd_cnt;
> >>> + u64 regs_phys;
> >>> + u32 ll_stride;
> >>> + u32 rsvd;
> >>> + u64 ll_wr_phys[NTB_EDMA_MAX_CH];
> >>> + u64 ll_rd_phys[NTB_EDMA_MAX_CH];
> >>> +
> >>> + u64 intr_dar_base;
> >>> +} __packed;
> >>> +
> >>> +struct ll_dma_addrs {
> >>> + dma_addr_t wr[EDMA_WR_CH_NUM];
> >>> + dma_addr_t rd[EDMA_RD_CH_NUM];
> >>> +};
> >>> +
> >>> +struct ntb_edma_chans {
> >>> + struct device *dev;
> >>> +
> >>> + struct dma_chan *wr_chan[EDMA_WR_CH_NUM];
> >>> + struct dma_chan *rd_chan[EDMA_RD_CH_NUM];
> >>> + struct dma_chan *intr_chan;
> >>> +
> >>> + unsigned int num_wr_chan;
> >>> + unsigned int num_rd_chan;
> >>> + atomic_t cur_wr_chan;
> >>> + atomic_t cur_rd_chan;
> >>> +};
> >>> +
> >>> +static __always_inline u32 ntb_edma_ring_idx(u32 v)
> >>> +{
> >>> + return v & NTB_EDMA_RING_MASK;
> >>> +}
> >>> +
> >>> +static __always_inline u32 ntb_edma_ring_used_entry(u32 head, u32 tail)
> >>> +{
> >>> + if (head >= tail) {
> >>> + WARN_ON_ONCE((head - tail) > (NTB_EDMA_RING_ENTRIES - 1));
> >>> + return head - tail;
> >>> + }
> >>> +
> >>> + WARN_ON_ONCE((U32_MAX - tail + head + 1) > (NTB_EDMA_RING_ENTRIES - 1));
> >>> + return U32_MAX - tail + head + 1;
> >>> +}
> >>> +
> >>> +static __always_inline u32 ntb_edma_ring_free_entry(u32 head, u32 tail)
> >>> +{
> >>> + return NTB_EDMA_RING_ENTRIES - ntb_edma_ring_used_entry(head, tail) - 1;
> >>> +}
> >>> +
> >>> +static __always_inline bool ntb_edma_ring_full(u32 head, u32 tail)
> >>> +{
> >>> + return ntb_edma_ring_free_entry(head, tail) == 0;
> >>> +}
> >>> +
> >>> +int ntb_edma_setup_isr(struct device *dev, struct device *epc_dev,
> >>> + ntb_edma_interrupt_cb_t cb, void *data);
> >>> +void ntb_edma_teardown_isr(struct device *dev);
> >>> +int ntb_edma_setup_mws(struct ntb_dev *ndev);
> >>> +int ntb_edma_setup_peer(struct ntb_dev *ndev);
> >>> +int ntb_edma_setup_chans(struct device *dma_dev, struct ntb_edma_chans *edma);
> >>> +struct dma_chan *ntb_edma_pick_chan(struct ntb_edma_chans *edma,
> >>> + remote_edma_dir_t dir);
> >>> +void ntb_edma_teardown_chans(struct ntb_edma_chans *edma);
> >>> +int ntb_edma_notify_peer(struct ntb_edma_chans *edma, int qp_num);
> >>> +
> >>> +#endif
> >>> diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport_core.c
> >>> similarity index 65%
> >>> rename from drivers/ntb/ntb_transport.c
> >>> rename to drivers/ntb/ntb_transport_core.c
> >>> index 907db6c93d4d..48d48921978d 100644
> >>> --- a/drivers/ntb/ntb_transport.c
> >>> +++ b/drivers/ntb/ntb_transport_core.c
> >>> @@ -47,6 +47,9 @@
> >>> * Contact Information:
> >>> * Jon Mason <jon.mason@intel.com>
> >>> */
> >>> +#include <linux/atomic.h>
> >>> +#include <linux/bug.h>
> >>> +#include <linux/compiler.h>
> >>> #include <linux/debugfs.h>
> >>> #include <linux/delay.h>
> >>> #include <linux/dmaengine.h>
> >>> @@ -71,6 +74,8 @@
> >>> #define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB"
> >>> #define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2)
> >>>
> >>> +#define NTB_EDMA_MAX_POLL 32
> >>> +
> >>> MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
> >>> MODULE_VERSION(NTB_TRANSPORT_VER);
> >>> MODULE_LICENSE("Dual BSD/GPL");
> >>> @@ -102,6 +107,13 @@ module_param(use_msi, bool, 0644);
> >>> MODULE_PARM_DESC(use_msi, "Use MSI interrupts instead of doorbells");
> >>> #endif
> >>>
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>
> >> This is a comment throughout this patch. Doing ifdefs inside C source is pretty frowed upon in the kernel. The preferred way is to only have ifdefs in the header files. So please give this a bit more consideration and see if it can be done differently to address this.
> >
> > I agree, there is no good reason to keep those remaining ifdefs at all.
> > I'll clean it up. Thanks for pointing this out.
> >
> >>
> >>> +#include "ntb_edma.h"
> >>> +static bool use_remote_edma;
> >>> +module_param(use_remote_edma, bool, 0644);
> >>> +MODULE_PARM_DESC(use_remote_edma, "Use remote eDMA mode (when enabled, use_msi is ignored)");
> >>> +#endif
> >>> +
> >>> static struct dentry *nt_debugfs_dir;
> >>>
> >>> /* Only two-ports NTB devices are supported */
> >>> @@ -125,6 +137,14 @@ struct ntb_queue_entry {
> >>> struct ntb_payload_header __iomem *tx_hdr;
> >>> struct ntb_payload_header *rx_hdr;
> >>> };
> >>> +
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>> + dma_addr_t addr;
> >>> +
> >>> + /* Used by RC side only */
> >>> + struct scatterlist sgl;
> >>> + struct work_struct dma_work;
> >>> +#endif
> >>> };
> >>>
> >>> struct ntb_rx_info {
> >>> @@ -202,6 +222,33 @@ struct ntb_transport_qp {
> >>> int msi_irq;
> >>> struct ntb_msi_desc msi_desc;
> >>> struct ntb_msi_desc peer_msi_desc;
> >>> +
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>> + /*
> >>> + * For ensuring peer notification in non-atomic context.
> >>> + * ntb_peer_db_set might sleep or schedule.
> >>> + */
> >>> + struct work_struct db_work;
> >>> +
> >>> + /*
> >>> + * wr: remote eDMA write transfer (EP -> RC direction)
> >>> + * rd: remote eDMA read transfer (RC -> EP direction)
> >>> + */
> >>> + u32 wr_cons;
> >>> + u32 rd_cons;
> >>> + u32 wr_prod;
> >>> + u32 rd_prod;
> >>> + u32 wr_issue;
> >>> + u32 rd_issue;
> >>> +
> >>> + spinlock_t ep_tx_lock;
> >>> + spinlock_t ep_rx_lock;
> >>> + spinlock_t rc_lock;
> >>> +
> >>> + /* Completion work for read/write transfers. */
> >>> + struct work_struct read_work;
> >>> + struct work_struct write_work;
> >>> +#endif
> >>
> >> For something like this, maybe it needs its own struct instead of an ifdef chunk. Perhaps 'ntb_rx_info' can serve as a core data struct with EDMA having 'ntb_rx_info_edma' and embed 'ntb_rx_info'.
> >
> > Thanks again for the suggestion. I'll reorganize things.
> >
> > Koichiro
> >
> >>
> >> DJ
> >>
> >>> };
> >>>
> >>> struct ntb_transport_mw {
> >>> @@ -249,6 +296,13 @@ struct ntb_transport_ctx {
> >>>
> >>> /* Make sure workq of link event be executed serially */
> >>> struct mutex link_event_lock;
> >>> +
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>> + remote_edma_mode_t remote_edma_mode;
> >>> + struct device *dma_dev;
> >>> + struct workqueue_struct *wq;
> >>> + struct ntb_edma_chans edma;
> >>> +#endif
> >>> };
> >>>
> >>> enum {
> >>> @@ -262,6 +316,19 @@ struct ntb_payload_header {
> >>> unsigned int flags;
> >>> };
> >>>
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>> +static void ntb_transport_edma_uninit(struct ntb_transport_ctx *nt);
> >>> +static int ntb_transport_edma_init(struct ntb_transport_ctx *nt,
> >>> + unsigned int *mw_count);
> >>> +static void ntb_transport_edma_init_queue(struct ntb_transport_ctx *nt,
> >>> + unsigned int qp_num);
> >>> +static void ntb_transport_edma_create_queue(struct ntb_transport_ctx *nt,
> >>> + struct ntb_transport_qp *qp);
> >>> +static int ntb_transport_edma_rc_init(struct ntb_transport_ctx *nt);
> >>> +static int ntb_transport_edma_ep_init(struct ntb_transport_ctx *nt);
> >>> +static void ntb_transport_edma_rc_dma_work(struct work_struct *work);
> >>> +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> >>> +
> >>> /*
> >>> * Return the device that should be used for DMA mapping.
> >>> *
> >>> @@ -298,7 +365,7 @@ enum {
> >>> container_of((__drv), struct ntb_transport_client, driver)
> >>>
> >>> #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count)
> >>> -#define NTB_QP_DEF_NUM_ENTRIES 100
> >>> +#define NTB_QP_DEF_NUM_ENTRIES 128
> >>> #define NTB_LINK_DOWN_TIMEOUT 10
> >>>
> >>> static void ntb_transport_rxc_db(unsigned long data);
> >>> @@ -1015,6 +1082,10 @@ static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
> >>> count = ntb_spad_count(nt->ndev);
> >>> for (i = 0; i < count; i++)
> >>> ntb_spad_write(nt->ndev, i, 0);
> >>> +
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>> + ntb_edma_teardown_chans(&nt->edma);
> >>> +#endif
> >>> }
> >>>
> >>> static void ntb_transport_link_cleanup_work(struct work_struct *work)
> >>> @@ -1051,6 +1122,14 @@ static void ntb_transport_link_work(struct work_struct *work)
> >>>
> >>> /* send the local info, in the opposite order of the way we read it */
> >>>
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>> + rc = ntb_transport_edma_ep_init(nt);
> >>> + if (rc) {
> >>> + dev_err(&pdev->dev, "Failed to init EP: %d\n", rc);
> >>> + return;
> >>> + }
> >>> +#endif
> >>> +
> >>> if (nt->use_msi) {
> >>> rc = ntb_msi_setup_mws(ndev);
> >>> if (rc) {
> >>> @@ -1132,6 +1211,14 @@ static void ntb_transport_link_work(struct work_struct *work)
> >>>
> >>> nt->link_is_up = true;
> >>>
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>> + rc = ntb_transport_edma_rc_init(nt);
> >>> + if (rc) {
> >>> + dev_err(&pdev->dev, "Failed to init RC: %d\n", rc);
> >>> + goto out1;
> >>> + }
> >>> +#endif
> >>> +
> >>> for (i = 0; i < nt->qp_count; i++) {
> >>> struct ntb_transport_qp *qp = &nt->qp_vec[i];
> >>>
> >>> @@ -1277,6 +1364,8 @@ static const struct ntb_transport_backend_ops default_backend_ops = {
> >>> .debugfs_stats_show = ntb_transport_default_debugfs_stats_show,
> >>> };
> >>>
> >>> +static const struct ntb_transport_backend_ops edma_backend_ops;
> >>> +
> >>> static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> >>> {
> >>> struct ntb_transport_ctx *nt;
> >>> @@ -1311,7 +1400,23 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> >>>
> >>> nt->ndev = ndev;
> >>>
> >>> - nt->backend_ops = default_backend_ops;
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>> + if (use_remote_edma) {
> >>> + rc = ntb_transport_edma_init(nt, &mw_count);
> >>> + if (rc) {
> >>> + nt->mw_count = 0;
> >>> + goto err;
> >>> + }
> >>> + nt->backend_ops = edma_backend_ops;
> >>> +
> >>> + /*
> >>> + * On remote eDMA mode, we reserve a read channel for Host->EP
> >>> + * interruption.
> >>> + */
> >>> + use_msi = false;
> >>> + } else
> >>> +#endif
> >>> + nt->backend_ops = default_backend_ops;
> >>>
> >>> /*
> >>> * If we are using MSI, and have at least one extra memory window,
> >>> @@ -1402,6 +1507,10 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> >>> rc = ntb_transport_init_queue(nt, i);
> >>> if (rc)
> >>> goto err2;
> >>> +
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>> + ntb_transport_edma_init_queue(nt, i);
> >>> +#endif
> >>> }
> >>>
> >>> INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);
> >>> @@ -1433,6 +1542,9 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
> >>> }
> >>> kfree(nt->mw_vec);
> >>> err:
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>> + ntb_transport_edma_uninit(nt);
> >>> +#endif
> >>> kfree(nt);
> >>> return rc;
> >>> }
> >>> @@ -2055,11 +2167,16 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
> >>>
> >>> nt->qp_bitmap_free &= ~qp_bit;
> >>>
> >>> + qp->qp_bit = qp_bit;
> >>> qp->cb_data = data;
> >>> qp->rx_handler = handlers->rx_handler;
> >>> qp->tx_handler = handlers->tx_handler;
> >>> qp->event_handler = handlers->event_handler;
> >>>
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>> + ntb_transport_edma_create_queue(nt, qp);
> >>> +#endif
> >>> +
> >>> dma_cap_zero(dma_mask);
> >>> dma_cap_set(DMA_MEMCPY, dma_mask);
> >>>
> >>> @@ -2105,6 +2222,9 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
> >>> goto err1;
> >>>
> >>> entry->qp = qp;
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>> + INIT_WORK(&entry->dma_work, ntb_transport_edma_rc_dma_work);
> >>> +#endif
> >>> ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> >>> &qp->rx_free_q);
> >>> }
> >>> @@ -2156,8 +2276,8 @@ EXPORT_SYMBOL_GPL(ntb_transport_create_queue);
> >>> */
> >>> void ntb_transport_free_queue(struct ntb_transport_qp *qp)
> >>> {
> >>> - struct pci_dev *pdev;
> >>> struct ntb_queue_entry *entry;
> >>> + struct pci_dev *pdev;
> >>> u64 qp_bit;
> >>>
> >>> if (!qp)
> >>> @@ -2208,6 +2328,10 @@ void ntb_transport_free_queue(struct ntb_transport_qp *qp)
> >>> tasklet_kill(&qp->rxc_db_work);
> >>>
> >>> cancel_delayed_work_sync(&qp->link_work);
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>> + cancel_work_sync(&qp->read_work);
> >>> + cancel_work_sync(&qp->write_work);
> >>> +#endif
> >>>
> >>> qp->cb_data = NULL;
> >>> qp->rx_handler = NULL;
> >>> @@ -2346,6 +2470,1157 @@ int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
> >>> }
> >>> EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue);
> >>>
> >>> +#ifdef CONFIG_NTB_TRANSPORT_EDMA
> >>> +/*
> >>> + * Remote eDMA mode implementation
> >>> + */
> >>> +struct ntb_edma_desc {
> >>> + u32 len;
> >>> + u32 flags;
> >>> + u64 addr; /* DMA address */
> >>> + u64 data;
> >>> +};
> >>> +
> >>> +struct ntb_edma_ring {
> >>> + struct ntb_edma_desc desc[NTB_EDMA_RING_ENTRIES];
> >>> + u32 head;
> >>> + u32 tail;
> >>> +};
> >>> +
> >>> +#define NTB_EDMA_DESC_OFF(i) ((size_t)(i) * sizeof(struct ntb_edma_desc))
> >>> +
> >>> +#define __NTB_EDMA_CHECK_INDEX(_i) \
> >>> +({ \
> >>> + unsigned long __i = (unsigned long)(_i); \
> >>> + WARN_ONCE(__i >= (unsigned long)NTB_EDMA_RING_ENTRIES, \
> >>> + "ntb_edma: index i=%lu >= ring_entries=%lu\n", \
> >>> + __i, (unsigned long)NTB_EDMA_RING_ENTRIES); \
> >>> + __i; \
> >>> +})
> >>> +
> >>> +#define NTB_EDMA_DESC_I(qp, i, n) \
> >>> +({ \
> >>> + typeof(qp) __qp = (qp); \
> >>> + unsigned long __i = __NTB_EDMA_CHECK_INDEX(i); \
> >>> + (struct ntb_edma_desc *) \
> >>> + ((char *)(__qp)->rx_buff + \
> >>> + (sizeof(struct ntb_edma_ring) * n) + \
> >>> + NTB_EDMA_DESC_OFF(__i)); \
> >>> +})
> >>> +
> >>> +#define NTB_EDMA_DESC_O(qp, i, n) \
> >>> +({ \
> >>> + typeof(qp) __qp = (qp); \
> >>> + unsigned long __i = __NTB_EDMA_CHECK_INDEX(i); \
> >>> + (struct ntb_edma_desc __iomem *) \
> >>> + ((char __iomem *)(__qp)->tx_mw + \
> >>> + (sizeof(struct ntb_edma_ring) * n) + \
> >>> + NTB_EDMA_DESC_OFF(__i)); \
> >>> +})
> >>> +
> >>> +#define NTB_EDMA_HEAD_I(qp, n) ((u32 *)((char *)qp->rx_buff + \
> >>> + (sizeof(struct ntb_edma_ring) * n) + \
> >>> + offsetof(struct ntb_edma_ring, head)))
> >>> +#define NTB_EDMA_HEAD_O(qp, n) ((u32 *)((char __iomem *)qp->tx_mw + \
> >>> + (sizeof(struct ntb_edma_ring) * n) + \
> >>> + offsetof(struct ntb_edma_ring, head)))
> >>> +#define NTB_EDMA_TAIL_I(qp, n) ((u32 *)((char *)qp->rx_buff + \
> >>> + (sizeof(struct ntb_edma_ring) * n) + \
> >>> + offsetof(struct ntb_edma_ring, tail)))
> >>> +#define NTB_EDMA_TAIL_O(qp, n) ((u32 *)((char __iomem *)qp->tx_mw + \
> >>> + (sizeof(struct ntb_edma_ring) * n) + \
> >>> + offsetof(struct ntb_edma_ring, tail)))
> >>> +
> >>> +/*
> >>> + * Macro naming rule:
> >>> + * NTB_DESC_RD_EP_I (as an example)
> >>> + * ^^ ^^ ^
> >>> + * : : `-- I(n) or O(ut). In = Read, Out = Write.
> >>> + * : `----- Who uses this macro.
> >>> + * `-------- DESC / HEAD / TAIL
> >>> + *
> >>> + * Read transfers (RC->EP):
> >>> + *
> >>> + * EP view (outbound, written via NTB):
> >>> + * - descs: NTB_DESC_RD_EP_O(qp, i) / NTB_DESC_RD_EP_I(qp, i)
> >>> + * [ len ][ flags ][ addr ][ data ]
> >>> + * [ len ][ flags ][ addr ][ data ]
> >>> + * :
> >>> + * [ len ][ flags ][ addr ][ data ]
> >>> + * - head: NTB_HEAD_RD_EP_O(qp)
> >>> + * - tail: NTB_TAIL_RD_EP_I(qp)
> >>> + *
> >>> + * RC view (inbound, local mapping):
> >>> + * - descs: NTB_DESC_RD_RC_I(qp, i) / NTB_DESC_RD_RC_O(qp, i)
> >>> + * [ len ][ flags ][ addr ][ data ]
> >>> + * [ len ][ flags ][ addr ][ data ]
> >>> + * :
> >>> + * [ len ][ flags ][ addr ][ data ]
> >>> + * - head: NTB_HEAD_RD_RC_I(qp)
> >>> + * - tail: NTB_TAIL_RD_RC_O(qp)
> >>> + *
> >>> + * Write transfers (EP -> RC) are analogous but use
> >>> + * NTB_DESC_WR_{EP_O,RC_I}(), NTB_HEAD_WR_{EP_O,RC_I}(),
> >>> + * and NTB_TAIL_WR_{EP_I,RC_O}().
> >>> + */
> >>> +#define NTB_DESC_RD_EP_I(qp, i) NTB_EDMA_DESC_I(qp, i, 0)
> >>> +#define NTB_DESC_RD_EP_O(qp, i) NTB_EDMA_DESC_O(qp, i, 0)
> >>> +#define NTB_DESC_WR_EP_I(qp, i) NTB_EDMA_DESC_I(qp, i, 1)
> >>> +#define NTB_DESC_WR_EP_O(qp, i) NTB_EDMA_DESC_O(qp, i, 1)
> >>> +#define NTB_DESC_RD_RC_I(qp, i) NTB_EDMA_DESC_I(qp, i, 0)
> >>> +#define NTB_DESC_RD_RC_O(qp, i) NTB_EDMA_DESC_O(qp, i, 0)
> >>> +#define NTB_DESC_WR_RC_I(qp, i) NTB_EDMA_DESC_I(qp, i, 1)
> >>> +#define NTB_DESC_WR_RC_O(qp, i) NTB_EDMA_DESC_O(qp, i, 1)
> >>> +
> >>> +#define NTB_HEAD_RD_EP_O(qp) NTB_EDMA_HEAD_O(qp, 0)
> >>> +#define NTB_HEAD_WR_EP_O(qp) NTB_EDMA_HEAD_O(qp, 1)
> >>> +#define NTB_HEAD_RD_RC_I(qp) NTB_EDMA_HEAD_I(qp, 0)
> >>> +#define NTB_HEAD_WR_RC_I(qp) NTB_EDMA_HEAD_I(qp, 1)
> >>> +
> >>> +#define NTB_TAIL_RD_EP_I(qp) NTB_EDMA_TAIL_I(qp, 0)
> >>> +#define NTB_TAIL_WR_EP_I(qp) NTB_EDMA_TAIL_I(qp, 1)
> >>> +#define NTB_TAIL_RD_RC_O(qp) NTB_EDMA_TAIL_O(qp, 0)
> >>> +#define NTB_TAIL_WR_RC_O(qp) NTB_EDMA_TAIL_O(qp, 1)
> >>> +
> >>> +static inline bool ntb_qp_edma_is_rc(struct ntb_transport_qp *qp)
> >>> +{
> >>> + return qp->transport->remote_edma_mode == REMOTE_EDMA_RC;
> >>> +}
> >>> +
> >>> +static inline bool ntb_qp_edma_is_ep(struct ntb_transport_qp *qp)
> >>> +{
> >>> + return qp->transport->remote_edma_mode == REMOTE_EDMA_EP;
> >>> +}
> >>> +
> >>> +static inline bool ntb_qp_edma_enabled(struct ntb_transport_qp *qp)
> >>> +{
> >>> + return ntb_qp_edma_is_rc(qp) || ntb_qp_edma_is_ep(qp);
> >>> +}
> >>> +
> >>> +static unsigned int ntb_transport_edma_tx_free_entry(struct ntb_transport_qp *qp)
> >>> +{
> >>> + unsigned int head, tail;
> >>> +
> >>> + if (ntb_qp_edma_is_ep(qp)) {
> >>> + scoped_guard(spinlock_irqsave, &qp->ep_tx_lock) {
> >>> + /* In this scope, only 'head' might proceed */
> >>> + tail = READ_ONCE(qp->wr_cons);
> >>> + head = READ_ONCE(qp->wr_prod);
> >>> + }
> >>> + return ntb_edma_ring_free_entry(head, tail);
> >>> + }
> >>> +
> >>> + scoped_guard(spinlock_irqsave, &qp->rc_lock) {
> >>> + /* In this scope, only 'head' might proceed */
> >>> + tail = READ_ONCE(qp->rd_issue);
> >>> + head = READ_ONCE(*NTB_HEAD_RD_RC_I(qp));
> >>> + }
> >>> + /*
> >>> + * On RC side, 'used' amount indicates how much EP side
> >>> + * has refilled, which are available for us to use for TX.
> >>> + */
> >>> + return ntb_edma_ring_used_entry(head, tail);
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_debugfs_stats_show(struct seq_file *s,
> >>> + struct ntb_transport_qp *qp)
> >>> +{
> >>> + seq_printf(s, "rx_bytes - \t%llu\n", qp->rx_bytes);
> >>> + seq_printf(s, "rx_pkts - \t%llu\n", qp->rx_pkts);
> >>> + seq_printf(s, "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
> >>> + seq_printf(s, "rx_buff - \t0x%p\n", qp->rx_buff);
> >>> + seq_printf(s, "rx_max_entry - \t%u\n", qp->rx_max_entry);
> >>> + seq_printf(s, "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry);
> >>> +
> >>> + seq_printf(s, "tx_bytes - \t%llu\n", qp->tx_bytes);
> >>> + seq_printf(s, "tx_pkts - \t%llu\n", qp->tx_pkts);
> >>> + seq_printf(s, "tx_ring_full - \t%llu\n", qp->tx_ring_full);
> >>> + seq_printf(s, "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
> >>> + seq_printf(s, "tx_mw - \t0x%p\n", qp->tx_mw);
> >>> + seq_printf(s, "tx_max_entry - \t%u\n", qp->tx_max_entry);
> >>> + seq_printf(s, "free tx - \t%u\n", ntb_transport_tx_free_entry(qp));
> >>> + seq_putc(s, '\n');
> >>> +
> >>> + seq_puts(s, "Using Remote eDMA - Yes\n");
> >>> + seq_printf(s, "QP Link - \t%s\n", qp->link_is_up ? "Up" : "Down");
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_uninit(struct ntb_transport_ctx *nt)
> >>> +{
> >>> + struct ntb_dev *ndev = nt->ndev;
> >>> +
> >>> + if (nt->remote_edma_mode == REMOTE_EDMA_EP && ndev && ndev->pdev)
> >>> + ntb_edma_teardown_isr(&ndev->pdev->dev);
> >>> +
> >>> + if (nt->wq)
> >>> + destroy_workqueue(nt->wq);
> >>> + nt->wq = NULL;
> >>> +}
> >>> +
> >>> +static int ntb_transport_edma_init(struct ntb_transport_ctx *nt,
> >>> + unsigned int *mw_count)
> >>> +{
> >>> + struct ntb_dev *ndev = nt->ndev;
> >>> +
> >>> + /*
> >>> + * We need at least one MW for the transport plus one MW reserved
> >>> + * for the remote eDMA window (see ntb_edma_setup_mws/peer).
> >>> + */
> >>> + if (*mw_count <= 1) {
> >>> + dev_err(&ndev->dev,
> >>> + "remote eDMA requires at least two MWS (have %u)\n",
> >>> + *mw_count);
> >>> + return -ENODEV;
> >>> + }
> >>> +
> >>> + nt->wq = alloc_workqueue("ntb-edma-wq", WQ_UNBOUND | WQ_SYSFS, 0);
> >>> + if (!nt->wq) {
> >>> + ntb_transport_edma_uninit(nt);
> >>> + return -ENOMEM;
> >>> + }
> >>> +
> >>> + /* Reserve the last peer MW exclusively for the eDMA window. */
> >>> + *mw_count -= 1;
> >>> +
> >>> + return 0;
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_db_work(struct work_struct *work)
> >>> +{
> >>> + struct ntb_transport_qp *qp =
> >>> + container_of(work, struct ntb_transport_qp, db_work);
> >>> +
> >>> + ntb_peer_db_set(qp->ndev, qp->qp_bit);
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_notify_peer(struct ntb_transport_qp *qp)
> >>> +{
> >>> + if (ntb_qp_edma_is_rc(qp))
> >>> + if (!ntb_edma_notify_peer(&qp->transport->edma, qp->qp_num))
> >>> + return;
> >>> +
> >>> + /*
> >>> + * Called from contexts that may be atomic. Since ntb_peer_db_set()
> >>> + * may sleep, delegate the actual doorbell write to a workqueue.
> >>> + */
> >>> + queue_work(system_highpri_wq, &qp->db_work);
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_isr(void *data, int qp_num)
> >>> +{
> >>> + struct ntb_transport_ctx *nt = data;
> >>> + struct ntb_transport_qp *qp;
> >>> +
> >>> + if (qp_num < 0 || qp_num >= nt->qp_count)
> >>> + return;
> >>> +
> >>> + qp = &nt->qp_vec[qp_num];
> >>> + if (WARN_ON(!qp))
> >>> + return;
> >>> +
> >>> + queue_work(nt->wq, &qp->read_work);
> >>> + queue_work(nt->wq, &qp->write_work);
> >>> +}
> >>> +
> >>> +static int ntb_transport_edma_rc_init(struct ntb_transport_ctx *nt)
> >>> +{
> >>> + struct ntb_dev *ndev = nt->ndev;
> >>> + struct pci_dev *pdev = ndev->pdev;
> >>> + int rc;
> >>> +
> >>> + if (!use_remote_edma || nt->remote_edma_mode != REMOTE_EDMA_UNKNOWN)
> >>> + return 0;
> >>> +
> >>> + rc = ntb_edma_setup_peer(ndev);
> >>> + if (rc) {
> >>> + dev_err(&pdev->dev, "Failed to enable remote eDMA: %d\n", rc);
> >>> + return rc;
> >>> + }
> >>> +
> >>> + rc = ntb_edma_setup_chans(get_dma_dev(ndev), &nt->edma);
> >>> + if (rc) {
> >>> + dev_err(&pdev->dev, "Failed to setup eDMA channels: %d\n", rc);
> >>> + return rc;
> >>> + }
> >>> +
> >>> + nt->remote_edma_mode = REMOTE_EDMA_RC;
> >>> + return 0;
> >>> +}
> >>> +
> >>> +static int ntb_transport_edma_ep_init(struct ntb_transport_ctx *nt)
> >>> +{
> >>> + struct ntb_dev *ndev = nt->ndev;
> >>> + struct pci_dev *pdev = ndev->pdev;
> >>> + struct pci_epc *epc;
> >>> + int rc;
> >>> +
> >>> + if (!use_remote_edma || nt->remote_edma_mode == REMOTE_EDMA_EP)
> >>> + return 0;
> >>> +
> >>> + /* Only EP side can return pci_epc */
> >>> + epc = ntb_get_pci_epc(ndev);
> >>> + if (!epc)
> >>> + return 0;
> >>> +
> >>> + rc = ntb_edma_setup_mws(ndev);
> >>> + if (rc) {
> >>> + dev_err(&pdev->dev,
> >>> + "Failed to set up memory window for eDMA: %d\n", rc);
> >>> + return rc;
> >>> + }
> >>> +
> >>> + rc = ntb_edma_setup_isr(&pdev->dev, &epc->dev, ntb_transport_edma_isr, nt);
> >>> + if (rc) {
> >>> + dev_err(&pdev->dev, "Failed to setup eDMA ISR (%d)\n", rc);
> >>> + return rc;
> >>> + }
> >>> +
> >>> + nt->remote_edma_mode = REMOTE_EDMA_EP;
> >>> + return 0;
> >>> +}
> >>> +
> >>> +static int ntb_transport_edma_setup_qp_mw(struct ntb_transport_ctx *nt,
> >>> + unsigned int qp_num)
> >>> +{
> >>> + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
> >>> + struct ntb_dev *ndev = nt->ndev;
> >>> + struct ntb_queue_entry *entry;
> >>> + struct ntb_transport_mw *mw;
> >>> + unsigned int mw_num, mw_count, qp_count;
> >>> + unsigned int qp_offset, rx_info_offset;
> >>> + unsigned int mw_size, mw_size_per_qp;
> >>> + unsigned int num_qps_mw;
> >>> + size_t edma_total;
> >>> + unsigned int i;
> >>> + int node;
> >>> +
> >>> + mw_count = nt->mw_count;
> >>> + qp_count = nt->qp_count;
> >>> +
> >>> + mw_num = QP_TO_MW(nt, qp_num);
> >>> + mw = &nt->mw_vec[mw_num];
> >>> +
> >>> + if (!mw->virt_addr)
> >>> + return -ENOMEM;
> >>> +
> >>> + if (mw_num < qp_count % mw_count)
> >>> + num_qps_mw = qp_count / mw_count + 1;
> >>> + else
> >>> + num_qps_mw = qp_count / mw_count;
> >>> +
> >>> + mw_size = min(nt->mw_vec[mw_num].phys_size, mw->xlat_size);
> >>> + if (max_mw_size && mw_size > max_mw_size)
> >>> + mw_size = max_mw_size;
> >>> +
> >>> + mw_size_per_qp = round_down((unsigned int)mw_size / num_qps_mw, SZ_64);
> >>> + qp_offset = mw_size_per_qp * (qp_num / mw_count);
> >>> + rx_info_offset = mw_size_per_qp - sizeof(struct ntb_rx_info);
> >>> +
> >>> + qp->tx_mw_size = mw_size_per_qp;
> >>> + qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;
> >>> + if (!qp->tx_mw)
> >>> + return -EINVAL;
> >>> + qp->tx_mw_phys = nt->mw_vec[mw_num].phys_addr + qp_offset;
> >>> + if (!qp->tx_mw_phys)
> >>> + return -EINVAL;
> >>> + qp->rx_info = qp->tx_mw + rx_info_offset;
> >>> + qp->rx_buff = mw->virt_addr + qp_offset;
> >>> + qp->remote_rx_info = qp->rx_buff + rx_info_offset;
> >>> +
> >>> + /* Due to housekeeping, there must be at least 2 buffs */
> >>> + qp->tx_max_frame = min(transport_mtu, mw_size_per_qp / 2);
> >>> + qp->rx_max_frame = min(transport_mtu, mw_size_per_qp / 2);
> >>> +
> >>> + /* In eDMA mode, decouple from MW sizing and force ring-sized entries */
> >>> + edma_total = 2 * sizeof(struct ntb_edma_ring);
> >>> + if (rx_info_offset < edma_total) {
> >>> + dev_err(&ndev->dev, "Ring space requires %luB (>=%uB)\n",
> >>> + edma_total, rx_info_offset);
> >>> + return -EINVAL;
> >>> + }
> >>> + qp->tx_max_entry = NTB_EDMA_RING_ENTRIES;
> >>> + qp->rx_max_entry = NTB_EDMA_RING_ENTRIES;
> >>> +
> >>> + /*
> >>> + * Checking to see if we have more entries than the default.
> >>> + * We should add additional entries if that is the case so we
> >>> + * can be in sync with the transport frames.
> >>> + */
> >>> + node = dev_to_node(&ndev->dev);
> >>> + for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) {
> >>> + entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
> >>> + if (!entry)
> >>> + return -ENOMEM;
> >>> +
> >>> + entry->qp = qp;
> >>> + INIT_WORK(&entry->dma_work, ntb_transport_edma_rc_dma_work);
> >>> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> >>> + &qp->rx_free_q);
> >>> + qp->rx_alloc_entry++;
> >>> + }
> >>> +
> >>> + memset(qp->rx_buff, 0, edma_total);
> >>> +
> >>> + qp->rx_pkts = 0;
> >>> + qp->tx_pkts = 0;
> >>> +
> >>> + return 0;
> >>> +}
> >>> +
> >>> +static int ntb_transport_edma_ep_read_complete(struct ntb_transport_qp *qp)
> >>> +{
> >>> + struct device *dma_dev = get_dma_dev(qp->ndev);
> >>> + struct ntb_queue_entry *entry;
> >>> + struct ntb_edma_desc *in;
> >>> + unsigned int len;
> >>> + u32 idx;
> >>> +
> >>> + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_TAIL_RD_EP_I(qp)),
> >>> + qp->rd_cons) == 0)
> >>> + return 0;
> >>> +
> >>> + idx = ntb_edma_ring_idx(qp->rd_cons);
> >>> + in = NTB_DESC_RD_EP_I(qp, idx);
> >>> + if (!(in->flags & DESC_DONE_FLAG))
> >>> + return 0;
> >>> +
> >>> + in->flags = 0;
> >>> + len = in->len; /* might be smaller than entry->len */
> >>> +
> >>> + entry = (struct ntb_queue_entry *)(in->data);
> >>> + if (WARN_ON(!entry))
> >>> + return 0;
> >>> +
> >>> + if (in->flags & LINK_DOWN_FLAG) {
> >>> + ntb_qp_link_down(qp);
> >>> + qp->rd_cons++;
> >>> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> >>> + return 1;
> >>> + }
> >>> +
> >>> + dma_unmap_single(dma_dev, entry->addr, entry->len, DMA_FROM_DEVICE);
> >>> +
> >>> + qp->rx_bytes += len;
> >>> + qp->rx_pkts++;
> >>> + qp->rd_cons++;
> >>> +
> >>> + if (qp->rx_handler && qp->client_ready)
> >>> + qp->rx_handler(qp, qp->cb_data, entry->cb_data, len);
> >>> +
> >>> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> >>> + return 1;
> >>> +}
> >>> +
> >>> +static int ntb_transport_edma_ep_write_complete(struct ntb_transport_qp *qp)
> >>> +{
> >>> + struct ntb_queue_entry *entry;
> >>> + struct ntb_edma_desc *in;
> >>> + u32 idx;
> >>> +
> >>> + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_TAIL_WR_EP_I(qp)),
> >>> + qp->wr_cons) == 0)
> >>> + return 0;
> >>> +
> >>> + idx = ntb_edma_ring_idx(qp->wr_cons);
> >>> + in = NTB_DESC_WR_EP_I(qp, idx);
> >>> +
> >>> + entry = (struct ntb_queue_entry *)(in->data);
> >>> + if (WARN_ON(!entry))
> >>> + return 0;
> >>> +
> >>> + qp->wr_cons++;
> >>> +
> >>> + if (qp->tx_handler)
> >>> + qp->tx_handler(qp, qp->cb_data, entry->cb_data, entry->len);
> >>> +
> >>> + ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q);
> >>> + return 1;
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_ep_read_work(struct work_struct *work)
> >>> +{
> >>> + struct ntb_transport_qp *qp = container_of(
> >>> + work, struct ntb_transport_qp, read_work);
> >>> + unsigned int i;
> >>> +
> >>> + for (i = 0; i < NTB_EDMA_MAX_POLL; i++) {
> >>> + if (!ntb_transport_edma_ep_read_complete(qp))
> >>> + break;
> >>> + }
> >>> +
> >>> + if (ntb_transport_edma_ep_read_complete(qp))
> >>> + queue_work(qp->transport->wq, &qp->read_work);
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_ep_write_work(struct work_struct *work)
> >>> +{
> >>> + struct ntb_transport_qp *qp = container_of(
> >>> + work, struct ntb_transport_qp, write_work);
> >>> + unsigned int i;
> >>> +
> >>> + for (i = 0; i < NTB_EDMA_MAX_POLL; i++) {
> >>> + if (!ntb_transport_edma_ep_write_complete(qp))
> >>> + break;
> >>> + }
> >>> +
> >>> + if (ntb_transport_edma_ep_write_complete(qp))
> >>> + queue_work(qp->transport->wq, &qp->write_work);
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_rc_write_complete_work(struct work_struct *work)
> >>> +{
> >>> + struct ntb_transport_qp *qp = container_of(
> >>> + work, struct ntb_transport_qp, write_work);
> >>> + struct ntb_queue_entry *entry;
> >>> + struct ntb_edma_desc *in;
> >>> + unsigned int len;
> >>> + void *cb_data;
> >>> + u32 idx;
> >>> +
> >>> + while (ntb_edma_ring_used_entry(READ_ONCE(qp->wr_issue),
> >>> + qp->wr_cons) != 0) {
> >>> + /* Paired with smp_wmb() in ntb_transport_edma_rc_poll() */
> >>> + smp_rmb();
> >>> +
> >>> + idx = ntb_edma_ring_idx(qp->wr_cons);
> >>> + in = NTB_DESC_WR_RC_I(qp, idx);
> >>> + entry = (struct ntb_queue_entry *)READ_ONCE(in->data);
> >>> + if (!entry || !(entry->flags & DESC_DONE_FLAG))
> >>> + break;
> >>> +
> >>> + in->data = 0;
> >>> +
> >>> + cb_data = entry->cb_data;
> >>> + len = entry->len;
> >>> +
> >>> + iowrite32(++qp->wr_cons, NTB_TAIL_WR_RC_O(qp));
> >>> +
> >>> + if (unlikely(entry->flags & LINK_DOWN_FLAG)) {
> >>> + ntb_qp_link_down(qp);
> >>> + continue;
> >>> + }
> >>> +
> >>> + ntb_transport_edma_notify_peer(qp);
> >>> +
> >>> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
> >>> +
> >>> + if (qp->rx_handler && qp->client_ready)
> >>> + qp->rx_handler(qp, qp->cb_data, cb_data, len);
> >>> +
> >>> + /* stat updates */
> >>> + qp->rx_bytes += len;
> >>> + qp->rx_pkts++;
> >>> + }
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_rc_write_cb(void *data,
> >>> + const struct dmaengine_result *res)
> >>> +{
> >>> + struct ntb_queue_entry *entry = data;
> >>> + struct ntb_transport_qp *qp = entry->qp;
> >>> + struct ntb_transport_ctx *nt = qp->transport;
> >>> + enum dmaengine_tx_result dma_err = res->result;
> >>> + struct device *dma_dev = get_dma_dev(qp->ndev);
> >>> +
> >>> + switch (dma_err) {
> >>> + case DMA_TRANS_READ_FAILED:
> >>> + case DMA_TRANS_WRITE_FAILED:
> >>> + case DMA_TRANS_ABORTED:
> >>> + entry->errors++;
> >>> + entry->len = -EIO;
> >>> + break;
> >>> + case DMA_TRANS_NOERROR:
> >>> + default:
> >>> + break;
> >>> + }
> >>> + dma_unmap_sg(dma_dev, &entry->sgl, 1, DMA_FROM_DEVICE);
> >>> + sg_dma_address(&entry->sgl) = 0;
> >>> +
> >>> + entry->flags |= DESC_DONE_FLAG;
> >>> +
> >>> + queue_work(nt->wq, &qp->write_work);
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_rc_read_complete_work(struct work_struct *work)
> >>> +{
> >>> + struct ntb_transport_qp *qp = container_of(
> >>> + work, struct ntb_transport_qp, read_work);
> >>> + struct ntb_edma_desc *in, __iomem *out;
> >>> + struct ntb_queue_entry *entry;
> >>> + unsigned int len;
> >>> + void *cb_data;
> >>> + u32 idx;
> >>> +
> >>> + while (ntb_edma_ring_used_entry(READ_ONCE(qp->rd_issue),
> >>> + qp->rd_cons) != 0) {
> >>> + /* Paired with smp_wmb() in ntb_transport_edma_rc_tx_enqueue() */
> >>> + smp_rmb();
> >>> +
> >>> + idx = ntb_edma_ring_idx(qp->rd_cons);
> >>> + in = NTB_DESC_RD_RC_I(qp, idx);
> >>> + entry = (struct ntb_queue_entry *)in->data;
> >>> + if (!entry || !(entry->flags & DESC_DONE_FLAG))
> >>> + break;
> >>> +
> >>> + in->data = 0;
> >>> +
> >>> + cb_data = entry->cb_data;
> >>> + len = entry->len;
> >>> +
> >>> + out = NTB_DESC_RD_RC_O(qp, idx);
> >>> +
> >>> + WRITE_ONCE(qp->rd_cons, qp->rd_cons + 1);
> >>> +
> >>> + /*
> >>> + * No need to add barrier in-between to enforce ordering here.
> >>> + * The other side proceeds only after both flags and tail are
> >>> + * updated.
> >>> + */
> >>> + iowrite32(entry->flags, &out->flags);
> >>> + iowrite32(qp->rd_cons, NTB_TAIL_RD_RC_O(qp));
> >>> +
> >>> + ntb_transport_edma_notify_peer(qp);
> >>> +
> >>> + ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
> >>> + &qp->tx_free_q);
> >>> +
> >>> + if (qp->tx_handler)
> >>> + qp->tx_handler(qp, qp->cb_data, cb_data, len);
> >>> +
> >>> + /* stat updates */
> >>> + qp->tx_bytes += len;
> >>> + qp->tx_pkts++;
> >>> + }
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_rc_read_cb(void *data,
> >>> + const struct dmaengine_result *res)
> >>> +{
> >>> + struct ntb_queue_entry *entry = data;
> >>> + struct ntb_transport_qp *qp = entry->qp;
> >>> + struct ntb_transport_ctx *nt = qp->transport;
> >>> + struct device *dma_dev = get_dma_dev(qp->ndev);
> >>> + enum dmaengine_tx_result dma_err = res->result;
> >>> +
> >>> + switch (dma_err) {
> >>> + case DMA_TRANS_READ_FAILED:
> >>> + case DMA_TRANS_WRITE_FAILED:
> >>> + case DMA_TRANS_ABORTED:
> >>> + entry->errors++;
> >>> + entry->len = -EIO;
> >>> + break;
> >>> + case DMA_TRANS_NOERROR:
> >>> + default:
> >>> + break;
> >>> + }
> >>> + dma_unmap_sg(dma_dev, &entry->sgl, 1, DMA_TO_DEVICE);
> >>> + sg_dma_address(&entry->sgl) = 0;
> >>> +
> >>> + entry->flags |= DESC_DONE_FLAG;
> >>> +
> >>> + queue_work(nt->wq, &qp->read_work);
> >>> +}
> >>> +
> >>> +static int ntb_transport_edma_rc_write_start(struct device *d,
> >>> + struct dma_chan *chan, size_t len,
> >>> + dma_addr_t ep_src, void *rc_dst,
> >>> + struct ntb_queue_entry *entry)
> >>> +{
> >>> + struct scatterlist *sgl = &entry->sgl;
> >>> + struct dma_async_tx_descriptor *txd;
> >>> + struct dma_slave_config cfg;
> >>> + dma_cookie_t cookie;
> >>> + int nents, rc;
> >>> +
> >>> + if (!d)
> >>> + return -ENODEV;
> >>> +
> >>> + if (!chan)
> >>> + return -ENXIO;
> >>> +
> >>> + if (WARN_ON(!ep_src || !rc_dst))
> >>> + return -EINVAL;
> >>> +
> >>> + if (WARN_ON(sg_dma_address(sgl)))
> >>> + return -EINVAL;
> >>> +
> >>> + sg_init_one(sgl, rc_dst, len);
> >>> + nents = dma_map_sg(d, sgl, 1, DMA_FROM_DEVICE);
> >>> + if (nents <= 0)
> >>> + return -EIO;
> >>> +
> >>> + memset(&cfg, 0, sizeof(cfg));
> >>> + cfg.src_addr = ep_src;
> >>> + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> >>> + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> >>> + cfg.direction = DMA_DEV_TO_MEM;
> >>> + rc = dmaengine_slave_config(chan, &cfg);
> >>> + if (rc)
> >>> + goto out_unmap;
> >>> +
> >>> + txd = dmaengine_prep_slave_sg(chan, sgl, 1, DMA_DEV_TO_MEM,
> >>> + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> >>> + if (!txd) {
> >>> + rc = -EIO;
> >>> + goto out_unmap;
> >>> + }
> >>> +
> >>> + txd->callback_result = ntb_transport_edma_rc_write_cb;
> >>> + txd->callback_param = entry;
> >>> +
> >>> + cookie = dmaengine_submit(txd);
> >>> + if (dma_submit_error(cookie)) {
> >>> + rc = -EIO;
> >>> + goto out_unmap;
> >>> + }
> >>> + dma_async_issue_pending(chan);
> >>> + return 0;
> >>> +out_unmap:
> >>> + dma_unmap_sg(d, sgl, 1, DMA_FROM_DEVICE);
> >>> + return rc;
> >>> +}
> >>> +
> >>> +static int ntb_transport_edma_rc_read_start(struct device *d,
> >>> + struct dma_chan *chan, size_t len,
> >>> + void *rc_src, dma_addr_t ep_dst,
> >>> + struct ntb_queue_entry *entry)
> >>> +{
> >>> + struct scatterlist *sgl = &entry->sgl;
> >>> + struct dma_async_tx_descriptor *txd;
> >>> + struct dma_slave_config cfg;
> >>> + dma_cookie_t cookie;
> >>> + int nents, rc;
> >>> +
> >>> + if (!d)
> >>> + return -ENODEV;
> >>> +
> >>> + if (!chan)
> >>> + return -ENXIO;
> >>> +
> >>> + if (WARN_ON(!rc_src || !ep_dst))
> >>> + return -EINVAL;
> >>> +
> >>> + if (WARN_ON(sg_dma_address(sgl)))
> >>> + return -EINVAL;
> >>> +
> >>> + sg_init_one(sgl, rc_src, len);
> >>> + nents = dma_map_sg(d, sgl, 1, DMA_TO_DEVICE);
> >>> + if (nents <= 0)
> >>> + return -EIO;
> >>> +
> >>> + memset(&cfg, 0, sizeof(cfg));
> >>> + cfg.dst_addr = ep_dst;
> >>> + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> >>> + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
> >>> + cfg.direction = DMA_MEM_TO_DEV;
> >>> + rc = dmaengine_slave_config(chan, &cfg);
> >>> + if (rc)
> >>> + goto out_unmap;
> >>> +
> >>> + txd = dmaengine_prep_slave_sg(chan, sgl, 1, DMA_MEM_TO_DEV,
> >>> + DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
> >>> + if (!txd) {
> >>> + rc = -EIO;
> >>> + goto out_unmap;
> >>> + }
> >>> +
> >>> + txd->callback_result = ntb_transport_edma_rc_read_cb;
> >>> + txd->callback_param = entry;
> >>> +
> >>> + cookie = dmaengine_submit(txd);
> >>> + if (dma_submit_error(cookie)) {
> >>> + rc = -EIO;
> >>> + goto out_unmap;
> >>> + }
> >>> + dma_async_issue_pending(chan);
> >>> + return 0;
> >>> +out_unmap:
> >>> + dma_unmap_sg(d, sgl, 1, DMA_TO_DEVICE);
> >>> + return rc;
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_rc_dma_work(struct work_struct *work)
> >>> +{
> >>> + struct ntb_queue_entry *entry = container_of(
> >>> + work, struct ntb_queue_entry, dma_work);
> >>> + struct ntb_transport_qp *qp = entry->qp;
> >>> + struct ntb_transport_ctx *nt = qp->transport;
> >>> + struct device *dma_dev = get_dma_dev(qp->ndev);
> >>> + struct dma_chan *chan;
> >>> + int rc;
> >>> +
> >>> + chan = ntb_edma_pick_chan(&nt->edma, REMOTE_EDMA_WRITE);
> >>> + rc = ntb_transport_edma_rc_write_start(dma_dev, chan, entry->len,
> >>> + entry->addr, entry->buf, entry);
> >>> + if (rc) {
> >>> + entry->errors++;
> >>> + entry->len = -EIO;
> >>> + entry->flags |= DESC_DONE_FLAG;
> >>> + queue_work(nt->wq, &qp->write_work);
> >>> + return;
> >>> + }
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_rc_poll(struct ntb_transport_qp *qp)
> >>> +{
> >>> + struct ntb_transport_ctx *nt = qp->transport;
> >>> + unsigned int budget = NTB_EDMA_MAX_POLL;
> >>> + struct ntb_queue_entry *entry;
> >>> + struct ntb_edma_desc *in;
> >>> + dma_addr_t ep_src;
> >>> + u32 len, idx;
> >>> +
> >>> + while (budget--) {
> >>> + if (ntb_edma_ring_used_entry(READ_ONCE(*NTB_HEAD_WR_RC_I(qp)),
> >>> + qp->wr_issue) == 0)
> >>> + break;
> >>> +
> >>> + idx = ntb_edma_ring_idx(qp->wr_issue);
> >>> + in = NTB_DESC_WR_RC_I(qp, idx);
> >>> +
> >>> + len = READ_ONCE(in->len);
> >>> + ep_src = (dma_addr_t)READ_ONCE(in->addr);
> >>> +
> >>> + /* Prepare 'entry' for write completion */
> >>> + entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q);
> >>> + if (!entry) {
> >>> + qp->rx_err_no_buf++;
> >>> + break;
> >>> + }
> >>> + if (WARN_ON(entry->flags & DESC_DONE_FLAG))
> >>> + entry->flags &= ~DESC_DONE_FLAG;
> >>> + entry->len = len; /* NB. entry->len can be <=0 */
> >>> + entry->addr = ep_src;
> >>> +
> >>> + /*
> >>> + * ntb_transport_edma_rc_write_complete_work() checks entry->flags
> >>> + * so it needs to be set before wr_issue++.
> >>> + */
> >>> + in->data = (uintptr_t)entry;
> >>> +
> >>> + /* Ensure in->data visible before wr_issue++ */
> >>> + smp_wmb();
> >>> +
> >>> + WRITE_ONCE(qp->wr_issue, qp->wr_issue + 1);
> >>> +
> >>> + if (!len) {
> >>> + entry->flags |= DESC_DONE_FLAG;
> >>> + queue_work(nt->wq, &qp->write_work);
> >>> + continue;
> >>> + }
> >>> +
> >>> + if (in->flags & LINK_DOWN_FLAG) {
> >>> + dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n");
> >>> + entry->flags |= DESC_DONE_FLAG | LINK_DOWN_FLAG;
> >>> + queue_work(nt->wq, &qp->write_work);
> >>> + continue;
> >>> + }
> >>> +
> >>> + queue_work(nt->wq, &entry->dma_work);
> >>> + }
> >>> +
> >>> + if (!budget)
> >>> + tasklet_schedule(&qp->rxc_db_work);
> >>> +}
> >>> +
> >>> +static int ntb_transport_edma_rc_tx_enqueue(struct ntb_transport_qp *qp,
> >>> + struct ntb_queue_entry *entry)
> >>> +{
> >>> + struct device *dma_dev = get_dma_dev(qp->ndev);
> >>> + struct ntb_transport_ctx *nt = qp->transport;
> >>> + struct ntb_edma_desc *in, __iomem *out;
> >>> + unsigned int len = entry->len;
> >>> + struct dma_chan *chan;
> >>> + u32 issue, idx, head;
> >>> + dma_addr_t ep_dst;
> >>> + int rc;
> >>> +
> >>> + WARN_ON_ONCE(entry->flags & DESC_DONE_FLAG);
> >>> +
> >>> + scoped_guard(spinlock_irqsave, &qp->rc_lock) {
> >>> + head = READ_ONCE(*NTB_HEAD_RD_RC_I(qp));
> >>> + issue = qp->rd_issue;
> >>> + if (ntb_edma_ring_used_entry(head, issue) == 0) {
> >>> + qp->tx_ring_full++;
> >>> + return -ENOSPC;
> >>> + }
> >>> +
> >>> + /*
> >>> + * ntb_transport_edma_rc_read_complete_work() checks entry->flags
> >>> + * so it needs to be set before rd_issue++.
> >>> + */
> >>> + idx = ntb_edma_ring_idx(issue);
> >>> + in = NTB_DESC_RD_RC_I(qp, idx);
> >>> + in->data = (uintptr_t)entry;
> >>> +
> >>> + /* Make in->data visible before rd_issue++ */
> >>> + smp_wmb();
> >>> +
> >>> + WRITE_ONCE(qp->rd_issue, qp->rd_issue + 1);
> >>> + }
> >>> +
> >>> + /* Publish the final transfer length to the EP side */
> >>> + out = NTB_DESC_RD_RC_O(qp, idx);
> >>> + iowrite32(len, &out->len);
> >>> + ioread32(&out->len);
> >>> +
> >>> + if (unlikely(!len)) {
> >>> + entry->flags |= DESC_DONE_FLAG;
> >>> + queue_work(nt->wq, &qp->read_work);
> >>> + return 0;
> >>> + }
> >>> +
> >>> + /* Paired with dma_wmb() in ntb_transport_edma_ep_rx_enqueue() */
> >>> + dma_rmb();
> >>> +
> >>> + /* kick remote eDMA read transfer */
> >>> + ep_dst = (dma_addr_t)in->addr;
> >>> + chan = ntb_edma_pick_chan(&nt->edma, REMOTE_EDMA_READ);
> >>> + rc = ntb_transport_edma_rc_read_start(dma_dev, chan, len,
> >>> + entry->buf, ep_dst, entry);
> >>> + if (rc) {
> >>> + entry->errors++;
> >>> + entry->len = -EIO;
> >>> + entry->flags |= DESC_DONE_FLAG;
> >>> + queue_work(nt->wq, &qp->read_work);
> >>> + }
> >>> + return 0;
> >>> +}
> >>> +
> >>> +static int ntb_transport_edma_ep_tx_enqueue(struct ntb_transport_qp *qp,
> >>> + struct ntb_queue_entry *entry)
> >>> +{
> >>> + struct device *dma_dev = get_dma_dev(qp->ndev);
> >>> + struct ntb_edma_desc *in, __iomem *out;
> >>> + unsigned int len = entry->len;
> >>> + dma_addr_t ep_src = 0;
> >>> + u32 idx;
> >>> + int rc;
> >>> +
> >>> + if (likely(len)) {
> >>> + ep_src = dma_map_single(dma_dev, entry->buf, len,
> >>> + DMA_TO_DEVICE);
> >>> + rc = dma_mapping_error(dma_dev, ep_src);
> >>> + if (rc)
> >>> + return rc;
> >>> + }
> >>> +
> >>> + scoped_guard(spinlock_irqsave, &qp->ep_tx_lock) {
> >>> + if (ntb_edma_ring_full(qp->wr_prod, qp->wr_cons)) {
> >>> + rc = -ENOSPC;
> >>> + qp->tx_ring_full++;
> >>> + goto out_unmap;
> >>> + }
> >>> +
> >>> + idx = ntb_edma_ring_idx(qp->wr_prod);
> >>> + in = NTB_DESC_WR_EP_I(qp, idx);
> >>> + out = NTB_DESC_WR_EP_O(qp, idx);
> >>> +
> >>> + WARN_ON(in->flags & DESC_DONE_FLAG);
> >>> + WARN_ON(entry->flags & DESC_DONE_FLAG);
> >>> + in->flags = 0;
> >>> + in->data = (uintptr_t)entry;
> >>> + entry->addr = ep_src;
> >>> +
> >>> + iowrite32(len, &out->len);
> >>> + iowrite32(entry->flags, &out->flags);
> >>> + iowrite64(ep_src, &out->addr);
> >>> + WRITE_ONCE(qp->wr_prod, qp->wr_prod + 1);
> >>> +
> >>> + dma_wmb();
> >>> + iowrite32(qp->wr_prod, NTB_HEAD_WR_EP_O(qp));
> >>> +
> >>> + qp->tx_bytes += len;
> >>> + qp->tx_pkts++;
> >>> + }
> >>> +
> >>> + ntb_transport_edma_notify_peer(qp);
> >>> +
> >>> + return 0;
> >>> +out_unmap:
> >>> + if (likely(len))
> >>> + dma_unmap_single(dma_dev, ep_src, len, DMA_TO_DEVICE);
> >>> + return rc;
> >>> +}
> >>> +
> >>> +static int ntb_transport_edma_tx_enqueue(struct ntb_transport_qp *qp,
> >>> + struct ntb_queue_entry *entry,
> >>> + void *cb, void *data, unsigned int len,
> >>> + unsigned int flags)
> >>> +{
> >>> + struct device *dma_dev;
> >>> +
> >>> + if (entry->addr) {
> >>> + /* Deferred unmap */
> >>> + dma_dev = get_dma_dev(qp->ndev);
> >>> + dma_unmap_single(dma_dev, entry->addr, entry->len, DMA_TO_DEVICE);
> >>> + }
> >>> +
> >>> + entry->cb_data = cb;
> >>> + entry->buf = data;
> >>> + entry->len = len;
> >>> + entry->flags = flags;
> >>> + entry->errors = 0;
> >>> + entry->addr = 0;
> >>> +
> >>> + WARN_ON_ONCE(!ntb_qp_edma_enabled(qp));
> >>> +
> >>> + if (ntb_qp_edma_is_ep(qp))
> >>> + return ntb_transport_edma_ep_tx_enqueue(qp, entry);
> >>> + else
> >>> + return ntb_transport_edma_rc_tx_enqueue(qp, entry);
> >>> +}
> >>> +
> >>> +static int ntb_transport_edma_ep_rx_enqueue(struct ntb_transport_qp *qp,
> >>> + struct ntb_queue_entry *entry)
> >>> +{
> >>> + struct device *dma_dev = get_dma_dev(qp->ndev);
> >>> + struct ntb_edma_desc *in, __iomem *out;
> >>> + unsigned int len = entry->len;
> >>> + void *data = entry->buf;
> >>> + dma_addr_t ep_dst;
> >>> + u32 idx;
> >>> + int rc;
> >>> +
> >>> + ep_dst = dma_map_single(dma_dev, data, len, DMA_FROM_DEVICE);
> >>> + rc = dma_mapping_error(dma_dev, ep_dst);
> >>> + if (rc)
> >>> + return rc;
> >>> +
> >>> + scoped_guard(spinlock_bh, &qp->ep_rx_lock) {
> >>> + if (ntb_edma_ring_full(READ_ONCE(qp->rd_prod),
> >>> + READ_ONCE(qp->rd_cons))) {
> >>> + rc = -ENOSPC;
> >>> + goto out_unmap;
> >>> + }
> >>> +
> >>> + idx = ntb_edma_ring_idx(qp->rd_prod);
> >>> + in = NTB_DESC_RD_EP_I(qp, idx);
> >>> + out = NTB_DESC_RD_EP_O(qp, idx);
> >>> +
> >>> + iowrite32(len, &out->len);
> >>> + iowrite64(ep_dst, &out->addr);
> >>> +
> >>> + WARN_ON(in->flags & DESC_DONE_FLAG);
> >>> + in->data = (uintptr_t)entry;
> >>> + entry->addr = ep_dst;
> >>> +
> >>> + /* Ensure len/addr are visible before the head update */
> >>> + dma_wmb();
> >>> +
> >>> + WRITE_ONCE(qp->rd_prod, qp->rd_prod + 1);
> >>> + iowrite32(qp->rd_prod, NTB_HEAD_RD_EP_O(qp));
> >>> + }
> >>> + return 0;
> >>> +out_unmap:
> >>> + dma_unmap_single(dma_dev, ep_dst, len, DMA_FROM_DEVICE);
> >>> + return rc;
> >>> +}
> >>> +
> >>> +static int ntb_transport_edma_rx_enqueue(struct ntb_transport_qp *qp,
> >>> + struct ntb_queue_entry *entry)
> >>> +{
> >>> + int rc;
> >>> +
> >>> + /* The behaviour is the same as the default backend for RC side */
> >>> + if (ntb_qp_edma_is_ep(qp)) {
> >>> + rc = ntb_transport_edma_ep_rx_enqueue(qp, entry);
> >>> + if (rc) {
> >>> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
> >>> + &qp->rx_free_q);
> >>> + return rc;
> >>> + }
> >>> + }
> >>> +
> >>> + ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q);
> >>> +
> >>> + if (qp->active)
> >>> + tasklet_schedule(&qp->rxc_db_work);
> >>> +
> >>> + return 0;
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_rx_poll(struct ntb_transport_qp *qp)
> >>> +{
> >>> + struct ntb_transport_ctx *nt = qp->transport;
> >>> +
> >>> + if (ntb_qp_edma_is_rc(qp))
> >>> + ntb_transport_edma_rc_poll(qp);
> >>> + else if (ntb_qp_edma_is_ep(qp)) {
> >>> + /*
> >>> + * Make sure we poll the rings even if an eDMA interrupt is
> >>> + * cleared on the RC side earlier.
> >>> + */
> >>> + queue_work(nt->wq, &qp->read_work);
> >>> + queue_work(nt->wq, &qp->write_work);
> >>> + } else
> >>> + /* Unreachable */
> >>> + WARN_ON_ONCE(1);
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_read_work(struct work_struct *work)
> >>> +{
> >>> + struct ntb_transport_qp *qp = container_of(
> >>> + work, struct ntb_transport_qp, read_work);
> >>> +
> >>> + if (ntb_qp_edma_is_rc(qp))
> >>> + ntb_transport_edma_rc_read_complete_work(work);
> >>> + else if (ntb_qp_edma_is_ep(qp))
> >>> + ntb_transport_edma_ep_read_work(work);
> >>> + else
> >>> + /* Unreachable */
> >>> + WARN_ON_ONCE(1);
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_write_work(struct work_struct *work)
> >>> +{
> >>> + struct ntb_transport_qp *qp = container_of(
> >>> + work, struct ntb_transport_qp, write_work);
> >>> +
> >>> + if (ntb_qp_edma_is_rc(qp))
> >>> + ntb_transport_edma_rc_write_complete_work(work);
> >>> + else if (ntb_qp_edma_is_ep(qp))
> >>> + ntb_transport_edma_ep_write_work(work);
> >>> + else
> >>> + /* Unreachable */
> >>> + WARN_ON_ONCE(1);
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_init_queue(struct ntb_transport_ctx *nt,
> >>> + unsigned int qp_num)
> >>> +{
> >>> + struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
> >>> +
> >>> + qp->wr_cons = 0;
> >>> + qp->rd_cons = 0;
> >>> + qp->wr_prod = 0;
> >>> + qp->rd_prod = 0;
> >>> + qp->wr_issue = 0;
> >>> + qp->rd_issue = 0;
> >>> +
> >>> + INIT_WORK(&qp->db_work, ntb_transport_edma_db_work);
> >>> + INIT_WORK(&qp->read_work, ntb_transport_edma_read_work);
> >>> + INIT_WORK(&qp->write_work, ntb_transport_edma_write_work);
> >>> +}
> >>> +
> >>> +static void ntb_transport_edma_create_queue(struct ntb_transport_ctx *nt,
> >>> + struct ntb_transport_qp *qp)
> >>> +{
> >>> + spin_lock_init(&qp->ep_tx_lock);
> >>> + spin_lock_init(&qp->ep_rx_lock);
> >>> + spin_lock_init(&qp->rc_lock);
> >>> +}
> >>> +
> >>> +static const struct ntb_transport_backend_ops edma_backend_ops = {
> >>> + .setup_qp_mw = ntb_transport_edma_setup_qp_mw,
> >>> + .tx_free_entry = ntb_transport_edma_tx_free_entry,
> >>> + .tx_enqueue = ntb_transport_edma_tx_enqueue,
> >>> + .rx_enqueue = ntb_transport_edma_rx_enqueue,
> >>> + .rx_poll = ntb_transport_edma_rx_poll,
> >>> + .debugfs_stats_show = ntb_transport_edma_debugfs_stats_show,
> >>> +};
> >>> +#endif /* CONFIG_NTB_TRANSPORT_EDMA */
> >>> +
> >>> /**
> >>> * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
> >>> * @qp: NTB transport layer queue to be enabled
> >>
>
^ permalink raw reply [flat|nested] 19+ messages in thread