[PATCH 0/3] Add the support of tun to RXE

[PATCH 0/3] Add the support of tun to RXE

[Zhu Yanjun]

Currently the tun device can work well with SIW while it can not work well
with SoftROCE.

The RXE device can be attached to the tun device with the command "rdma
link add ...". But when running rping command over this RXE device, some
error will occur.

The following bash script can verify this patch series.

"
ip tuntap add mode tun tun0
ip -4 a
ip addr add 1.1.1.1/24 dev tun0
ip link set tun0 up
ip -4 a
rdma link add rxe0 type rxe netdev tun0
rdma link
rping -s -a 1.1.1.1&
dmesg -c
rping -c -a 1.1.1.1 -d -v -C 3
rdma link del rxe0
rdma link
ip addr del 1.1.1.1/24 dev tun0
ip tuntap del mode tun tun0
modprobe -v -r tun
"

In the above script, a tun device is added and an ip address is configured
on it. Then a RXE device is attached to this tun device. A rping test is
executed with this RXE device. It should be successful with this patchset.
Finally all the RXE device and tun device should be released after the
rping command is executed.

Without this patch series, when running "rping -c -a ..." command, the
following errors will appear.

"
...
created cm_id 0x556b45faf2a0
cma_event type RDMA_CM_EVENT_ADDR_ERROR cma_id 0x556b45faf2a0 (parent)
cma event RDMA_CM_EVENT_ADDR_ERROR, error -19
waiting for addr/route resolution state 1
destroy cm_id 0x556b45faf2a0
"

Zhu Yanjun (3):
  RDMA/rxe: Replace netdev dev addr with raw_gid
  RDMA/rxe: Add query_gid support
  RDMA/rxe: Make rping work with tun device

 drivers/infiniband/core/cma.c         | 24 +++++++++++++++++++-----
 drivers/infiniband/sw/rxe/rxe.c       | 15 +++++++++++++--
 drivers/infiniband/sw/rxe/rxe_verbs.c | 15 ++++++++++++++-
 drivers/infiniband/sw/rxe/rxe_verbs.h |  4 +++-
 4 files changed, 49 insertions(+), 9 deletions(-)

-- 
2.34.1

[PATCH 1/3] RDMA/rxe: Replace netdev dev addr with raw_gid

[Zhu Yanjun]

Because TUN device does not have dev_addr, but a gid in rdma is needed,
as such, a raw_gid is generated to act as the gid. The similar commit is
in SIW. This commit learns from the similar commit bad5b6e34ffb
("RDMA/siw: Fabricate a GID on tun and loopback devices") in SIW.

Signed-off-by: Zhu Yanjun <yanjun.zhu@linux.dev>
---
 drivers/infiniband/sw/rxe/rxe.c       | 15 +++++++++++++--
 drivers/infiniband/sw/rxe/rxe_verbs.c |  2 +-
 drivers/infiniband/sw/rxe/rxe_verbs.h |  4 +++-
 3 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index 1ba4a0c8726a..432e54a29b99 100644
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -78,8 +78,19 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
 	if (!ndev)
 		return;
 
+	if (ndev->addr_len) {
+		memcpy(rxe->raw_gid, ndev->dev_addr,
+			min_t(unsigned int, ndev->addr_len, ETH_ALEN));
+	} else {
+		/*
+		 * This device does not have a HW address, but
+		 * connection mangagement requires a unique gid.
+		 */
+		eth_random_addr(rxe->raw_gid);
+	}
+
 	addrconf_addr_eui48((unsigned char *)&rxe->attr.sys_image_guid,
-			ndev->dev_addr);
+			rxe->raw_gid);
 
 	dev_put(ndev);
 
@@ -125,7 +136,7 @@ static void rxe_init_ports(struct rxe_dev *rxe)
 	if (!ndev)
 		return;
 	addrconf_addr_eui48((unsigned char *)&port->port_guid,
-			    ndev->dev_addr);
+			    rxe->raw_gid);
 	dev_put(ndev);
 	spin_lock_init(&port->port_lock);
 }
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 8a5fc20fd186..837566cd682c 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -1523,7 +1523,7 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name,
 	dev->num_comp_vectors = num_possible_cpus();
 	dev->local_dma_lkey = 0;
 	addrconf_addr_eui48((unsigned char *)&dev->node_guid,
-			    ndev->dev_addr);
+			    rxe->raw_gid);
 
 	dev->uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_POST_SEND) |
 				BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ);
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 6573ceec0ef5..729a6ada46af 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -376,7 +376,9 @@ struct rxe_dev {
 	struct ib_device_attr	attr;
 	int			max_ucontext;
 	int			max_inline_data;
-	struct mutex	usdev_lock;
+	struct mutex		usdev_lock;
+
+	char			raw_gid[ETH_ALEN];
 
 	struct rxe_pool		uc_pool;
 	struct rxe_pool		pd_pool;
-- 
2.34.1

[PATCH 2/3] RDMA/rxe: Add query_gid support

[Zhu Yanjun]

The query_gid is not implemented in RXE. After the raw_gid is added,
this query_gid should be implemented in RXE.

Signed-off-by: Zhu Yanjun <yanjun.zhu@linux.dev>
---
 drivers/infiniband/sw/rxe/rxe_verbs.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 837566cd682c..22d6312ecbc7 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -79,6 +79,18 @@ static int rxe_query_port(struct ib_device *ibdev,
 	return err;
 }
 
+static int rxe_query_gid(struct ib_device *ibdev, u32 port, int idx,
+			 union ib_gid *gid)
+{
+	struct rxe_dev *rxe = to_rdev(ibdev);
+
+	/* subnet_prefix == interface_id == 0; */
+	memset(gid, 0, sizeof(*gid));
+	memcpy(gid->raw, rxe->raw_gid, ETH_ALEN);
+
+	return 0;
+}
+
 static int rxe_query_pkey(struct ib_device *ibdev,
 			  u32 port_num, u16 index, u16 *pkey)
 {
@@ -1493,6 +1505,7 @@ static const struct ib_device_ops rxe_dev_ops = {
 	.query_ah = rxe_query_ah,
 	.query_device = rxe_query_device,
 	.query_pkey = rxe_query_pkey,
+	.query_gid = rxe_query_gid,
 	.query_port = rxe_query_port,
 	.query_qp = rxe_query_qp,
 	.query_srq = rxe_query_srq,
-- 
2.34.1

[PATCH 3/3] RDMA/rxe: Make rping work with tun device

[Zhu Yanjun]

Because the type of tun device is ARPHRD_NONE, in cma, ARPHRD_NONE is
not recognized. To RXE device, just as SIW does, ARPHRD_NONE is added
to support.

Signed-off-by: Zhu Yanjun <yanjun.zhu@linux.dev>
---
 drivers/infiniband/core/cma.c | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 91db10515d74..fedcdb56fb6b 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -739,12 +739,26 @@ cma_validate_port(struct ib_device *device, u32 port,
 		goto out;
 	}
 
-	if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port)) {
-		ndev = dev_get_by_index(dev_addr->net, bound_if_index);
-		if (!ndev)
-			goto out;
+	/*
+	 * For a RXE device, it should work with TUN device and normal ethernet
+	 * devices. Use driver_id to check if a device is a RXE device or not.
+	 * ARPHDR_NONE means a TUN device.
+	 */
+	if (device->ops.driver_id == RDMA_DRIVER_RXE) {
+		if ((dev_type == ARPHRD_NONE || dev_type == ARPHRD_ETHER)
+			&& rdma_protocol_roce(device, port)) {
+			ndev = dev_get_by_index(dev_addr->net, bound_if_index);
+			if (!ndev)
+				goto out;
+		}
 	} else {
-		gid_type = IB_GID_TYPE_IB;
+		if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port)) {
+			ndev = dev_get_by_index(dev_addr->net, bound_if_index);
+			if (!ndev)
+				goto out;
+		} else {
+			gid_type = IB_GID_TYPE_IB;
+		}
 	}
 
 	sgid_attr = rdma_find_gid_by_port(device, gid, gid_type, port, ndev);
-- 
2.34.1

Re: [PATCH 0/3] Add the support of tun to RXE

[Leon Romanovsky]

On Sun, 19 Jan 2025 18:28:28 +0100, Zhu Yanjun wrote:
> Currently the tun device can work well with SIW while it can not work well
> with SoftROCE.
> 
> The RXE device can be attached to the tun device with the command "rdma
> link add ...". But when running rping command over this RXE device, some
> error will occur.
> 
> [...]

Applied, thanks!

[1/3] RDMA/rxe: Replace netdev dev addr with raw_gid
      https://git.kernel.org/rdma/rdma/c/d34d0bdb500e6f
[2/3] RDMA/rxe: Add query_gid support
      https://git.kernel.org/rdma/rdma/c/93486fc96f0e26
[3/3] RDMA/rxe: Make rping work with tun device
      https://git.kernel.org/rdma/rdma/c/190797d47f16d2

Best regards,
-- 
Leon Romanovsky <leon@kernel.org>