Re: [for-next PATCH v5 06/11] RDMA/rxe: Extend rxe packet format to support flush

Yanjun Zhu
2022-11-11T08:43:53Z

在 2022/9/27 13:53, Li Zhijian 写道:
> Extend rxe opcode tables, headers, helper and constants to support
> flush operations.
>
> Refer to the IBA A19.4.1 for more FETH definition details
>
> Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
> ---
> V5: new FETH structure and simplify header helper
> new names and new patch split scheme, suggested by Bob.
> ---
> drivers/infiniband/sw/rxe/rxe_hdr.h | 47 ++++++++++++++++++++++++++
> drivers/infiniband/sw/rxe/rxe_opcode.c | 17 ++++++++++
> drivers/infiniband/sw/rxe/rxe_opcode.h | 16 +++++----
> 3 files changed, 74 insertions(+), 6 deletions(-)
>
> diff --git a/drivers/infiniband/sw/rxe/rxe_hdr.h b/drivers/infiniband/sw/rxe/rxe_hdr.h
> index e432f9e37795..e995a97c54fd 100644
> --- a/drivers/infiniband/sw/rxe/rxe_hdr.h
> +++ b/drivers/infiniband/sw/rxe/rxe_hdr.h
> @@ -607,6 +607,52 @@ static inline void reth_set_len(struct rxe_pkt_info *pkt, u32 len)
> rxe_opcode[pkt->opcode].offset[RXE_RETH], len);
> }
>
> +/******************************************************************************
> + * FLUSH Extended Transport Header
> + ******************************************************************************/
> +
> +struct rxe_feth {
> + __be32 bits;
> +};
> +
> +#define FETH_PLT_MASK (0x0000000f) /* bits 3-0 */
> +#define FETH_SEL_MASK (0x00000030) /* bits 5-4 */
> +#define FETH_SEL_SHIFT (4U)
> +
> +static inline u32 __feth_plt(void *arg)
> +{
> + struct rxe_feth *feth = arg;
> +
> + return be32_to_cpu(feth->bits) & FETH_PLT_MASK;
> +}
> +
> +static inline u32 __feth_sel(void *arg)
> +{
> + struct rxe_feth *feth = arg;
> +
> + return (be32_to_cpu(feth->bits) & FETH_SEL_MASK) >> FETH_SEL_SHIFT;
> +}
> +
> +static inline u32 feth_plt(struct rxe_pkt_info *pkt)
> +{
> + return __feth_plt(pkt->hdr + rxe_opcode[pkt->opcode].offset[RXE_FETH]);
> +}
> +
> +static inline u32 feth_sel(struct rxe_pkt_info *pkt)
> +{
> + return __feth_sel(pkt->hdr + rxe_opcode[pkt->opcode].offset[RXE_FETH]);
> +}
> +
> +static inline void feth_init(struct rxe_pkt_info *pkt, u8 type, u8 level)
> +{
> + struct rxe_feth *feth = (struct rxe_feth *)
> + (pkt->hdr + rxe_opcode[pkt->opcode].offset[RXE_FETH]);
> + u32 bits = ((level << FETH_SEL_SHIFT) & FETH_SEL_MASK) |
> + (type & FETH_PLT_MASK);
> +
> + feth->bits = cpu_to_be32(bits);
> +}
> +
> /******************************************************************************
> * Atomic Extended Transport Header
> ******************************************************************************/
> @@ -910,6 +956,7 @@ enum rxe_hdr_length {
> RXE_ATMETH_BYTES = sizeof(struct rxe_atmeth),
> RXE_IETH_BYTES = sizeof(struct rxe_ieth),
> RXE_RDETH_BYTES = sizeof(struct rxe_rdeth),
> + RXE_FETH_BYTES = sizeof(struct rxe_feth),
> };
>
> static inline size_t header_size(struct rxe_pkt_info *pkt)
> diff --git a/drivers/infiniband/sw/rxe/rxe_opcode.c b/drivers/infiniband/sw/rxe/rxe_opcode.c
> index d4ba4d506f17..55aad13e57bb 100644
> --- a/drivers/infiniband/sw/rxe/rxe_opcode.c
> +++ b/drivers/infiniband/sw/rxe/rxe_opcode.c
> @@ -101,6 +101,12 @@ struct rxe_wr_opcode_info rxe_wr_opcode_info[] = {
> [IB_QPT_UC] = WR_LOCAL_OP_MASK,
> },
> },
> + [IB_WR_FLUSH] = {
> + .name = "IB_WR_FLUSH",
> + .mask = {
> + [IB_QPT_RC] = WR_FLUSH_MASK,
> + },
> + },
> };

Hi, Zhijian

I am making tests with it. Except rc, other modes are supported? such as
rd, xrc?

Zhu Yanjun

>
> struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = {
> @@ -378,6 +384,17 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = {
> RXE_IETH_BYTES,
> }
> },
> + [IB_OPCODE_RC_FLUSH] = {
> + .name = "IB_OPCODE_RC_FLUSH",
> + .mask = RXE_FETH_MASK | RXE_RETH_MASK | RXE_FLUSH_MASK |
> + RXE_START_MASK | RXE_END_MASK | RXE_REQ_MASK,
> + .length = RXE_BTH_BYTES + RXE_FETH_BYTES + RXE_RETH_BYTES,
> + .offset = {
> + [RXE_BTH] = 0,
> + [RXE_FETH] = RXE_BTH_BYTES,
> + [RXE_RETH] = RXE_BTH_BYTES + RXE_FETH_BYTES,
> + }
> + },
>
> /* UC */
> [IB_OPCODE_UC_SEND_FIRST] = {
> diff --git a/drivers/infiniband/sw/rxe/rxe_opcode.h b/drivers/infiniband/sw/rxe/rxe_opcode.h
> index 8f9aaaf260f2..02d256745793 100644
> --- a/drivers/infiniband/sw/rxe/rxe_opcode.h
> +++ b/drivers/infiniband/sw/rxe/rxe_opcode.h
> @@ -19,7 +19,8 @@ enum rxe_wr_mask {
> WR_SEND_MASK = BIT(2),
> WR_READ_MASK = BIT(3),
> WR_WRITE_MASK = BIT(4),
> - WR_LOCAL_OP_MASK = BIT(5),
> + WR_FLUSH_MASK = BIT(5),
> + WR_LOCAL_OP_MASK = BIT(6),
>
> WR_READ_OR_WRITE_MASK = WR_READ_MASK | WR_WRITE_MASK,
> WR_WRITE_OR_SEND_MASK = WR_WRITE_MASK | WR_SEND_MASK,
> @@ -47,6 +48,7 @@ enum rxe_hdr_type {
> RXE_RDETH,
> RXE_DETH,
> RXE_IMMDT,
> + RXE_FETH,
> RXE_PAYLOAD,
> NUM_HDR_TYPES
> };
> @@ -63,6 +65,7 @@ enum rxe_hdr_mask {
> RXE_IETH_MASK = BIT(RXE_IETH),
> RXE_RDETH_MASK = BIT(RXE_RDETH),
> RXE_DETH_MASK = BIT(RXE_DETH),
> + RXE_FETH_MASK = BIT(RXE_FETH),
> RXE_PAYLOAD_MASK = BIT(RXE_PAYLOAD),
>
> RXE_REQ_MASK = BIT(NUM_HDR_TYPES + 0),
> @@ -71,13 +74,14 @@ enum rxe_hdr_mask {
> RXE_WRITE_MASK = BIT(NUM_HDR_TYPES + 3),
> RXE_READ_MASK = BIT(NUM_HDR_TYPES + 4),
> RXE_ATOMIC_MASK = BIT(NUM_HDR_TYPES + 5),
> + RXE_FLUSH_MASK = BIT(NUM_HDR_TYPES + 6),
>
> - RXE_RWR_MASK = BIT(NUM_HDR_TYPES + 6),
> - RXE_COMP_MASK = BIT(NUM_HDR_TYPES + 7),
> + RXE_RWR_MASK = BIT(NUM_HDR_TYPES + 7),
> + RXE_COMP_MASK = BIT(NUM_HDR_TYPES + 8),
>
> - RXE_START_MASK = BIT(NUM_HDR_TYPES + 8),
> - RXE_MIDDLE_MASK = BIT(NUM_HDR_TYPES + 9),
> - RXE_END_MASK = BIT(NUM_HDR_TYPES + 10),
> + RXE_START_MASK = BIT(NUM_HDR_TYPES + 9),
> + RXE_MIDDLE_MASK = BIT(NUM_HDR_TYPES + 10),
> + RXE_END_MASK = BIT(NUM_HDR_TYPES + 11),
>
> RXE_LOOPBACK_MASK = BIT(NUM_HDR_TYPES + 12),
>