Re: [PATCH v3 02/31] elx: libefc_sli: SLI Descriptors and Queue entries

[Hannes Reinecke <hare@suse.de>]

On 4/12/20 5:32 AM, James Smart wrote:
> This patch continues the libefc_sli SLI-4 library population.
> 
> This patch add SLI-4 Data structures and defines for:
> - Buffer Descriptors (BDEs)
> - Scatter/Gather List elements (SGEs)
> - Queues and their Entry Descriptions for:
>     Event Queues (EQs), Completion Queues (CQs),
>     Receive Queues (RQs), and the Mailbox Queue (MQ).
> 
> Signed-off-by: Ram Vegesna <ram.vegesna@broadcom.com>
> Signed-off-by: James Smart <jsmart2021@gmail.com>
> 
> ---
> v3:
>    Changed anonymous enums to named.
>    SLI defines to spell out _MASK value directly.
>    Change multiple defines to named Enums for consistency.
>    Single Enum to #define.
> ---
>   drivers/scsi/elx/include/efc_common.h |   25 +
>   drivers/scsi/elx/libefc_sli/sli4.h    | 1761 +++++++++++++++++++++++++++++++++
>   2 files changed, 1786 insertions(+)
>   create mode 100644 drivers/scsi/elx/include/efc_common.h
> 
> diff --git a/drivers/scsi/elx/include/efc_common.h b/drivers/scsi/elx/include/efc_common.h
> new file mode 100644
> index 000000000000..c427f75da4d5
> --- /dev/null
> +++ b/drivers/scsi/elx/include/efc_common.h
> @@ -0,0 +1,25 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2019 Broadcom. All Rights Reserved. The term
> + * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
> + */
> +
> +#ifndef __EFC_COMMON_H__
> +#define __EFC_COMMON_H__
> +
> +#include <linux/pci.h>
> +
> +#define EFC_SUCCESS	0
> +#define EFC_FAIL	1
> +
> +struct efc_dma {
> +	void		*virt;
> +	void            *alloc;
> +	dma_addr_t	phys;
> +
> +	size_t		size;
> +	size_t          len;
> +	struct pci_dev	*pdev;
> +};
> +
> +#endif /* __EFC_COMMON_H__ */
> diff --git a/drivers/scsi/elx/libefc_sli/sli4.h b/drivers/scsi/elx/libefc_sli/sli4.h
> index 1fad48643f94..07eef8df9690 100644
> --- a/drivers/scsi/elx/libefc_sli/sli4.h
> +++ b/drivers/scsi/elx/libefc_sli/sli4.h
> @@ -249,4 +249,1765 @@ struct sli4_reg {
>   	u32	off;
>   };
>   
> +struct sli4_dmaaddr {
> +	__le32 low;
> +	__le32 high;
> +};
> +
> +/* a 3-word BDE with address 1st 2 words, length last word */
> +struct sli4_bufptr {
> +	struct sli4_dmaaddr addr;
> +	__le32 length;
> +};
> +
> +/* a 3-word BDE with length as first word, address last 2 words */
> +struct sli4_bufptr_len1st {
> +	__le32 length0;
> +	struct sli4_dmaaddr addr;
> +};
> +
> +/* Buffer Descriptor Entry (BDE) */
> +enum sli4_bde_e {
> +	SLI4_BDE_MASK_BUFFER_LEN	= 0x00ffffff,
> +	SLI4_BDE_MASK_BDE_TYPE		= 0xff000000,
> +};
> +
> +struct sli4_bde {
> +	__le32		bde_type_buflen;
> +	union {
> +		struct sli4_dmaaddr data;
> +		struct {
> +			__le32	offset;
> +			__le32	rsvd2;
> +		} imm;
> +		struct sli4_dmaaddr blp;
> +	} u;
> +};
> +
> +/* Buffer Descriptors */
> +enum sli4_bde_type {
> +	BDE_TYPE_SHIFT		= 24,
> +	BDE_TYPE_BDE_64		= 0x00,	/* Generic 64-bit data */
> +	BDE_TYPE_BDE_IMM	= 0x01,	/* Immediate data */
> +	BDE_TYPE_BLP		= 0x40,	/* Buffer List Pointer */
> +};
> +
> +/* Scatter-Gather Entry (SGE) */
> +#define SLI4_SGE_MAX_RESERVED			3
> +
> +enum sli4_sge_type {
> +	/* DW2 */
> +	SLI4_SGE_DATA_OFFSET_MASK	= 0x07FFFFFF,
> +	/*DW2W1*/
> +	SLI4_SGE_TYPE_SHIFT		= 27,
> +	SLI4_SGE_TYPE_MASK		= 0x78000000,
> +	/*SGE Types*/
> +	SLI4_SGE_TYPE_DATA		= 0x00,
> +	SLI4_SGE_TYPE_DIF		= 0x04,	/* Data Integrity Field */
> +	SLI4_SGE_TYPE_LSP		= 0x05,	/* List Segment Pointer */
> +	SLI4_SGE_TYPE_PEDIF		= 0x06,	/* Post Encryption Engine DIF */
> +	SLI4_SGE_TYPE_PESEED		= 0x07,	/* Post Encryption DIF Seed */
> +	SLI4_SGE_TYPE_DISEED		= 0x08,	/* DIF Seed */
> +	SLI4_SGE_TYPE_ENC		= 0x09,	/* Encryption */
> +	SLI4_SGE_TYPE_ATM		= 0x0a,	/* DIF Application Tag Mask */
> +	SLI4_SGE_TYPE_SKIP		= 0x0c,	/* SKIP */
> +
> +	SLI4_SGE_LAST			= (1 << 31),
> +};
> +
> +struct sli4_sge {
> +	__le32		buffer_address_high;
> +	__le32		buffer_address_low;
> +	__le32		dw2_flags;
> +	__le32		buffer_length;
> +};
> +
> +/* T10 DIF Scatter-Gather Entry (SGE) */
> +struct sli4_dif_sge {
> +	__le32		buffer_address_high;
> +	__le32		buffer_address_low;
> +	__le32		dw2_flags;
> +	__le32		rsvd12;
> +};
> +
> +/* Data Integrity Seed (DISEED) SGE */
> +enum sli4_diseed_sge_flags {
> +	/* DW2W1 */
> +	DISEED_SGE_HS			= (1 << 2),
> +	DISEED_SGE_WS			= (1 << 3),
> +	DISEED_SGE_IC			= (1 << 4),
> +	DISEED_SGE_ICS			= (1 << 5),
> +	DISEED_SGE_ATRT			= (1 << 6),
> +	DISEED_SGE_AT			= (1 << 7),
> +	DISEED_SGE_FAT			= (1 << 8),
> +	DISEED_SGE_NA			= (1 << 9),
> +	DISEED_SGE_HI			= (1 << 10),
> +
> +	/* DW3W1 */
> +	DISEED_SGE_BS_MASK		= 0x0007,
> +	DISEED_SGE_AI			= (1 << 3),
> +	DISEED_SGE_ME			= (1 << 4),
> +	DISEED_SGE_RE			= (1 << 5),
> +	DISEED_SGE_CE			= (1 << 6),
> +	DISEED_SGE_NR			= (1 << 7),
> +
> +	DISEED_SGE_OP_RX_SHIFT		= 8,
> +	DISEED_SGE_OP_RX_MASK		= 0x0F00,
> +	DISEED_SGE_OP_TX_SHIFT		= 12,
> +	DISEED_SGE_OP_TX_MASK		= 0xF000,
> +};
> +
> +/* Opcode values */
> +enum sli4_diseed_sge_opcodes {
> +	DISEED_SGE_OP_IN_NODIF_OUT_CRC,
> +	DISEED_SGE_OP_IN_CRC_OUT_NODIF,
> +	DISEED_SGE_OP_IN_NODIF_OUT_CSUM,
> +	DISEED_SGE_OP_IN_CSUM_OUT_NODIF,
> +	DISEED_SGE_OP_IN_CRC_OUT_CRC,
> +	DISEED_SGE_OP_IN_CSUM_OUT_CSUM,
> +	DISEED_SGE_OP_IN_CRC_OUT_CSUM,
> +	DISEED_SGE_OP_IN_CSUM_OUT_CRC,
> +	DISEED_SGE_OP_IN_RAW_OUT_RAW,
> +};
> +
> +#define DISEED_SGE_OP_RX_VALUE(stype) \
> +	(DISEED_SGE_OP_##stype << DISEED_SGE_OP_RX_SHIFT)
> +#define DISEED_SGE_OP_TX_VALUE(stype) \
> +	(DISEED_SGE_OP_##stype << DISEED_SGE_OP_TX_SHIFT)
> +
> +struct sli4_diseed_sge {
> +	__le32		ref_tag_cmp;
> +	__le32		ref_tag_repl;
> +	__le16		app_tag_repl;
> +	__le16		dw2w1_flags;
> +	__le16		app_tag_cmp;
> +	__le16		dw3w1_flags;
> +};
> +
> +/* List Segment Pointer Scatter-Gather Entry (SGE) */
> +#define SLI4_LSP_SGE_SEGLEN	0x00ffffff
> +
> +struct sli4_lsp_sge {
> +	__le32		buffer_address_high;
> +	__le32		buffer_address_low;
> +	__le32		dw2_flags;
> +	__le32		dw3_seglen;
> +};
> +
> +enum sli4_eqe_e {
> +	SLI4_EQE_VALID	= 1,
> +	SLI4_EQE_MJCODE	= 0xe,
> +	SLI4_EQE_MNCODE	= 0xfff0,
> +};
> +
> +struct sli4_eqe {
> +	__le16		dw0w0_flags;
> +	__le16		resource_id;
> +};
> +
> +#define SLI4_MAJOR_CODE_STANDARD	0
> +#define SLI4_MAJOR_CODE_SENTINEL	1
> +
> +/* Sentinel EQE indicating the EQ is full */
> +#define SLI4_EQE_STATUS_EQ_FULL		2
> +
> +enum sli4_mcqe_e {
> +	SLI4_MCQE_CONSUMED	= (1 << 27),
> +	SLI4_MCQE_COMPLETED	= (1 << 28),
> +	SLI4_MCQE_AE		= (1 << 30),
> +	SLI4_MCQE_VALID		= (1 << 31),
> +};
> +
> +/* Entry was consumed but not completed */
> +#define SLI4_MCQE_STATUS_NOT_COMPLETED	-2
> +
> +struct sli4_mcqe {
> +	__le16		completion_status;
> +	__le16		extended_status;
> +	__le32		mqe_tag_low;
> +	__le32		mqe_tag_high;
> +	__le32		dw3_flags;
> +};
> +
> +enum sli4_acqe_e {
> +	SLI4_ACQE_AE	= (1 << 6), /* async event - this is an ACQE */
> +	SLI4_ACQE_VAL	= (1 << 7), /* valid - contents of CQE are valid */
> +};
> +
> +struct sli4_acqe {
> +	__le32		event_data[3];
> +	u8		rsvd12;
> +	u8		event_code;
> +	u8		event_type;
> +	u8		ae_val;
> +};
> +
> +enum sli4_acqe_event_code {
> +	SLI4_ACQE_EVENT_CODE_LINK_STATE		= 0x01,
> +	SLI4_ACQE_EVENT_CODE_FIP		= 0x02,
> +	SLI4_ACQE_EVENT_CODE_DCBX		= 0x03,
> +	SLI4_ACQE_EVENT_CODE_ISCSI		= 0x04,
> +	SLI4_ACQE_EVENT_CODE_GRP_5		= 0x05,
> +	SLI4_ACQE_EVENT_CODE_FC_LINK_EVENT	= 0x10,
> +	SLI4_ACQE_EVENT_CODE_SLI_PORT_EVENT	= 0x11,
> +	SLI4_ACQE_EVENT_CODE_VF_EVENT		= 0x12,
> +	SLI4_ACQE_EVENT_CODE_MR_EVENT		= 0x13,
> +};
> +
> +enum sli4_qtype {
> +	SLI_QTYPE_EQ,
> +	SLI_QTYPE_CQ,
> +	SLI_QTYPE_MQ,
> +	SLI_QTYPE_WQ,
> +	SLI_QTYPE_RQ,
> +	SLI_QTYPE_MAX,			/* must be last */
> +};
> +
> +#define SLI_USER_MQ_COUNT	1
> +#define SLI_MAX_CQ_SET_COUNT	16
> +#define SLI_MAX_RQ_SET_COUNT	16
> +
> +enum sli4_qentry {
> +	SLI_QENTRY_ASYNC,
> +	SLI_QENTRY_MQ,
> +	SLI_QENTRY_RQ,
> +	SLI_QENTRY_WQ,
> +	SLI_QENTRY_WQ_RELEASE,
> +	SLI_QENTRY_OPT_WRITE_CMD,
> +	SLI_QENTRY_OPT_WRITE_DATA,
> +	SLI_QENTRY_XABT,
> +	SLI_QENTRY_MAX			/* must be last */
> +};
> +
> +enum sli4_queue_flags {
> +	/* CQ has MQ/Async completion */
> +	SLI4_QUEUE_FLAG_MQ	= (1 << 0),
> +
> +	/* RQ for packet headers */
> +	SLI4_QUEUE_FLAG_HDR	= (1 << 1),
> +
> +	/* RQ index increment by 8 */
> +	SLI4_QUEUE_FLAG_RQBATCH	= (1 << 2),
> +};
> +
> +struct sli4_queue {
> +	/* Common to all queue types */
> +	struct efc_dma	dma;
> +	spinlock_t	lock;	/* protect the queue operations */
> +	u32	index;		/* current host entry index */
> +	u16	size;		/* entry size */
> +	u16	length;		/* number of entries */
> +	u16	n_posted;	/* number entries posted */
> +	u16	id;		/* Port assigned xQ_ID */
> +	u16	ulp;		/* ULP assigned to this queue */
> +	void __iomem    *db_regaddr;	/* register address for the doorbell */
> +	u8		type;		/* queue type ie EQ, CQ, ... */

Alignment?
Having an u8 following a pointer is a guaranteed misalignment for the 
remaining entries.
Also, consider casting it to the above qtype enum.

> +	u32	proc_limit;	/* limit CQE processed per iteration */
> +	u32	posted_limit;	/* CQE/EQE process before ring doorbel */
> +	u32	max_num_processed;
> +	time_t		max_process_time;
> +	u16	phase;		/* For if_type = 6, this value toggle
> +				 * for each iteration of the queue,
> +				 * a queue entry is valid when a cqe
> +				 * valid bit matches this value
> +				 */
> +
Same here. u16 between u32 is not making the cachline happy.

> +	union {
> +		u32	r_idx;	/* "read" index (MQ only) */
> +		struct {
> +			u32	dword;
> +		} flag;
> +	} u;
> +};
> +
> +/* Generic Command Request header */

And the remainder seems to be all hardware-dependent structures.
Would it be possible to rearrange stuff so that hardware/SLI related 
structures are being kept separate from the software/driver dependent ones?
Just so that one is clear which structures can or must mot be changed.

Cheers,

Hannes
-- 
Dr. Hannes Reinecke            Teamlead Storage & Networking
hare@suse.de                               +49 911 74053 688
SUSE Software Solutions GmbH, Maxfeldstr. 5, 90409 Nürnberg
HRB 36809 (AG Nürnberg), Geschäftsführer: Felix Imendörffer