Re: [PATCH v9 4/6] iommu/arm-smmu-v3: Add CS_NONE quirk for CONFIG_TEGRA241_CMDQV

[Nicolin Chen <nicolinc@nvidia.com>]

On Tue, Jul 02, 2024 at 07:49:42PM +0100, Will Deacon wrote:
> On Tue, Jul 02, 2024 at 11:19:56AM -0700, Nicolin Chen wrote:
> > Hi Will,
> >
> > On Tue, Jul 02, 2024 at 06:43:07PM +0100, Will Deacon wrote:
> > > On Wed, Jun 12, 2024 at 02:45:31PM -0700, Nicolin Chen wrote:
> > > > The CMDQV extension in NVIDIA Tegra241 SoC only supports CS_NONE in the
> > > > CS field of CMD_SYNC. Add a quirk flag to accommodate that.
> > > >
> > > > Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
> > > > Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
> > > > ---
> > > >  drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 8 +++++++-
> > > >  drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 3 +++
> > > >  2 files changed, 10 insertions(+), 1 deletion(-)
> > > >
> > > > diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> > > > index c864c634cd23..ba0e24d5ffbf 100644
> > > > --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> > > > +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> > > > @@ -345,6 +345,11 @@ static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
> > > >                FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH) |
> > > >                FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
> > > >
> > > > +     if (q->quirks & CMDQ_QUIRK_SYNC_CS_NONE_ONLY) {
> > > > +             cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_NONE);
> > > > +             return;
> > > > +     }
> > > > +
> > > >       if (!(smmu->options & ARM_SMMU_OPT_MSIPOLL)) {
> > > >               cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
> > > >               return;
> > > > @@ -690,7 +695,8 @@ static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
> > > >                                        struct arm_smmu_cmdq *cmdq,
> > > >                                        struct arm_smmu_ll_queue *llq)
> > > >  {
> > > > -     if (smmu->options & ARM_SMMU_OPT_MSIPOLL)
> > > > +     if (smmu->options & ARM_SMMU_OPT_MSIPOLL &&
> > > > +         !(cmdq->q.quirks & CMDQ_QUIRK_SYNC_CS_NONE_ONLY))
> > > >               return __arm_smmu_cmdq_poll_until_msi(smmu, cmdq, llq);
> > > >
> > > >       return __arm_smmu_cmdq_poll_until_consumed(smmu, cmdq, llq);
> > > > diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
> > > > index 180c0b1e0658..01227c0de290 100644
> > > > --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
> > > > +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
> > > > @@ -543,6 +543,9 @@ struct arm_smmu_queue {
> > > >
> > > >       u32 __iomem                     *prod_reg;
> > > >       u32 __iomem                     *cons_reg;
> > > > +
> > > > +#define CMDQ_QUIRK_SYNC_CS_NONE_ONLY BIT(0)  /* CMD_SYNC CS field supports CS_NONE only */
> > > > +     u32                             quirks;
> > >
> > > Please can you use the existing smmu->options field instead of adding
> > > another place to track quirks? Or do you need this only for some of the
> > > queues for a given SMMU device?
> >
> > VCMDQs are extension of a regular SMMU (with its own CMDQ). So,
> > SMMU CMDQ still supports SIG_IRQ for the CS field, while VCMDQs
> > could only support SIG_NONE. In another word, this quirk is not
> > per SMMU but per Queue.
> >
> > I can highlight this in the commit message, if that would make
> > it clear.
> 
> I think we could still use smmu->options and have something like
> ARM_SMMU_OPT_SECONDARY_CMDQ_CS_NONE_ONLY which could be applied
> when the queue is != arm_smmu_get_cmdq(smmu).

A queue can be cmdq, ecmdq, vcmdq. Only VCMDQ has such a quirk.
So arm_smmu_get_cmdq(smmu) is unlikely going to work if we add
ECMDQ later. Also, ARM_SMMU_OPT_SECONDARY_CMDQ_CS_NONE_ONLY is
very ambiguous IMHO. What we need is to check clearly if VCMDQ
is being used, so that leaves us an alternative:
--------------------------------------------------------------
enum arm_smmu_cmdq_type {
	ARM_SMMU_CMDQ,
	ARM_SMMU_ECMDQ,
	TEGRA241_VCMDQ,
};

@@ -543,6 +543,9 @@ struct arm_smmu_queue {

 	u32 __iomem			*prod_reg;
 	u32 __iomem			*cons_reg;
+
+	enum arm_smmu_queue_type	type;
 };

 struct arm_smmu_queue_poll {
@@ -345,6 +345,11 @@ static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
 		FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH) |
 		FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
 
+	if (cmdq->type == TEGRA241_VCMDQ) {
+		cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_NONE);
+		return;
+	}
+
	if (!(smmu->options & ARM_SMMU_OPT_MSIPOLL)) {
		cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
		return;
@@ -690,7 +695,8 @@ static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
 					struct arm_smmu_cmdq *cmdq,
 					struct arm_smmu_ll_queue *llq)
 {
-	if (smmu->options & ARM_SMMU_OPT_MSIPOLL)
+	if (smmu->options & ARM_SMMU_OPT_MSIPOLL &&
+	    cmdq->type != TEGRA241_VCMDQ) {
 		return __arm_smmu_cmdq_poll_until_msi(smmu, cmdq, llq);
 
--------------------------------------------------------------

Would you prefer this one? I feel CMDQ_QUIRK_SYNC_CS_NONE_ONLY
is more general looking though..

Thanks
Nicolin