[RFC PATCH 0/3] nvme-pci: dynamic interrupt moderation

[RFC PATCH 0/3] nvme-pci: dynamic interrupt moderation

[Keith Busch]

From: Keith Busch <kbusch@kernel.org>

Just adding another nvme hybrid interrupt and polling mechanism to the
lists. There's some recent alternatives mentioned in the links below,
but I'm trying one last time to leverage the DIM framework. Performance
wise, this at least doesn't seem any worse than anything else proposed,
though I've only been able to test on mid-level hardware so far.

This one here as some irq prep patches that help some when transitioning
from irq to polling for MSI-x enabled NVMe completions. These should be
generally applicable, so isn't necessarily tied to how it gets used in
the last patch if someone else finds it useful too.

Links:

  [1] https://lore.kernel.org/linux-nvme/20261009050556.2817978-1-namjain@linux.microsoft.com/

  [2] https://lore.kernel.org/linux-nvme/20260818033846.53790-1-changfengnan@bytedance.com/

Keith Busch (3):
  genirq: Add disable_irq_nowait()
  PCI/MSI: Implement irq_mask_nowait() for MSI-X
  nvme-pci: hybrid irq+polling

 drivers/irqchip/irq-msi-lib.c |   1 +
 drivers/nvme/host/Kconfig     |   2 +
 drivers/nvme/host/pci.c       | 624 ++++++++++++++++++++++++++++++++--
 drivers/nvme/host/trace.c     |   1 +
 drivers/nvme/host/trace.h     |  26 ++
 drivers/pci/msi/irqdomain.c   |   6 +
 drivers/pci/msi/msi.h         |   8 +-
 include/linux/interrupt.h     |   1 +
 include/linux/irq.h           |   4 +
 kernel/irq/chip.c             |  31 ++
 kernel/irq/internals.h        |   1 +
 kernel/irq/manage.c           |  21 ++
 12 files changed, 698 insertions(+), 28 deletions(-)

-- 
2.52.0

[RFC PATCH 1/3] genirq: Add disable_irq_nowait()

[Keith Busch]

From: Keith Busch <kbusch@kernel.org>

Drivers that switch between interrupt and polled completions don't need
to wait for the mask to reach the device; the disabled state already
keeps their handler from running and a late interrupt is replayed on
enable. Add an optional irq_chip irq_mask_nowait() callback, falling
back to irq_mask(), and disable_irq_nowait() to mask through it right
away rather than lazily.

Signed-off-by: Keith Busch <kbusch@kernel.org>
---
 include/linux/interrupt.h |  1 +
 include/linux/irq.h       |  4 ++++
 kernel/irq/chip.c         | 31 +++++++++++++++++++++++++++++++
 kernel/irq/internals.h    |  1 +
 kernel/irq/manage.c       | 21 +++++++++++++++++++++
 5 files changed, 58 insertions(+)

diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 3bf969ad8fe07..d40c992ddb8a1 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -231,6 +231,7 @@ extern void devm_free_irq(struct device *dev, unsigned int irq, void *dev_id);
 
 bool irq_has_action(unsigned int irq);
 extern void disable_irq_nosync(unsigned int irq);
+extern void disable_irq_nowait(unsigned int irq);
 extern bool disable_hardirq(unsigned int irq);
 extern void disable_irq(unsigned int irq);
 extern void disable_percpu_irq(unsigned int irq);
diff --git a/include/linux/irq.h b/include/linux/irq.h
index f485369b1b4f7..de095299f20ed 100644
--- a/include/linux/irq.h
+++ b/include/linux/irq.h
@@ -455,6 +455,9 @@ static inline irq_hw_number_t irqd_to_hwirq(struct irq_data *d)
  * @irq_disable:	disable the interrupt
  * @irq_ack:		start of a new interrupt
  * @irq_mask:		mask an interrupt source
+ * @irq_mask_nowait:	mask an interrupt source without waiting for the
+ *			hardware to acknowledge it (optional, falls back to
+ *			@irq_mask)
  * @irq_mask_ack:	ack and mask an interrupt source
  * @irq_unmask:		unmask an interrupt source
  * @irq_eoi:		end of interrupt
@@ -505,6 +508,7 @@ struct irq_chip {
 
 	void		(*irq_ack)(struct irq_data *data);
 	void		(*irq_mask)(struct irq_data *data);
+	void		(*irq_mask_nowait)(struct irq_data *data);
 	void		(*irq_mask_ack)(struct irq_data *data);
 	void		(*irq_unmask)(struct irq_data *data);
 	void		(*irq_eoi)(struct irq_data *data);
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index de754db414d1d..4e76f025b5f68 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -399,6 +399,37 @@ void irq_disable(struct irq_desc *desc)
 	__irq_disable(desc, irq_settings_disable_unlazy(desc));
 }
 
+static void mask_irq_nowait(struct irq_desc *desc)
+{
+	if (irqd_irq_masked(&desc->irq_data))
+		return;
+
+	if (desc->irq_data.chip->irq_mask_nowait) {
+		desc->irq_data.chip->irq_mask_nowait(&desc->irq_data);
+		irq_state_set_masked(desc);
+	} else {
+		mask_irq(desc);
+	}
+}
+
+/**
+ * irq_disable_nowait - Mark interrupt disabled and mask it without waiting
+ * @desc:	irq descriptor which should be disabled
+ *
+ * Like irq_disable() with lazy disable turned off, but the hardware may
+ * still deliver an interrupt after this returns. The flow handler deals
+ * with that the same way it does for a lazily disabled interrupt.
+ */
+void irq_disable_nowait(struct irq_desc *desc)
+{
+	if (desc->irq_data.chip->irq_disable) {
+		__irq_disable(desc, true);
+		return;
+	}
+	irq_state_set_disabled(desc);
+	mask_irq_nowait(desc);
+}
+
 void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
 {
 	if (desc->irq_data.chip->irq_enable)
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h
index 0ce21dd454047..3ba13932ba831 100644
--- a/kernel/irq/internals.h
+++ b/kernel/irq/internals.h
@@ -98,6 +98,7 @@ extern void irq_startup_managed(struct irq_desc *desc);
 extern void irq_shutdown(struct irq_desc *desc);
 extern void irq_shutdown_and_deactivate(struct irq_desc *desc);
 extern void irq_disable(struct irq_desc *desc);
+extern void irq_disable_nowait(struct irq_desc *desc);
 extern void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu);
 extern void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu);
 extern void mask_irq(struct irq_desc *desc);
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 57eff26fa646a..7a4d7f1024133 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -705,6 +705,27 @@ void disable_irq_nosync(unsigned int irq)
 }
 EXPORT_SYMBOL(disable_irq_nosync);
 
+/**
+ * disable_irq_nowait - mask an irq without waiting for the hardware
+ * @irq: Interrupt to disable
+ *
+ * Like disable_irq_nosync(), but masks the interrupt at the chip right away
+ * instead of lazily, and does not wait for the hardware to acknowledge the
+ * mask. An interrupt that still arrives is held pending and replayed by
+ * enable_irq(). Use this when an extra interrupt is cheaper than flushing
+ * the mask, e.g. switching from interrupts to polling.
+ *
+ * This function may be called from IRQ context.
+ */
+void disable_irq_nowait(unsigned int irq)
+{
+	scoped_irqdesc_get_and_buslock(irq, IRQ_GET_DESC_CHECK_GLOBAL) {
+		if (!scoped_irqdesc->depth++)
+			irq_disable_nowait(scoped_irqdesc);
+	}
+}
+EXPORT_SYMBOL_GPL(disable_irq_nowait);
+
 /**
  * disable_irq - disable an irq and wait for completion
  * @irq: Interrupt to disable
-- 
2.52.0

[RFC PATCH 2/3] PCI/MSI: Implement irq_mask_nowait() for MSI-x

[Keith Busch]

From: Keith Busch <kbusch@kernel.org>

Skip the read-back that flushes the MSI-x mask write. Parent domains
that mask on behalf of PCI/MSI continue using their own irq_mask().

Signed-off-by: Keith Busch <kbusch@kernel.org>
---
 drivers/irqchip/irq-msi-lib.c | 1 +
 drivers/pci/msi/irqdomain.c   | 6 ++++++
 drivers/pci/msi/msi.h         | 8 +++++++-
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/drivers/irqchip/irq-msi-lib.c b/drivers/irqchip/irq-msi-lib.c
index 45e0ed3134ce1..bff622b4839aa 100644
--- a/drivers/irqchip/irq-msi-lib.c
+++ b/drivers/irqchip/irq-msi-lib.c
@@ -126,6 +126,7 @@ bool msi_lib_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
 	 */
 	if (info->flags & MSI_FLAG_PCI_MSI_MASK_PARENT) {
 		chip->irq_mask		= irq_chip_mask_parent;
+		chip->irq_mask_nowait	= NULL;
 		chip->irq_unmask	= irq_chip_unmask_parent;
 	}
 
diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c
index 6e65f0f44112e..92a53c2772659 100644
--- a/drivers/pci/msi/irqdomain.c
+++ b/drivers/pci/msi/irqdomain.c
@@ -162,6 +162,11 @@ static void pci_irq_mask_msix(struct irq_data *data)
 	pci_msix_mask(irq_data_get_msi_desc(data));
 }
 
+static void pci_irq_mask_nowait_msix(struct irq_data *data)
+{
+	pci_msix_mask_nowait(irq_data_get_msi_desc(data));
+}
+
 static void pci_irq_unmask_msix(struct irq_data *data)
 {
 	pci_msix_unmask(irq_data_get_msi_desc(data));
@@ -182,6 +187,7 @@ static const struct msi_domain_template pci_msix_template = {
 		.irq_startup		= pci_irq_startup_msix,
 		.irq_shutdown		= pci_irq_shutdown_msix,
 		.irq_mask		= pci_irq_mask_msix,
+		.irq_mask_nowait	= pci_irq_mask_nowait_msix,
 		.irq_unmask		= pci_irq_unmask_msix,
 		.irq_write_msi_msg	= pci_msi_domain_write_msg,
 		.flags			= IRQCHIP_ONESHOT_SAFE,
diff --git a/drivers/pci/msi/msi.h b/drivers/pci/msi/msi.h
index 0b420b319f50f..337ff9d9da0ed 100644
--- a/drivers/pci/msi/msi.h
+++ b/drivers/pci/msi/msi.h
@@ -40,10 +40,16 @@ static inline void pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl)
 		writel(ctrl, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
 }
 
-static inline void pci_msix_mask(struct msi_desc *desc)
+/* Posted write only: the device may still send an interrupt after return */
+static inline void pci_msix_mask_nowait(struct msi_desc *desc)
 {
 	desc->pci.msix_ctrl |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
 	pci_msix_write_vector_ctrl(desc, desc->pci.msix_ctrl);
+}
+
+static inline void pci_msix_mask(struct msi_desc *desc)
+{
+	pci_msix_mask_nowait(desc);
 	/* Flush write to device */
 	readl(desc->pci.mask_base);
 }
-- 
2.52.0

[RFC PATCH 3/3] nvme-pci: hybrid irq+polling

[Keith Busch]

From: Keith Busch <kbusch@kernel.org>

High depth workloads can trigger interrupt storms that would be better
off polled. This proposal uses the DIM framework to adjust the
completion method from interrupt driven to polling based on the
workload. 

Signed-off-by: Keith Busch <kbusch@kernel.org>
---
 drivers/nvme/host/Kconfig |   2 +
 drivers/nvme/host/pci.c   | 624 ++++++++++++++++++++++++++++++++++++--
 drivers/nvme/host/trace.c |   1 +
 drivers/nvme/host/trace.h |  26 ++
 4 files changed, 626 insertions(+), 27 deletions(-)

diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
index 31974c7dd20c9..184e9ccfde8b2 100644
--- a/drivers/nvme/host/Kconfig
+++ b/drivers/nvme/host/Kconfig
@@ -6,6 +6,8 @@ config BLK_DEV_NVME
 	tristate "NVM Express block device"
 	depends on PCI && BLOCK
 	select NVME_CORE
+	select DIMLIB
+	select IRQ_POLL
 	help
 	  The NVM Express driver is for solid state drives directly
 	  connected to the PCI or PCI Express bus.  If you know you
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 79b3c1092e4d1..1e4dbb9b1435d 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -9,10 +9,15 @@
 #include <linux/blkdev.h>
 #include <linux/blk-mq-dma.h>
 #include <linux/blk-integrity.h>
+#include <linux/delay.h>
+#include <linux/dim.h>
 #include <linux/dmi.h>
+#include <linux/hrtimer.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
+#include <linux/irq_poll.h>
+#include <linux/ktime.h>
 #include <linux/kstrtox.h>
 #include <linux/memremap.h>
 #include <linux/mm.h>
@@ -83,6 +88,26 @@ struct quirk_entry {
 static int use_threaded_interrupts;
 module_param(use_threaded_interrupts, int, 0444);
 
+static bool use_adaptive_irq_polling;
+module_param(use_adaptive_irq_polling, bool, 0444);
+MODULE_PARM_DESC(use_adaptive_irq_polling,
+		 "use adaptive IRQ polling on dedicated I/O vectors; ignored with threaded interrupts");
+
+static unsigned int irq_poll_timeout_ms = 2;
+module_param(irq_poll_timeout_ms, uint, 0644);
+MODULE_PARM_DESC(irq_poll_timeout_ms,
+	"milliseconds the threaded handler may wait for completions before handing the queue back");
+
+static unsigned int irq_poll_sleep_max_us = 100;
+module_param(irq_poll_sleep_max_us, uint, 0644);
+MODULE_PARM_DESC(irq_poll_sleep_max_us,
+	"upper bound on the measured inter-completion poll sleep");
+
+static int irq_poll_profile = -1;
+module_param(irq_poll_profile, int, 0644);
+MODULE_PARM_DESC(irq_poll_profile,
+	"pin the moderation profile, or -1 to let it adapt");
+
 static bool use_cmb_sqes = true;
 module_param(use_cmb_sqes, bool, 0444);
 MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
@@ -284,6 +309,7 @@ struct nvme_queue;
 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
 static void nvme_delete_io_queues(struct nvme_dev *dev);
 static void nvme_update_attrs(struct nvme_dev *dev);
+static bool nvme_pci_ctrl_is_dead(struct nvme_dev *dev);
 
 struct nvme_descriptor_pools {
 	struct dma_pool *large;
@@ -309,6 +335,7 @@ struct nvme_dev {
 	void __iomem *bar;
 	unsigned long bar_mapped_size;
 	struct mutex shutdown_lock;
+	bool use_int_mask;
 	bool subsystem;
 	u64 cmb_size;
 	bool cmb_use_sqes;
@@ -387,11 +414,32 @@ struct nvme_queue {
 	u16 qid;
 	u8 cq_phase;
 	u8 sqes;
+	/* Adaptive polling knobs, seeded from the irq_poll_* module params. */
+	unsigned int poll_thresh;
+	unsigned int poll_sleep_us;	/* servo'd poll wait */
+	unsigned int poll_idle_rounds;
+	unsigned int poll_budget;
+	/* Serializes IRQ admission and the irq_poll callback tail. */
+	spinlock_t poll_lock;
+	/* Softirq poll episode state. */
+	struct hrtimer poll_timer;
+	int irq;
+	struct irq_poll iopoll;
+	unsigned long poll_deadline;
+	unsigned int poll_idle;
+	/* Adaptive interrupt moderation (DIM) sampling state. */
+	struct dim dim;
+	u16 dim_events;			/* cumulative interrupts (BIT_GAP-safe) */
+	u32 dim_comps;			/* cumulative completions */
 	unsigned long flags;
 #define NVMEQ_ENABLED		0
 #define NVMEQ_SQ_CMB		1
 #define NVMEQ_DELETE_ERROR	2
 #define NVMEQ_POLLED		3
+#define NVMEQ_POLLING		4
+#define NVMEQ_IRQ_MASKED	5
+#define NVMEQ_HYBRID		6
+#define NVMEQ_POLL_PAUSED	7
 	__le32 *dbbuf_sq_db;
 	__le32 *dbbuf_cq_db;
 	__le32 *dbbuf_sq_ei;
@@ -1838,13 +1886,13 @@ static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
 	}
 }
 
-static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
-			        struct io_comp_batch *iob)
+static inline int nvme_poll_cq_bounded(struct nvme_queue *nvmeq,
+				       struct io_comp_batch *iob, int budget)
 {
-	bool found = false;
+	int found = 0;
 
 	while (nvme_cqe_pending(nvmeq)) {
-		found = true;
+		found++;
 		/*
 		 * load-load control dependency between phase and the rest of
 		 * the cqe requires a full read memory barrier
@@ -1852,23 +1900,434 @@ static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
 		dma_rmb();
 		nvme_handle_cqe(nvmeq, iob, nvmeq->cq_head);
 		nvme_update_cq_head(nvmeq);
+		if (budget && found == budget)
+			break;
 	}
 
+	return found;
+}
+
+static inline int __nvme_poll_cq(struct nvme_queue *nvmeq,
+				 struct io_comp_batch *iob, int budget)
+{
+	int found = nvme_poll_cq_bounded(nvmeq, iob, budget);
+
 	if (found)
 		nvme_ring_cq_doorbell(nvmeq);
 	return found;
 }
 
+/* budget == 0 means reap the whole queue. */
+static int nvme_poll_cq(struct nvme_queue *nvmeq, int budget)
+{
+	DEFINE_IO_COMP_BATCH(iob);
+	int found = __nvme_poll_cq(nvmeq, &iob, budget);
+
+	if (found && !rq_list_empty(&iob.req_list))
+		nvme_pci_complete_batch(&iob);
+	nvmeq->dim_comps += found;
+	return found;
+}
+
+static inline unsigned int nvmeq_outstanding(struct nvme_queue *nvmeq)
+{
+	u16 sq_tail = READ_ONCE(nvmeq->sq_tail);
+	u16 cq_head = nvmeq->cq_head;
+
+	if (sq_tail >= cq_head)
+		return sq_tail - cq_head;
+	return nvmeq->q_depth - cq_head + sq_tail;
+}
+
+static inline bool nvmeq_cq_continue(struct nvme_queue *nvmeq)
+{
+	return nvmeq_outstanding(nvmeq) > nvmeq->poll_thresh;
+}
+
+static inline bool nvmeq_cq_idle(struct nvme_queue *nvmeq)
+{
+	return nvmeq->cq_head == READ_ONCE(nvmeq->sq_tail);
+}
+
+/*
+ * Interrupt-moderation profiles, ordered from least moderation (index 0: enter
+ * polling late, short sleep, brief linger -> lowest latency, most interrupts)
+ * to most moderation (enter early, long sleep, long linger -> fewest
+ * interrupts).  The hill-climb walks this table to keep completions-per-
+ * interrupt high, which is what bounds the interrupt rate.
+ */
+static const struct nvme_poll_prof {
+	u16 thresh;
+	u16 batch;
+	u16 idle_rounds;
+	u16 budget;
+} nvme_poll_prof[] = {
+	{ 31,  0,  1,  0 },
+	{ 15,  1,  2, 16 },
+	{  7,  2,  4,  8 },
+	{  3,  4,  8,  4 },
+	{  1,  8, 16,  2 },
+};
+
+#define NVME_DIM_START_PROFILE 2
+
+static u8 nvme_fixed_profile(void)
+{
+	int pin = READ_ONCE(irq_poll_profile);
+
+	if (pin < 0)
+		return NVME_DIM_START_PROFILE;
+	return min_t(int, pin, ARRAY_SIZE(nvme_poll_prof) - 1);
+}
+
+static void nvme_apply_profile(struct nvme_queue *nvmeq)
+{
+	const struct nvme_poll_prof *p = &nvme_poll_prof[nvmeq->dim.profile_ix];
+
+	nvmeq->poll_thresh = p->thresh;
+	nvmeq->poll_idle_rounds = p->idle_rounds;
+	nvmeq->poll_budget = p->budget;
+}
+
+static void nvme_poll_calibrate(struct nvme_queue *nvmeq, int found)
+{
+	unsigned int target = nvme_poll_prof[nvmeq->dim.profile_ix].batch;
+	unsigned int sleep = nvmeq->poll_sleep_us;
+
+	if (!target)
+		return;
+
+	target = min(target, found + nvmeq_outstanding(nvmeq) / 2);
+	if (!target)
+		return;
+
+	if (found < target)
+		sleep += max(sleep >> 2, 1U);
+	else
+		sleep -= sleep >> 2;
+
+	nvmeq->poll_sleep_us = min(sleep, irq_poll_sleep_max_us);
+}
+
+static void nvme_poll_arm(struct nvme_queue *nvmeq)
+{
+	u64 delay = (u64)max(nvmeq->poll_sleep_us, 1U) * NSEC_PER_USEC;
+
+	hrtimer_start(&nvmeq->poll_timer, ns_to_ktime(delay),
+		      HRTIMER_MODE_REL_PINNED_HARD);
+}
+
+static bool nvme_poll_accessible(struct nvme_queue *nvmeq)
+{
+	struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
+
+	return pci_is_enabled(pdev) &&
+		READ_ONCE(pdev->error_state) == pci_channel_io_normal;
+}
+
+static void nvme_poll_mask(struct nvme_queue *nvmeq)
+{
+	struct nvme_dev *dev = nvmeq->dev;
+
+	if (test_and_set_bit(NVMEQ_IRQ_MASKED, &nvmeq->flags))
+		return;
+	if (dev->use_int_mask)
+		writel(BIT(nvmeq->cq_vector), dev->bar + NVME_REG_INTMS);
+	else
+		disable_irq_nowait(nvmeq->irq);
+}
+
+static void nvme_poll_unmask(struct nvme_queue *nvmeq)
+{
+	struct nvme_dev *dev = nvmeq->dev;
+
+	if (!test_and_clear_bit(NVMEQ_IRQ_MASKED, &nvmeq->flags))
+		return;
+	if (dev->use_int_mask)
+		writel(BIT(nvmeq->cq_vector), dev->bar + NVME_REG_INTMC);
+	else
+		enable_irq(nvmeq->irq);
+}
+
+static void nvme_poll_end(struct nvme_queue *nvmeq)
+{
+	nvme_poll_unmask(nvmeq);
+	clear_bit(NVMEQ_POLLING, &nvmeq->flags);
+
+	if (nvme_cqe_pending(nvmeq) &&
+	    test_bit(NVMEQ_ENABLED, &nvmeq->flags) &&
+	    !test_and_set_bit(NVMEQ_POLLING, &nvmeq->flags)) {
+		nvme_poll_mask(nvmeq);
+		nvmeq->poll_idle = 0;
+		nvme_poll_arm(nvmeq);
+	}
+}
+
+static enum hrtimer_restart nvme_poll_timer(struct hrtimer *t)
+{
+	struct nvme_queue *nvmeq = container_of(t, struct nvme_queue, poll_timer);
+
+	if (test_bit(NVMEQ_POLLING, &nvmeq->flags))
+		irq_poll_sched(&nvmeq->iopoll);
+	return HRTIMER_NORESTART;
+}
+
+static int nvme_poll_softirq(struct irq_poll *iop, int budget)
+{
+	struct nvme_queue *nvmeq = container_of(iop, struct nvme_queue, iopoll);
+	DEFINE_IO_COMP_BATCH(iob);
+	unsigned long flags;
+	int found = 0;
+
+	spin_lock_irqsave(&nvmeq->poll_lock, flags);
+	if (unlikely(!test_bit(NVMEQ_POLLING, &nvmeq->flags)))
+		goto complete;
+	if (unlikely(test_bit(NVMEQ_POLL_PAUSED, &nvmeq->flags) ||
+		     !test_bit(NVMEQ_ENABLED, &nvmeq->flags) ||
+		     !nvme_poll_accessible(nvmeq)))
+		goto stopped;
+	spin_unlock_irqrestore(&nvmeq->poll_lock, flags);
+
+	spin_lock(&nvmeq->cq_poll_lock);
+	found = nvme_poll_cq_bounded(nvmeq, &iob, budget);
+	/* AER can freeze the channel while the CQ is being drained. */
+	if (found && nvme_poll_accessible(nvmeq))
+		nvme_ring_cq_doorbell(nvmeq);
+	spin_unlock(&nvmeq->cq_poll_lock);
+	if (!rq_list_empty(&iob.req_list))
+		nvme_pci_complete_batch(&iob);
+
+	spin_lock_irqsave(&nvmeq->poll_lock, flags);
+	nvmeq->dim_comps += found;
+	nvme_poll_calibrate(nvmeq, found);
+
+	/* A full-budget return leaves irq_poll ownership with the core. */
+	if (found >= budget)
+		goto unlock;
+
+	if (unlikely(test_bit(NVMEQ_POLL_PAUSED, &nvmeq->flags) ||
+		     !test_bit(NVMEQ_ENABLED, &nvmeq->flags) ||
+		     !nvme_poll_accessible(nvmeq)))
+		goto stopped;
+
+	/* poll_lock joins all queue accesses after releasing SCHED. */
+	irq_poll_complete(iop);
+	if (found)
+		nvmeq->poll_idle = 0;
+	else
+		nvmeq->poll_idle++;
+
+	if (nvmeq->poll_idle > nvmeq->poll_idle_rounds ||
+	    nvmeq_cq_idle(nvmeq) ||
+	    time_after_eq(jiffies, nvmeq->poll_deadline))
+		nvme_poll_end(nvmeq);
+	else
+		nvme_poll_arm(nvmeq);
+	goto unlock;
+
+stopped:
+	/* Pause or teardown retains ownership of the interrupt mask. */
+	clear_bit(NVMEQ_POLLING, &nvmeq->flags);
+complete:
+	irq_poll_complete(iop);
+unlock:
+	spin_unlock_irqrestore(&nvmeq->poll_lock, flags);
+	return found;
+}
+
+static void nvme_poll_start(struct nvme_queue *nvmeq)
+{
+	nvme_poll_mask(nvmeq);
+	nvmeq->poll_idle = 0;
+	nvmeq->poll_deadline = jiffies + msecs_to_jiffies(irq_poll_timeout_ms);
+	nvme_poll_arm(nvmeq);
+}
+
+/* Caller holds shutdown_lock to serialize pause, resume and IRQ removal. */
+static void nvme_poll_pause(struct nvme_queue *nvmeq)
+{
+	unsigned long flags;
+
+	lockdep_assert_held(&nvmeq->dev->shutdown_lock);
+	if (!test_bit(NVMEQ_HYBRID, &nvmeq->flags))
+		return;
+
+	spin_lock_irqsave(&nvmeq->poll_lock, flags);
+	if (test_and_set_bit(NVMEQ_POLL_PAUSED, &nvmeq->flags)) {
+		spin_unlock_irqrestore(&nvmeq->poll_lock, flags);
+		return;
+	}
+	if (nvme_poll_accessible(nvmeq))
+		nvme_poll_mask(nvmeq);
+	spin_unlock_irqrestore(&nvmeq->poll_lock, flags);
+
+	synchronize_irq(nvmeq->irq);
+	irq_poll_disable(&nvmeq->iopoll);
+	/* irq_poll_complete() can release SCHED before its caller returns. */
+	spin_lock_irqsave(&nvmeq->poll_lock, flags);
+	clear_bit(NVMEQ_POLLING, &nvmeq->flags);
+	spin_unlock_irqrestore(&nvmeq->poll_lock, flags);
+	hrtimer_cancel(&nvmeq->poll_timer);
+}
+
+static void nvme_poll_resume(struct nvme_queue *nvmeq)
+{
+	unsigned long flags;
+
+	lockdep_assert_held(&nvmeq->dev->shutdown_lock);
+	if (!test_bit(NVMEQ_HYBRID, &nvmeq->flags) ||
+	    !test_bit(NVMEQ_ENABLED, &nvmeq->flags))
+		return;
+
+	spin_lock_irqsave(&nvmeq->poll_lock, flags);
+	if (test_bit(NVMEQ_POLL_PAUSED, &nvmeq->flags) &&
+	    nvme_poll_accessible(nvmeq)) {
+		irq_poll_enable(&nvmeq->iopoll);
+		clear_bit(NVMEQ_POLL_PAUSED, &nvmeq->flags);
+		nvme_poll_unmask(nvmeq);
+	}
+	spin_unlock_irqrestore(&nvmeq->poll_lock, flags);
+}
+
+static void nvme_pause_io_poll(struct nvme_dev *dev)
+{
+	int i;
+
+	for (i = dev->ctrl.queue_count - 1; i > 0; i--)
+		nvme_poll_pause(&dev->queues[i]);
+}
+
+static bool nvme_resume_io_poll(struct nvme_dev *dev)
+{
+	int i;
+
+	lockdep_assert_held(&dev->shutdown_lock);
+	if (!use_adaptive_irq_polling || use_threaded_interrupts)
+		return true;
+	if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_LIVE ||
+	    nvme_pci_ctrl_is_dead(dev))
+		return false;
+	for (i = 1; i < dev->ctrl.queue_count; i++)
+		nvme_poll_resume(&dev->queues[i]);
+	return true;
+}
+
+static void nvme_poll_quiesce(struct nvme_queue *nvmeq)
+{
+	unsigned long flags;
+
+	nvme_poll_pause(nvmeq);
+	spin_lock_irqsave(&nvmeq->poll_lock, flags);
+	/* PAUSED makes a replay harmless until pci_free_irq() joins it. */
+	if (nvme_poll_accessible(nvmeq))
+		nvme_poll_unmask(nvmeq);
+	spin_unlock_irqrestore(&nvmeq->poll_lock, flags);
+	irq_poll_enable(&nvmeq->iopoll);
+}
+
+/* Mirror of rdma_dim_step() bounded to our profile table. */
+static int nvme_dim_step(struct dim *dim)
+{
+	if (dim->tune_state == DIM_GOING_RIGHT) {
+		if (dim->profile_ix == ARRAY_SIZE(nvme_poll_prof) - 1)
+			return DIM_ON_EDGE;
+		dim->profile_ix++;
+		dim->steps_right++;
+	}
+	if (dim->tune_state == DIM_GOING_LEFT) {
+		if (dim->profile_ix == 0)
+			return DIM_ON_EDGE;
+		dim->profile_ix--;
+		dim->steps_left++;
+	}
+	return DIM_STEPPED;
+}
+
+/* Mirror of rdma_dim_stats_compare(): completion rate first, then batching. */
+static int nvme_dim_stats_compare(struct dim_stats *curr, struct dim_stats *prev)
+{
+	if (!prev->cpms)
+		return DIM_STATS_SAME;
+
+	if (IS_SIGNIFICANT_DIFF(curr->cpms, prev->cpms))
+		return curr->cpms > prev->cpms ? DIM_STATS_BETTER :
+						 DIM_STATS_WORSE;
+
+	if (IS_SIGNIFICANT_DIFF(curr->cpe_ratio, prev->cpe_ratio))
+		return curr->cpe_ratio > prev->cpe_ratio ? DIM_STATS_BETTER :
+							   DIM_STATS_WORSE;
+
+	return DIM_STATS_SAME;
+}
+
+/* Mirror of rdma_dim_decision(); returns true if the profile changed. */
+static bool nvme_dim_decision(struct dim_stats *curr, struct dim *dim)
+{
+	int prev_ix = dim->profile_ix;
+	int stats_res;
+
+	stats_res = nvme_dim_stats_compare(curr, &dim->prev_stats);
+	switch (stats_res) {
+	case DIM_STATS_SAME:
+		if (curr->cpe_ratio <= 50 * prev_ix)
+			dim->profile_ix = 0;
+		break;
+	case DIM_STATS_WORSE:
+		dim_turn(dim);
+		fallthrough;
+	case DIM_STATS_BETTER:
+		if (nvme_dim_step(dim) == DIM_ON_EDGE)
+			dim_turn(dim);
+		break;
+	}
+
+	dim->prev_stats = *curr;
+	return dim->profile_ix != prev_ix;
+}
+
+static void nvme_dim(struct nvme_queue *nvmeq)
+{
+	struct dim *dim = &nvmeq->dim;
+	struct dim_sample end;
+	struct dim_stats stats;
+
+	if (dim->state == DIM_START_MEASURE) {
+		dim_update_sample_with_comps(nvmeq->dim_events, 0, 0,
+					     nvmeq->dim_comps, &dim->start_sample);
+		dim->state = DIM_MEASURE_IN_PROGRESS;
+		return;
+	}
+
+	/* Cheap gate: only recompute once a full window of events accrues. */
+	if ((u16)(nvmeq->dim_events - dim->start_sample.event_ctr) < DIM_NEVENTS)
+		return;
+
+	dim_update_sample_with_comps(nvmeq->dim_events, 0, 0, nvmeq->dim_comps,
+				     &end);
+	if (dim_calc_stats(&dim->start_sample, &end, &stats)) {
+		if (READ_ONCE(irq_poll_profile) >= 0) {
+			u8 ix = nvme_fixed_profile();
+
+			if (dim->profile_ix != ix) {
+				dim->profile_ix = ix;
+				nvme_apply_profile(nvmeq);
+			}
+		} else if (nvme_dim_decision(&stats, dim)) {
+			nvme_apply_profile(nvmeq);
+		}
+		trace_nvme_dim(nvmeq->qid, stats.cpms, stats.epms,
+			       stats.cpe_ratio, dim->profile_ix,
+			       nvmeq->poll_sleep_us);
+	}
+	dim->start_sample = end;
+}
+
 static irqreturn_t nvme_irq(int irq, void *data)
 {
 	struct nvme_queue *nvmeq = data;
-	DEFINE_IO_COMP_BATCH(iob);
 
-	if (nvme_poll_cq(nvmeq, &iob)) {
-		if (!rq_list_empty(&iob.req_list))
-			nvme_pci_complete_batch(&iob);
+	if (nvme_poll_cq(nvmeq, 0))
 		return IRQ_HANDLED;
-	}
 	return IRQ_NONE;
 }
 
@@ -1881,6 +2340,56 @@ static irqreturn_t nvme_irq_check(int irq, void *data)
 	return IRQ_NONE;
 }
 
+static irqreturn_t nvme_irq_adaptive(int irq, void *data)
+{
+	struct nvme_queue *nvmeq = data;
+	DEFINE_IO_COMP_BATCH(iob);
+	unsigned long flags;
+	irqreturn_t ret = IRQ_HANDLED;
+	bool enabled;
+	int found;
+
+	spin_lock_irqsave(&nvmeq->poll_lock, flags);
+	if (unlikely(test_bit(NVMEQ_POLL_PAUSED, &nvmeq->flags) ||
+		     !nvme_poll_accessible(nvmeq)))
+		goto unlock;
+	nvmeq->dim_events++;
+	if (test_and_set_bit(NVMEQ_POLLING, &nvmeq->flags))
+		goto unlock;
+	enabled = test_bit(NVMEQ_ENABLED, &nvmeq->flags);
+	spin_unlock_irqrestore(&nvmeq->poll_lock, flags);
+
+	found = nvme_poll_cq_bounded(nvmeq, &iob,
+				     enabled ? nvmeq->poll_budget : 0);
+	if (found && nvme_poll_accessible(nvmeq))
+		nvme_ring_cq_doorbell(nvmeq);
+	if (!rq_list_empty(&iob.req_list))
+		nvme_pci_complete_batch(&iob);
+
+	spin_lock_irqsave(&nvmeq->poll_lock, flags);
+	nvmeq->dim_comps += found;
+	if (unlikely(test_bit(NVMEQ_POLL_PAUSED, &nvmeq->flags) ||
+		     !nvme_poll_accessible(nvmeq)))
+		goto release;
+	nvme_dim(nvmeq);
+	if (!found) {
+		ret = IRQ_NONE;
+		goto release;
+	}
+
+	/* Leftover CQEs have already spent their interrupt. */
+	if (enabled && test_bit(NVMEQ_ENABLED, &nvmeq->flags) &&
+	    (nvme_cqe_pending(nvmeq) || nvmeq_cq_continue(nvmeq))) {
+		nvme_poll_start(nvmeq);
+		goto unlock;
+	}
+release:
+	clear_bit(NVMEQ_POLLING, &nvmeq->flags);
+unlock:
+	spin_unlock_irqrestore(&nvmeq->poll_lock, flags);
+	return ret;
+}
+
 /*
  * Poll for completions for any interrupt driven queue
  * Can be called from any context.
@@ -1894,24 +2403,24 @@ static void nvme_poll_irqdisable(struct nvme_queue *nvmeq)
 
 	irq = pci_irq_vector(pdev, nvmeq->cq_vector);
 	disable_irq(irq);
-	spin_lock(&nvmeq->cq_poll_lock);
-	nvme_poll_cq(nvmeq, NULL);
-	spin_unlock(&nvmeq->cq_poll_lock);
+	spin_lock_bh(&nvmeq->cq_poll_lock);
+	__nvme_poll_cq(nvmeq, NULL, 0);
+	spin_unlock_bh(&nvmeq->cq_poll_lock);
 	enable_irq(irq);
 }
 
 static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
 {
 	struct nvme_queue *nvmeq = hctx->driver_data;
-	bool found;
+	int found;
 
 	if (!test_bit(NVMEQ_POLLED, &nvmeq->flags) ||
 	    !nvme_cqe_pending(nvmeq))
 		return 0;
 
-	spin_lock(&nvmeq->cq_poll_lock);
-	found = nvme_poll_cq(nvmeq, iob);
-	spin_unlock(&nvmeq->cq_poll_lock);
+	spin_lock_bh(&nvmeq->cq_poll_lock);
+	found = __nvme_poll_cq(nvmeq, iob, 0);
+	spin_unlock_bh(&nvmeq->cq_poll_lock);
 
 	return found;
 }
@@ -1953,6 +2462,7 @@ static int nvme_pci_subsystem_reset(struct nvme_ctrl *ctrl)
 		goto unlock;
 	}
 
+	nvme_pause_io_poll(dev);
 	writel(NVME_SUBSYS_RESET, dev->bar + NVME_REG_NSSR);
 
 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING) ||
@@ -1964,6 +2474,8 @@ static int nvme_pci_subsystem_reset(struct nvme_ctrl *ctrl)
 	 * pcie read error.
 	 */
 	readl(dev->bar + NVME_REG_CSTS);
+	/* Leave polling paused if recovery has made the controller unusable. */
+	nvme_resume_io_poll(dev);
 unlock:
 	mutex_unlock(&dev->shutdown_lock);
 	return ret;
@@ -2284,8 +2796,12 @@ static void nvme_suspend_queue(struct nvme_dev *dev, unsigned int qid)
 	nvmeq->dev->online_queues--;
 	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
 		nvme_quiesce_admin_queue(&nvmeq->dev->ctrl);
-	if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags))
+	if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags)) {
+		if (test_bit(NVMEQ_HYBRID, &nvmeq->flags))
+			nvme_poll_quiesce(nvmeq);
 		pci_free_irq(to_pci_dev(dev->dev), nvmeq->cq_vector, nvmeq);
+		clear_bit(NVMEQ_HYBRID, &nvmeq->flags);
+	}
 }
 
 static void nvme_suspend_io_queues(struct nvme_dev *dev)
@@ -2307,9 +2823,10 @@ static void nvme_reap_pending_cqes(struct nvme_dev *dev)
 	int i;
 
 	for (i = dev->ctrl.queue_count - 1; i > 0; i--) {
-		spin_lock(&dev->queues[i].cq_poll_lock);
-		nvme_poll_cq(&dev->queues[i], NULL);
-		spin_unlock(&dev->queues[i].cq_poll_lock);
+		spin_lock_bh(&dev->queues[i].cq_poll_lock);
+		/* The controller is disabled; only reap CQ memory. */
+		nvme_poll_cq_bounded(&dev->queues[i], NULL, 0);
+		spin_unlock_bh(&dev->queues[i].cq_poll_lock);
 	}
 }
 
@@ -2385,6 +2902,10 @@ static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)
 	nvmeq->dev = dev;
 	spin_lock_init(&nvmeq->sq_lock);
 	spin_lock_init(&nvmeq->cq_poll_lock);
+	spin_lock_init(&nvmeq->poll_lock);
+	hrtimer_setup(&nvmeq->poll_timer, nvme_poll_timer, CLOCK_MONOTONIC,
+		      HRTIMER_MODE_REL_PINNED_HARD);
+	irq_poll_init(&nvmeq->iopoll, 64, nvme_poll_softirq);
 	nvmeq->cq_head = 0;
 	nvmeq->cq_phase = 1;
 	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
@@ -2404,14 +2925,24 @@ static int queue_request_irq(struct nvme_queue *nvmeq)
 {
 	struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
 	int nr = nvmeq->dev->ctrl.instance;
+	int ret;
 
-	if (use_threaded_interrupts) {
+	if (use_threaded_interrupts)
 		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
 				nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
-	} else {
-		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
-				NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
+
+	/* Masking requires a vector this I/O queue does not share. */
+	if (use_adaptive_irq_polling && nvmeq->qid &&
+	    nvmeq->dev->num_vecs > 1) {
+		nvmeq->irq = pci_irq_vector(pdev, nvmeq->cq_vector);
+		ret = pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_adaptive,
+				      NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
+		if (!ret)
+			set_bit(NVMEQ_HYBRID, &nvmeq->flags);
+		return ret;
 	}
+	return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
+			NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
 }
 
 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
@@ -2423,6 +2954,19 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
 	nvmeq->last_sq_tail = 0;
 	nvmeq->cq_head = 0;
 	nvmeq->cq_phase = 1;
+	memset(&nvmeq->dim, 0, sizeof(nvmeq->dim));
+	nvmeq->dim.priv = nvmeq;
+	nvmeq->dim.profile_ix = nvme_fixed_profile();
+	nvme_apply_profile(nvmeq);
+	/* No completion rate measured yet: poll without sleeping until one is. */
+	nvmeq->poll_sleep_us = 0;
+	nvmeq->dim.tune_state = DIM_GOING_RIGHT;
+	nvmeq->dim_events = 0;
+	nvmeq->dim_comps = 0;
+	clear_bit(NVMEQ_POLLING, &nvmeq->flags);
+	clear_bit(NVMEQ_IRQ_MASKED, &nvmeq->flags);
+	clear_bit(NVMEQ_HYBRID, &nvmeq->flags);
+	clear_bit(NVMEQ_POLL_PAUSED, &nvmeq->flags);
 	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
 	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq));
 	nvme_dbbuf_init(dev, nvmeq, qid);
@@ -3272,6 +3816,7 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
 		goto out_unlock;
 	}
 
+	dev->use_int_mask = !pdev->msix_enabled;
 	dev->num_vecs = result;
 	result = max(result - 1, 1);
 	dev->max_qid = result + dev->io_queues[HCTX_TYPE_POLL];
@@ -3294,10 +3839,11 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
 
 	if (dev->online_queues - 1 < dev->max_qid) {
 		nr_io_queues = dev->online_queues - 1;
-		nvme_delete_io_queues(dev);
 		result = nvme_setup_io_queues_trylock(dev);
 		if (result)
 			return result;
+		nvme_pause_io_poll(dev);
+		nvme_delete_io_queues(dev);
 		nvme_suspend_io_queues(dev);
 		goto retry;
 	}
@@ -3551,6 +4097,7 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
 	}
 
 	nvme_quiesce_io_queues(&dev->ctrl);
+	nvme_pause_io_poll(dev);
 
 	if (!dead && dev->ctrl.queue_count > 0) {
 		nvme_delete_io_queues(dev);
@@ -4175,6 +4722,7 @@ static int nvme_resume(struct device *dev)
 {
 	struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
 	struct nvme_ctrl *ctrl = &ndev->ctrl;
+	bool resumed;
 
 	if (ndev->last_ps == U32_MAX ||
 	    nvme_set_power_state(ctrl, ndev->last_ps) != 0)
@@ -4182,9 +4730,16 @@ static int nvme_resume(struct device *dev)
 	if (ctrl->hmpre && nvme_setup_host_mem(ndev))
 		goto reset;
 
+	mutex_lock(&ndev->shutdown_lock);
+	resumed = nvme_resume_io_poll(ndev);
+	mutex_unlock(&ndev->shutdown_lock);
+	if (!resumed)
+		goto reset;
 	return 0;
 reset:
-	return nvme_try_sched_reset(ctrl);
+	if (nvme_ctrl_state(ctrl) == NVME_CTRL_RESETTING)
+		return nvme_try_sched_reset(ctrl);
+	return nvme_reset_ctrl(ctrl);
 }
 
 static int nvme_suspend(struct device *dev)
@@ -4192,6 +4747,7 @@ static int nvme_suspend(struct device *dev)
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct nvme_dev *ndev = pci_get_drvdata(pdev);
 	struct nvme_ctrl *ctrl = &ndev->ctrl;
+	bool poll_paused = false;
 	int ret = -EBUSY;
 
 	ndev->last_ps = U32_MAX;
@@ -4219,8 +4775,14 @@ static int nvme_suspend(struct device *dev)
 	nvme_wait_freeze(ctrl);
 	nvme_sync_queues(ctrl);
 
-	if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
+	mutex_lock(&ndev->shutdown_lock);
+	if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) {
+		mutex_unlock(&ndev->shutdown_lock);
 		goto unfreeze;
+	}
+	nvme_pause_io_poll(ndev);
+	poll_paused = true;
+	mutex_unlock(&ndev->shutdown_lock);
 
 	/*
 	 * Host memory access may not be successful in a system suspend state,
@@ -4260,6 +4822,11 @@ static int nvme_suspend(struct device *dev)
 		ctrl->npss = 0;
 	}
 unfreeze:
+	if (ret && poll_paused) {
+		mutex_lock(&ndev->shutdown_lock);
+		nvme_resume_io_poll(ndev);
+		mutex_unlock(&ndev->shutdown_lock);
+	}
 	nvme_unfreeze(ctrl);
 	return ret;
 }
@@ -4314,6 +4881,9 @@ static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,
 	case pci_channel_io_perm_failure:
 		dev_warn(dev->ctrl.device,
 			"failure state error detected, request disconnect\n");
+		mutex_lock(&dev->shutdown_lock);
+		nvme_pause_io_poll(dev);
+		mutex_unlock(&dev->shutdown_lock);
 		return PCI_ERS_RESULT_DISCONNECT;
 	}
 	return PCI_ERS_RESULT_NEED_RESET;
diff --git a/drivers/nvme/host/trace.c b/drivers/nvme/host/trace.c
index ad25ad1e40412..4bdd5abc60880 100644
--- a/drivers/nvme/host/trace.c
+++ b/drivers/nvme/host/trace.c
@@ -498,3 +498,4 @@ const char *nvme_trace_disk_name(struct trace_seq *p, char *name)
 }
 
 EXPORT_TRACEPOINT_SYMBOL_GPL(nvme_sq);
+EXPORT_TRACEPOINT_SYMBOL_GPL(nvme_dim);
diff --git a/drivers/nvme/host/trace.h b/drivers/nvme/host/trace.h
index 4fb5922ffdac5..c7020388f9688 100644
--- a/drivers/nvme/host/trace.h
+++ b/drivers/nvme/host/trace.h
@@ -161,6 +161,32 @@ TRACE_EVENT(nvme_sq,
 	)
 );
 
+TRACE_EVENT(nvme_dim,
+	TP_PROTO(u16 qid, int cpms, int epms, int cpe_ratio, u8 profile,
+		 unsigned int sleep_us),
+	TP_ARGS(qid, cpms, epms, cpe_ratio, profile, sleep_us),
+	TP_STRUCT__entry(
+		__field(u16, qid)
+		__field(int, cpms)
+		__field(int, epms)
+		__field(int, cpe_ratio)
+		__field(u8, profile)
+		__field(unsigned int, sleep_us)
+	),
+	TP_fast_assign(
+		__entry->qid = qid;
+		__entry->cpms = cpms;
+		__entry->epms = epms;
+		__entry->cpe_ratio = cpe_ratio;
+		__entry->profile = profile;
+		__entry->sleep_us = sleep_us;
+	),
+	TP_printk("qid=%u cpms=%d epms=%d cpe_ratio=%d profile=%u sleep_us=%u",
+		__entry->qid, __entry->cpms, __entry->epms,
+		__entry->cpe_ratio, __entry->profile, __entry->sleep_us
+	)
+);
+
 #endif /* _TRACE_NVME_H */
 
 #undef TRACE_INCLUDE_PATH
-- 
2.52.0

Re: [RFC PATCH 2/3] PCI/MSI: Implement irq_mask_nowait() for MSI-x

[sashiko-bot]

> From: Keith Busch <kbusch@kernel.org>
> 
> Skip the read-back that flushes the MSI-x mask write. Parent domains
> that mask on behalf of PCI/MSI continue using their own irq_mask().
> 
> Signed-off-by: Keith Busch <kbusch@kernel.org>

Sashiko has reviewed this patch and found no issues. It looks great!

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20261013223543.395054-1-kbusch@meta.com?part=2

Re: [RFC PATCH 1/3] genirq: Add disable_irq_nowait()

[sashiko-bot]

> From: Keith Busch <kbusch@kernel.org>
> 
> Drivers that switch between interrupt and polled completions don't need
> to wait for the mask to reach the device; the disabled state already
> keeps their handler from running and a late interrupt is replayed on
> enable. Add an optional irq_chip irq_mask_nowait() callback, falling
> back to irq_mask(), and disable_irq_nowait() to mask through it right
> away rather than lazily.
> 
> Signed-off-by: Keith Busch <kbusch@kernel.org>

Sashiko has reviewed this patch and found no issues. It looks great!

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20261013223543.395054-1-kbusch@meta.com?part=1

Re: [RFC PATCH 3/3] nvme-pci: hybrid irq+polling

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] kconfig: BLK_DEV_NVME selects DIMLIB without NET dependency
--

commit a8cd25c467c1c40e704c2bd340830c637e44f89b
Author: Keith Busch <kbusch@kernel.org>

    nvme-pci: hybrid irq+polling

This commit introduces hybrid irq+polling for nvme-pci using the DIM framework
to adjust completion methods dynamically based on the workload.

> diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
> index 31974c7dd20c9..184e9ccfde8b2 100644
> --- a/drivers/nvme/host/Kconfig
> +++ b/drivers/nvme/host/Kconfig
> @@ -6,6 +6,8 @@ config BLK_DEV_NVME
>  	tristate "NVM Express block device"
>  	depends on PCI && BLOCK
>  	select NVME_CORE
> +	select DIMLIB

[Severity: Medium]
Since DIMLIB is defined in lib/Kconfig with a dependency on NET, does
unconditionally selecting it here risk Kconfig unmet direct dependency
warnings and potential build failures on configurations where CONFIG_NET is
disabled?

> +	select IRQ_POLL
>  	help
>  	  The NVM Express driver is for solid state drives directly
>  	  connected to the PCI or PCI Express bus.  If you know you

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20261013223543.395054-1-kbusch@meta.com?part=3

Re: [RFC PATCH 0/3] nvme-pci: dynamic interrupt moderation

[Fengnan]

Hi Keith:

I ran some quick tests using your patch on the PS1010 and SN861. 
For 4K random reads with QD=8, both drives showed a performance drop
of about 19%. 
With QD=64/128, there was an improvement of roughly 10–20%.

I think the reason is that the current logic still relies on the QD
depth to determine whether to switch, but a high QD doesn't necessarily
mean that using the poll mode will be more efficient.

As for DIM, I tried using it based on my V2 patch, but the result was
that I either had to add several new DIM functions to perform similar
checks, or I still ended up with a performance regression.

Here is detail data 
https://docs.google.com/spreadsheets/d/1VOR0xB5yk-z6IMwwRvp8R0yd4k_tMi_wI27Pl-dEQVg/edit?gid=789521759#gid=789521759

Based on the results, it doesn’t perform as well as my previous patch：
https://docs.google.com/spreadsheets/d/1rer4Nl5HHxSO-D3NrrHuHSfpR9s1kLhpLjzh7h7Mbmw/edit?gid=875909538#gid=875909538



在 2026/10/14 06:35, Keith Busch 写道:
> From: Keith Busch <kbusch@kernel.org>
> 
> Just adding another nvme hybrid interrupt and polling mechanism to the
> lists. There's some recent alternatives mentioned in the links below,
> but I'm trying one last time to leverage the DIM framework. Performance
> wise, this at least doesn't seem any worse than anything else proposed,
> though I've only been able to test on mid-level hardware so far.
> 
> This one here as some irq prep patches that help some when transitioning
> from irq to polling for MSI-x enabled NVMe completions. These should be
> generally applicable, so isn't necessarily tied to how it gets used in
> the last patch if someone else finds it useful too.
> 
> Links:
> 
>   [1] https://lore.kernel.org/linux-nvme/20261009050556.2817978-1-namjain@linux.microsoft.com/
> 
>   [2] https://lore.kernel.org/linux-nvme/20260818033846.53790-1-changfengnan@bytedance.com/
> 
> Keith Busch (3):
>   genirq: Add disable_irq_nowait()
>   PCI/MSI: Implement irq_mask_nowait() for MSI-X
>   nvme-pci: hybrid irq+polling
> 
>  drivers/irqchip/irq-msi-lib.c |   1 +
>  drivers/nvme/host/Kconfig     |   2 +
>  drivers/nvme/host/pci.c       | 624 ++++++++++++++++++++++++++++++++--
>  drivers/nvme/host/trace.c     |   1 +
>  drivers/nvme/host/trace.h     |  26 ++
>  drivers/pci/msi/irqdomain.c   |   6 +
>  drivers/pci/msi/msi.h         |   8 +-
>  include/linux/interrupt.h     |   1 +
>  include/linux/irq.h           |   4 +
>  kernel/irq/chip.c             |  31 ++
>  kernel/irq/internals.h        |   1 +
>  kernel/irq/manage.c           |  21 ++
>  12 files changed, 698 insertions(+), 28 deletions(-)
>

Re: [RFC PATCH 2/3] PCI/MSI: Implement irq_mask_nowait() for MSI-x

[Bjorn Helgaas]

[+cc Luigi, cover letter at
https://lore.kernel.org/all/20261013223543.395054-1-kbusch@meta.com]

On Tue, Oct 13, 2026 at 03:35:42PM -0700, Keith Busch wrote:
> From: Keith Busch <kbusch@kernel.org>
> 
> Skip the read-back that flushes the MSI-x mask write. Parent domains
> that mask on behalf of PCI/MSI continue using their own irq_mask().
> 
> Signed-off-by: Keith Busch <kbusch@kernel.org>

I assume Thomas will review/ack the pci/msi/ parts.

Cc'd Luigi since I know he's interested in interrupt moderation in
general.

> ---
>  drivers/irqchip/irq-msi-lib.c | 1 +
>  drivers/pci/msi/irqdomain.c   | 6 ++++++
>  drivers/pci/msi/msi.h         | 8 +++++++-
>  3 files changed, 14 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/irqchip/irq-msi-lib.c b/drivers/irqchip/irq-msi-lib.c
> index 45e0ed3134ce1..bff622b4839aa 100644
> --- a/drivers/irqchip/irq-msi-lib.c
> +++ b/drivers/irqchip/irq-msi-lib.c
> @@ -126,6 +126,7 @@ bool msi_lib_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
>  	 */
>  	if (info->flags & MSI_FLAG_PCI_MSI_MASK_PARENT) {
>  		chip->irq_mask		= irq_chip_mask_parent;
> +		chip->irq_mask_nowait	= NULL;
>  		chip->irq_unmask	= irq_chip_unmask_parent;
>  	}
>  
> diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c
> index 6e65f0f44112e..92a53c2772659 100644
> --- a/drivers/pci/msi/irqdomain.c
> +++ b/drivers/pci/msi/irqdomain.c
> @@ -162,6 +162,11 @@ static void pci_irq_mask_msix(struct irq_data *data)
>  	pci_msix_mask(irq_data_get_msi_desc(data));
>  }
>  
> +static void pci_irq_mask_nowait_msix(struct irq_data *data)
> +{
> +	pci_msix_mask_nowait(irq_data_get_msi_desc(data));
> +}
> +
>  static void pci_irq_unmask_msix(struct irq_data *data)
>  {
>  	pci_msix_unmask(irq_data_get_msi_desc(data));
> @@ -182,6 +187,7 @@ static const struct msi_domain_template pci_msix_template = {
>  		.irq_startup		= pci_irq_startup_msix,
>  		.irq_shutdown		= pci_irq_shutdown_msix,
>  		.irq_mask		= pci_irq_mask_msix,
> +		.irq_mask_nowait	= pci_irq_mask_nowait_msix,
>  		.irq_unmask		= pci_irq_unmask_msix,
>  		.irq_write_msi_msg	= pci_msi_domain_write_msg,
>  		.flags			= IRQCHIP_ONESHOT_SAFE,
> diff --git a/drivers/pci/msi/msi.h b/drivers/pci/msi/msi.h
> index 0b420b319f50f..337ff9d9da0ed 100644
> --- a/drivers/pci/msi/msi.h
> +++ b/drivers/pci/msi/msi.h
> @@ -40,10 +40,16 @@ static inline void pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl)
>  		writel(ctrl, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
>  }
>  
> -static inline void pci_msix_mask(struct msi_desc *desc)
> +/* Posted write only: the device may still send an interrupt after return */
> +static inline void pci_msix_mask_nowait(struct msi_desc *desc)
>  {
>  	desc->pci.msix_ctrl |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
>  	pci_msix_write_vector_ctrl(desc, desc->pci.msix_ctrl);
> +}
> +
> +static inline void pci_msix_mask(struct msi_desc *desc)
> +{
> +	pci_msix_mask_nowait(desc);
>  	/* Flush write to device */
>  	readl(desc->pci.mask_base);
>  }
> -- 
> 2.52.0
> 

Re: [RFC PATCH 2/3] PCI/MSI: Implement irq_mask_nowait() for MSI-x

[Thomas Gleixner]

On Wed, Oct 14 2026 at 12:31, Bjorn Helgaas wrote:
>> Skip the read-back that flushes the MSI-x mask write. Parent domains
>> that mask on behalf of PCI/MSI continue using their own irq_mask().
>> 
>> Signed-off-by: Keith Busch <kbusch@kernel.org>
>
> I assume Thomas will review/ack the pci/msi/ parts.

Yes. I actually suggested that "nowait" mechanism to Keith at LPC.

Though I hate the name of that callback because irq_mask() does not
really wait. It guarantees that the mask store has reached the device
which means that after that point no further interrupts will be sent by
the device (assuming sane hardware). That means it's a synchronous
operation which involves flushing the store, so maybe irq_mask_noflush()
or irq_mask_async() might be more accurate and descriptive.

> Cc'd Luigi since I know he's interested in interrupt moderation in
> general.

Correct. I'm curious whether the interrupt frequency based moderation
mechanism Luigi is proposing does solve the NVME issues as well or can
be combined with some device specific hint mechanism.

That said, Luigi's approach should benefit from the "nowait" mechanism
as well.

Luigi?

Thanks,

        tglx

Re: [RFC PATCH 2/3] PCI/MSI: Implement irq_mask_nowait() for MSI-x

[Keith Busch]

On Wed, Oct 14, 2026 at 11:10:45PM +0200, Thomas Gleixner wrote:
> On Wed, Oct 14 2026 at 12:31, Bjorn Helgaas wrote:
> >> Skip the read-back that flushes the MSI-x mask write. Parent domains
> >> that mask on behalf of PCI/MSI continue using their own irq_mask().
> >> 
> >> Signed-off-by: Keith Busch <kbusch@kernel.org>
> >
> > I assume Thomas will review/ack the pci/msi/ parts.
> 
> Yes. I actually suggested that "nowait" mechanism to Keith at LPC.
> 
> Though I hate the name of that callback because irq_mask() does not
> really wait. It guarantees that the mask store has reached the device
> which means that after that point no further interrupts will be sent by
> the device (assuming sane hardware). That means it's a synchronous
> operation which involves flushing the store, so maybe irq_mask_noflush()
> or irq_mask_async() might be more accurate and descriptive.

Right thank you, I misremembered your suggested name. I am totally fine
with "noflush", and I've made that change in my local branch. I'll just
wait a bit for any other feedback before the next version.

> > Cc'd Luigi since I know he's interested in interrupt moderation in
> > general.
> 
> Correct. I'm curious whether the interrupt frequency based moderation
> mechanism Luigi is proposing does solve the NVME issues as well or can
> be combined with some device specific hint mechanism.
> 
> That said, Luigi's approach should benefit from the "nowait" mechanism
> as well.
> 
> Luigi?
> 
> Thanks,
> 
>         tglx

Re: [RFC PATCH 2/3] PCI/MSI: Implement irq_mask_nowait() for MSI-x

[Luigi Rizzo]

On Wed, Oct 14, 2026 at 11:10 PM Thomas Gleixner <tglx@kernel.org> wrote:
>
> On Wed, Oct 14 2026 at 12:31, Bjorn Helgaas wrote:
> >> Skip the read-back that flushes the MSI-x mask write. Parent domains
> >> that mask on behalf of PCI/MSI continue using their own irq_mask().
> >>
> >> Signed-off-by: Keith Busch <kbusch@kernel.org>
> >
> > I assume Thomas will review/ack the pci/msi/ parts.
>
> Yes. I actually suggested that "nowait" mechanism to Keith at LPC.

The topic also came up around last December. The readback wastes 1000-1500ns
under load. I hacked a conditional in pci_msix_mask()
https://lore.kernel.org/all/20251220193120.3339162-1-lrizzo@google.com/
and you suggested a callback irq_mask_dev calling pci_irq_mask_msix_lazy()
https://lore.kernel.org/all/87jyxqiuta.ffs@tglx/

As usual naming is the hard part...

The mechanism is definitely useful (with moderation the mask is a frequent
operation, so saving the extra 1-2us per call is measurable).
I was a bit worried about adding yet another callback, but there are
already 30 or so,
hence one or two more does not make much of a difference.

On passing, on ARM this will not help unless we partially revert the
optimization
that skips PCIe if the interrupt has MSI_FLAG_PCI_MSI_MASK_PARENT
https://lore.kernel.org/all/20250517103011.2573288-1-maz@kernel.org/

> Though I hate the name of that callback because irq_mask() does not
> really wait. It guarantees that the mask store has reached the device
> which means that after that point no further interrupts will be sent by
> the device (assuming sane hardware). That means it's a synchronous
> operation which involves flushing the store, so maybe irq_mask_noflush()
> or irq_mask_async() might be more accurate and descriptive.
>
> > Cc'd Luigi since I know he's interested in interrupt moderation in
> > general.
>
> Correct. I'm curious whether the interrupt frequency based moderation
> mechanism Luigi is proposing does solve the NVME issues as well or can
> be combined with some device specific hint mechanism.

I really think we need both. It depends on what the issue is:

- for hardware level problems (backpressure on PCIe from the interrupt
dispatch pipeline)
  we need to actually disable interrupts at the PCIe sources, and
address the problem at
  a global SoC level, not on the single device (NICs, NVME, VFs used by VMs...).
  I honestly believe something global like GSIM is the best answer,
because the alternative
  of having all drivers implement moderation and coordinate the delays
would be hard to manage.

- for pure CPU overload due to e.g. excessive interrupts from a single source,
  GSIM can help a bit (e.g. preventing livelock, or cutting some
10-20% interrupt overhead),
  but driver specific solutions have a lot more visibility on what is happening
   and may possibly do a better job. As an example,
   Fengnan's solution not only moderates (or does timed polling, GSIM
has that too),
   but can also dynamically switch completion processing between
hardirq and softirq.

   https://lore.kernel.org/linux-nvme/20260818033846.53790-1-changfengnan@bytedance.com/

cheers
luigi

>
> That said, Luigi's approach should benefit from the "nowait" mechanism
> as well.
>
> Luigi?
>
> Thanks,
>
>         tglx

Re: [RFC PATCH 0/3] nvme-pci: dynamic interrupt moderation

[Fengnan]

[-- Attachment #1: Type: text/plain, Size: 2597 bytes --]

Hi Keith:

I made some modifications to DIM and then implemented a solution
using the DIM framework. 
Based on current testing, the results are now comparable to those of
the previous patch.
Here are the preliminary test results:
https://docs.google.com/spreadsheets/d/1e8LQPnaDhmv50XlAM3XxI0TMQkhbk0bTZwlJP1Hzzdk/edit?gid=188992110#gid=188992110

If you agree with this approach, I’ll continue testing and optimizing.

Our design uses two DIM profiles: IRQ and polling at a fixed 10 µs
interval, comparing the CQ’s own completion rate.

The patch is attached. 
It's just a proof-of-concept to prove that the DIM framework can also
achieve good results, most of the code was written by AI.
The current code logic is a bit complex and needs to be simplified a bit.


[1]
https://lore.kernel.org/linux-nvme/20260826061546.56006-1-changfengnan@bytedance.com/

Thanks.

在 14/10/26 06:35, Keith Busch 写道:
> From: Keith Busch <kbusch@kernel.org>
> 
> Just adding another nvme hybrid interrupt and polling mechanism to the
> lists. There's some recent alternatives mentioned in the links below,
> but I'm trying one last time to leverage the DIM framework. Performance
> wise, this at least doesn't seem any worse than anything else proposed,
> though I've only been able to test on mid-level hardware so far.
> 
> This one here as some irq prep patches that help some when transitioning
> from irq to polling for MSI-x enabled NVMe completions. These should be
> generally applicable, so isn't necessarily tied to how it gets used in
> the last patch if someone else finds it useful too.
> 
> Links:
> 
>   [1] https://lore.kernel.org/linux-nvme/20261009050556.2817978-1-namjain@linux.microsoft.com/
> 
>   [2] https://lore.kernel.org/linux-nvme/20260818033846.53790-1-changfengnan@bytedance.com/
> 
> Keith Busch (3):
>   genirq: Add disable_irq_nowait()
>   PCI/MSI: Implement irq_mask_nowait() for MSI-X
>   nvme-pci: hybrid irq+polling
> 
>  drivers/irqchip/irq-msi-lib.c |   1 +
>  drivers/nvme/host/Kconfig     |   2 +
>  drivers/nvme/host/pci.c       | 624 ++++++++++++++++++++++++++++++++--
>  drivers/nvme/host/trace.c     |   1 +
>  drivers/nvme/host/trace.h     |  26 ++
>  drivers/pci/msi/irqdomain.c   |   6 +
>  drivers/pci/msi/msi.h         |   8 +-
>  include/linux/interrupt.h     |   1 +
>  include/linux/irq.h           |   4 +
>  kernel/irq/chip.c             |  31 ++
>  kernel/irq/internals.h        |   1 +
>  kernel/irq/manage.c           |  21 ++
>  12 files changed, 698 insertions(+), 28 deletions(-)
>

[-- Attachment #2: 0001-lib-dim-allow-DIMLIB-without-CONFIG_NET.patch --]
[-- Type: text/plain, Size: 1352 bytes --]

From ab88b22a6f0ced47fa7ced19b24a846520e449ef Mon Sep 17 00:00:00 2001
From: Fengnan Chang <changfengnan@bytedance.com>
Date: Mon, 19 Oct 2026 11:03:45 +0000
Subject: [RFC PATCH 1/4] lib/dim: allow DIMLIB without CONFIG_NET

Only net_dim.c needs networking: it manages net_device moderation
profiles. dim.c and rdma_dim.c are generic, yet DIMLIB as a whole
depends on NET, so a block driver cannot select it on a NET=n kernel.

Drop the NET dependency from DIMLIB and build net_dim.o only when
CONFIG_NET is enabled. All net_dim users already depend on NET.

Signed-off-by: Fengnan Chang <changfengnan@bytedance.com>
---
 lib/Kconfig      | 1 -
 lib/dim/Makefile | 3 ++-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/Kconfig b/lib/Kconfig
index 55748b68714e0..af2fc199338f4 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -475,7 +475,6 @@ config SIGNATURE
 
 config DIMLIB
 	tristate
-	depends on NET
 	help
 	  Dynamic Interrupt Moderation library.
 	  Implements an algorithm for dynamically changing CQ moderation values
diff --git a/lib/dim/Makefile b/lib/dim/Makefile
index 5b9bfaac7ac1f..47a0d5fe69649 100644
--- a/lib/dim/Makefile
+++ b/lib/dim/Makefile
@@ -4,4 +4,5 @@
 
 obj-$(CONFIG_DIMLIB) += dimlib.o
 
-dimlib-y := dim.o net_dim.o rdma_dim.o
+dimlib-y := dim.o rdma_dim.o
+dimlib-$(CONFIG_NET) += net_dim.o
-- 
2.39.5


[-- Attachment #3: 0002-lib-dim-factor-the-tuning-state-machine-out-of-net_d.patch --]
[-- Type: text/plain, Size: 7676 bytes --]

From af0e78ec947217a0995cddc42fce6526b015c159 Mon Sep 17 00:00:00 2001
From: Fengnan Chang <changfengnan@bytedance.com>
Date: Mon, 19 Oct 2026 11:03:46 +0000
Subject: [RFC PATCH 2/4] lib/dim: factor the tuning state machine out of
 net_dim

net_dim_decision() walks a profile table: it steps towards the better
profile, turns when a step does not help, parks on top, and parks tired
after too many steps. Only the statistics comparison and the number of
profiles are specific to networking.

Move the state machine into dim.c as dim_decide(), which takes the
comparison as a callback and the number of profiles as an argument, so
that other drivers can reuse the DIM search with their own measurements
and profile tables. The tired limit becomes twice the number of
profiles, which is unchanged for net_dim. net_dim() now calls
dim_decide() with net_dim_stats_compare() and NET_DIM_PARAMS_NUM_PROFILES;
no functional change.

Signed-off-by: Fengnan Chang <changfengnan@bytedance.com>
---
 include/linux/dim.h | 21 +++++++++++
 lib/dim/dim.c       | 88 +++++++++++++++++++++++++++++++++++++++++++
 lib/dim/net_dim.c   | 91 ++-------------------------------------------
 3 files changed, 112 insertions(+), 88 deletions(-)

diff --git a/include/linux/dim.h b/include/linux/dim.h
index 06543fd40fccb..1a3f7cc1ddc5d 100644
--- a/include/linux/dim.h
+++ b/include/linux/dim.h
@@ -341,6 +341,27 @@ void dim_park_on_top(struct dim *dim);
  */
 void dim_park_tired(struct dim *dim);
 
+/**
+ *	dim_decide - one tuning decision on a new statistics sample
+ *	@dim: DIM context
+ *	@curr_stats: statistics of the sample just measured
+ *	@compare: callback returning %DIM_STATS_BETTER,
+ *		  %DIM_STATS_SAME or %DIM_STATS_WORSE, given @dim and both samples
+ *	@nprofiles: number of profiles, from 1 to 127
+ *
+ * Run the DIM tuning state machine used by net_dim: step towards the
+ * better profile, turn when a step does not help, park on top, and park
+ * tired after 2 * @nprofiles steps without settling. The caller serializes
+ * access to @dim and applies the new profile. profile_ix must be below
+ * @nprofiles; a tired parked context must have a nonzero tired count.
+ *
+ * Return: true if profile_ix changed.
+ */
+bool dim_decide(struct dim *dim, struct dim_stats *curr_stats,
+		int (*compare)(struct dim *dim, struct dim_stats *curr,
+			       struct dim_stats *prev),
+		u8 nprofiles);
+
 /**
  *	dim_calc_stats - calculate the difference between two samples
  *	@start: start sample
diff --git a/lib/dim/dim.c b/lib/dim/dim.c
index 97c3d084ebf02..11b0a080c019a 100644
--- a/lib/dim/dim.c
+++ b/lib/dim/dim.c
@@ -54,6 +54,94 @@ void dim_park_tired(struct dim *dim)
 }
 EXPORT_SYMBOL(dim_park_tired);
 
+static int dim_step(struct dim *dim, u8 nprofiles)
+{
+	if (dim->tired == nprofiles * 2)
+		return DIM_TOO_TIRED;
+
+	switch (dim->tune_state) {
+	case DIM_PARKING_ON_TOP:
+	case DIM_PARKING_TIRED:
+		break;
+	case DIM_GOING_RIGHT:
+		if (dim->profile_ix == nprofiles - 1)
+			return DIM_ON_EDGE;
+		dim->profile_ix++;
+		dim->steps_right++;
+		break;
+	case DIM_GOING_LEFT:
+		if (dim->profile_ix == 0)
+			return DIM_ON_EDGE;
+		dim->profile_ix--;
+		dim->steps_left++;
+		break;
+	}
+
+	dim->tired++;
+	return DIM_STEPPED;
+}
+
+static void dim_exit_parking(struct dim *dim, u8 nprofiles)
+{
+	dim->tune_state = dim->profile_ix ? DIM_GOING_LEFT : DIM_GOING_RIGHT;
+	dim_step(dim, nprofiles);
+}
+
+bool dim_decide(struct dim *dim, struct dim_stats *curr_stats,
+		int (*compare)(struct dim *dim, struct dim_stats *curr,
+			       struct dim_stats *prev),
+		u8 nprofiles)
+{
+	int prev_state = dim->tune_state;
+	int prev_ix = dim->profile_ix;
+	int stats_res;
+	int step_res;
+
+	switch (dim->tune_state) {
+	case DIM_PARKING_ON_TOP:
+		stats_res = compare(dim, curr_stats, &dim->prev_stats);
+		if (stats_res != DIM_STATS_SAME)
+			dim_exit_parking(dim, nprofiles);
+		break;
+
+	case DIM_PARKING_TIRED:
+		dim->tired--;
+		if (!dim->tired)
+			dim_exit_parking(dim, nprofiles);
+		break;
+
+	case DIM_GOING_RIGHT:
+	case DIM_GOING_LEFT:
+		stats_res = compare(dim, curr_stats, &dim->prev_stats);
+		if (stats_res != DIM_STATS_BETTER)
+			dim_turn(dim);
+
+		if (dim_on_top(dim)) {
+			dim_park_on_top(dim);
+			break;
+		}
+
+		step_res = dim_step(dim, nprofiles);
+		switch (step_res) {
+		case DIM_ON_EDGE:
+			dim_park_on_top(dim);
+			break;
+		case DIM_TOO_TIRED:
+			dim_park_tired(dim);
+			break;
+		}
+
+		break;
+	}
+
+	if (prev_state != DIM_PARKING_ON_TOP ||
+	    dim->tune_state != DIM_PARKING_ON_TOP)
+		dim->prev_stats = *curr_stats;
+
+	return dim->profile_ix != prev_ix;
+}
+EXPORT_SYMBOL(dim_decide);
+
 bool dim_calc_stats(const struct dim_sample *start,
 		    const struct dim_sample *end,
 		    struct dim_stats *curr_stats)
diff --git a/lib/dim/net_dim.c b/lib/dim/net_dim.c
index d8d4f65535596..9ab1a9ec65fd8 100644
--- a/lib/dim/net_dim.c
+++ b/lib/dim/net_dim.c
@@ -233,40 +233,7 @@ void net_dim_set_tx_mode(struct net_device *dev, u8 tx_mode)
 }
 EXPORT_SYMBOL(net_dim_set_tx_mode);
 
-static int net_dim_step(struct dim *dim)
-{
-	if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2))
-		return DIM_TOO_TIRED;
-
-	switch (dim->tune_state) {
-	case DIM_PARKING_ON_TOP:
-	case DIM_PARKING_TIRED:
-		break;
-	case DIM_GOING_RIGHT:
-		if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1))
-			return DIM_ON_EDGE;
-		dim->profile_ix++;
-		dim->steps_right++;
-		break;
-	case DIM_GOING_LEFT:
-		if (dim->profile_ix == 0)
-			return DIM_ON_EDGE;
-		dim->profile_ix--;
-		dim->steps_left++;
-		break;
-	}
-
-	dim->tired++;
-	return DIM_STEPPED;
-}
-
-static void net_dim_exit_parking(struct dim *dim)
-{
-	dim->tune_state = dim->profile_ix ? DIM_GOING_LEFT : DIM_GOING_RIGHT;
-	net_dim_step(dim);
-}
-
-static int net_dim_stats_compare(struct dim_stats *curr,
+static int net_dim_stats_compare(struct dim *dim, struct dim_stats *curr,
 				 struct dim_stats *prev)
 {
 	if (!prev->bpms)
@@ -294,59 +261,6 @@ static int net_dim_stats_compare(struct dim_stats *curr,
 	return DIM_STATS_SAME;
 }
 
-static bool net_dim_decision(struct dim_stats *curr_stats, struct dim *dim)
-{
-	int prev_state = dim->tune_state;
-	int prev_ix = dim->profile_ix;
-	int stats_res;
-	int step_res;
-
-	switch (dim->tune_state) {
-	case DIM_PARKING_ON_TOP:
-		stats_res = net_dim_stats_compare(curr_stats,
-						  &dim->prev_stats);
-		if (stats_res != DIM_STATS_SAME)
-			net_dim_exit_parking(dim);
-		break;
-
-	case DIM_PARKING_TIRED:
-		dim->tired--;
-		if (!dim->tired)
-			net_dim_exit_parking(dim);
-		break;
-
-	case DIM_GOING_RIGHT:
-	case DIM_GOING_LEFT:
-		stats_res = net_dim_stats_compare(curr_stats,
-						  &dim->prev_stats);
-		if (stats_res != DIM_STATS_BETTER)
-			dim_turn(dim);
-
-		if (dim_on_top(dim)) {
-			dim_park_on_top(dim);
-			break;
-		}
-
-		step_res = net_dim_step(dim);
-		switch (step_res) {
-		case DIM_ON_EDGE:
-			dim_park_on_top(dim);
-			break;
-		case DIM_TOO_TIRED:
-			dim_park_tired(dim);
-			break;
-		}
-
-		break;
-	}
-
-	if (prev_state != DIM_PARKING_ON_TOP ||
-	    dim->tune_state != DIM_PARKING_ON_TOP)
-		dim->prev_stats = *curr_stats;
-
-	return dim->profile_ix != prev_ix;
-}
-
 void net_dim(struct dim *dim, const struct dim_sample *end_sample)
 {
 	struct dim_stats curr_stats;
@@ -361,7 +275,8 @@ void net_dim(struct dim *dim, const struct dim_sample *end_sample)
 			break;
 		if (!dim_calc_stats(&dim->start_sample, end_sample, &curr_stats))
 			break;
-		if (net_dim_decision(&curr_stats, dim)) {
+		if (dim_decide(dim, &curr_stats, net_dim_stats_compare,
+			       NET_DIM_PARAMS_NUM_PROFILES)) {
 			dim->state = DIM_APPLY_NEW_PROFILE;
 			schedule_work(&dim->work);
 			break;
-- 
2.39.5


[-- Attachment #4: 0003-nvme-pci-return-completion-count-from-nvme_poll_cq.patch --]
[-- Type: text/plain, Size: 2381 bytes --]

From db6ef734309963486a7d9640850bb71e4bd5168a Mon Sep 17 00:00:00 2001
From: Fengnan Chang <changfengnan@bytedance.com>
Date: Wed, 19 Aug 2026 15:31:27 +0800
Subject: [RFC PATCH 3/4] nvme-pci: return completion count from nvme_poll_cq

Return the number of completions consumed by nvme_poll_cq() instead of
a boolean. Existing callers still treat zero vs non-zero as before;
nvme_poll() keeps returning 0/1 via a bool local.

Extract nvme_irq_complete_batch() from nvme_irq() so the adaptive
handler in the next patch can reuse the batch completion path.

Signed-off-by: Fengnan Chang <changfengnan@bytedance.com>
---
 drivers/nvme/host/pci.c | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 69932d640b537..620d0430601e3 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -1606,13 +1606,12 @@ static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
 	}
 }
 
-static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
-			        struct io_comp_batch *iob)
+static inline unsigned int nvme_poll_cq(struct nvme_queue *nvmeq,
+					struct io_comp_batch *iob)
 {
-	bool found = false;
+	unsigned int found = 0;
 
 	while (nvme_cqe_pending(nvmeq)) {
-		found = true;
 		/*
 		 * load-load control dependency between phase and the rest of
 		 * the cqe requires a full read memory barrier
@@ -1620,6 +1619,7 @@ static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
 		dma_rmb();
 		nvme_handle_cqe(nvmeq, iob, nvmeq->cq_head);
 		nvme_update_cq_head(nvmeq);
+		found++;
 	}
 
 	if (found)
@@ -1627,17 +1627,22 @@ static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
 	return found;
 }
 
+static irqreturn_t nvme_irq_complete_batch(struct io_comp_batch *iob,
+					   unsigned int completions)
+{
+	if (!completions)
+		return IRQ_NONE;
+	if (!rq_list_empty(&iob->req_list))
+		nvme_pci_complete_batch(iob);
+	return IRQ_HANDLED;
+}
+
 static irqreturn_t nvme_irq(int irq, void *data)
 {
 	struct nvme_queue *nvmeq = data;
 	DEFINE_IO_COMP_BATCH(iob);
 
-	if (nvme_poll_cq(nvmeq, &iob)) {
-		if (!rq_list_empty(&iob.req_list))
-			nvme_pci_complete_batch(&iob);
-		return IRQ_HANDLED;
-	}
-	return IRQ_NONE;
+	return nvme_irq_complete_batch(&iob, nvme_poll_cq(nvmeq, &iob));
 }
 
 static irqreturn_t nvme_irq_check(int irq, void *data)
-- 
2.39.5


[-- Attachment #5: 0004-nvme-pci-add-per-queue-adaptive-interrupt-polling.patch --]
[-- Type: text/plain, Size: 34955 bytes --]

From 0b70853ac7e5d9dcc3d39714e0e55729f6533cda Mon Sep 17 00:00:00 2001
From: Fengnan Chang <changfengnan@bytedance.com>
Date: Wed, 21 Oct 2026 16:32:42 +0800
Subject: [RFC PATCH 4/4] nvme-pci: add per-queue adaptive interrupt polling

At high IOPS, handling an interrupt for each completion competes with
I/O submission for CPU time. Periodic polling can amortize this work,
but it can delay completions and reduce throughput at low queue depth.

Let each eligible I/O queue use DIM to choose between interrupts and
polling every 10 us, using its own completion rate. Queues on the same
IRQ CPU share a pinned timer. The timer only schedules irq_poll work;
completion processing and DIM decisions stay under the queue lock.

Measure 4096 completions per sample and compare rates with a 3% band.
After a step, remeasure the old mode and compare the new sample against
the mean of both old samples (A/B/A), unless the difference exceeds
12%. During a search, comparable rates favor interrupts unless both
samples reach 100 completions per millisecond. This tie preference is
not an admission threshold for polling.

While parked, follow slow drift with a 1/8 EWMA and restart exploration
after three consecutive deviations in the same direction. Return to
interrupts when a polling sample stalls; reset measurements after a
physical mode change or an IRQ CPU change without disturbing peers.

Join polling after completing the interrupt batch and disabling the
Linux IRQ. Release irq_poll ownership before enabling the IRQ again,
including when a full-budget callback has to defer the transition.
Serialize queue stops with CPU offline and synchronize IRQ and poll
callbacks before freeing queue state.

The use_adaptive_irq_polling module parameter defaults to off. The
per-controller adaptive_irq_polling sysfs attribute changes the policy
at runtime after freezing namespace I/O. Only dedicated MSI-X I/O queues
with hardirq handlers on non-RT kernels and timer resolution at most
10 us are eligible. Threaded handlers remain interrupt driven.

This policy optimizes a local completion-rate estimate, not total CPU
cost or tail latency. Concurrent searches on shared CPUs can affect
the estimate, and per-queue decisions do not guarantee the best total
throughput when many queues contend for one CPU.

Signed-off-by: Fengnan Chang <changfengnan@bytedance.com>
---
 Documentation/ABI/testing/sysfs-nvme |  20 +
 drivers/nvme/host/Kconfig            |   2 +
 drivers/nvme/host/pci.c              | 642 +++++++++++++++++++++++++--
 3 files changed, 638 insertions(+), 26 deletions(-)

diff --git a/Documentation/ABI/testing/sysfs-nvme b/Documentation/ABI/testing/sysfs-nvme
index 499d5f843cd43..51b3affda6054 100644
--- a/Documentation/ABI/testing/sysfs-nvme
+++ b/Documentation/ABI/testing/sysfs-nvme
@@ -11,3 +11,23 @@ Description:
 		(REPLACETLSPSK) with the target. After a reauthentication
 		the value returned by tls_configured_key will be the new
 		serial.
+
+What:		/sys/class/nvme/nvmeX/adaptive_irq_polling
+Date:		August 2026
+KernelVersion:	7.3
+Contact:	Linux NVMe mailing list <linux-nvme@lists.infradead.org>
+Description:
+		Set the adaptive IRQ polling policy (0 or 1) for eligible
+		I/O queues of one PCI NVMe controller. Reading returns the
+		policy, not whether a queue is currently polling. Changing it
+		freezes the namespace request queues and waits for outstanding
+		namespace I/O. Writes fail with EBUSY unless the controller is
+		live.
+
+		Eligible queues use dedicated MSI-X vectors on non-RT kernels,
+		with handlers running in hardirq context and timer resolution
+		at most 10 us. Forced-threaded handlers (threadirqs) stay in
+		IRQ mode. The attribute is unavailable on PREEMPT_RT kernels
+		or with use_threaded_interrupts; otherwise it reports policy
+		even when no queue is eligible. The module parameter supplies
+		only the initial policy for new controllers.
diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
index 31974c7dd20c9..4c0bad4c63e83 100644
--- a/drivers/nvme/host/Kconfig
+++ b/drivers/nvme/host/Kconfig
@@ -5,6 +5,8 @@ config NVME_CORE
 config BLK_DEV_NVME
 	tristate "NVM Express block device"
 	depends on PCI && BLOCK
+	select DIMLIB
+	select IRQ_POLL
 	select NVME_CORE
 	help
 	  The NVM Express driver is for solid state drives directly
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 620d0430601e3..7e0bfeda79f86 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -9,10 +9,15 @@
 #include <linux/blkdev.h>
 #include <linux/blk-mq-dma.h>
 #include <linux/blk-integrity.h>
+#include <linux/cpuhotplug.h>
+#include <linux/dim.h>
 #include <linux/dmi.h>
+#include <linux/hrtimer.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
+#include <linux/irq_poll.h>
+#include <linux/ktime.h>
 #include <linux/kstrtox.h>
 #include <linux/memremap.h>
 #include <linux/mm.h>
@@ -21,6 +26,7 @@
 #include <linux/nodemask.h>
 #include <linux/once.h>
 #include <linux/pci.h>
+#include <linux/percpu.h>
 #include <linux/suspend.h>
 #include <linux/t10-pi.h>
 #include <linux/types.h>
@@ -82,6 +88,11 @@ struct quirk_entry {
 static int use_threaded_interrupts;
 module_param(use_threaded_interrupts, int, 0444);
 
+static bool use_adaptive_irq_polling;
+module_param(use_adaptive_irq_polling, bool, 0444);
+MODULE_PARM_DESC(use_adaptive_irq_polling,
+		 "default adaptive polling policy for eligible I/O queues");
+
 static bool use_cmb_sqes = true;
 module_param(use_cmb_sqes, bool, 0444);
 MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
@@ -280,6 +291,7 @@ struct nvme_dev;
 struct nvme_queue;
 
 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
+static void nvme_adaptive_stop(struct nvme_dev *dev, bool stop);
 static void nvme_delete_io_queues(struct nvme_dev *dev);
 static void nvme_update_attrs(struct nvme_dev *dev);
 
@@ -307,6 +319,8 @@ struct nvme_dev {
 	void __iomem *bar;
 	unsigned long bar_mapped_size;
 	struct mutex shutdown_lock;
+	bool adaptive_irq_polling;
+	bool adaptive_stopped;
 	bool subsystem;
 	u64 cmb_size;
 	bool cmb_use_sqes;
@@ -358,6 +372,79 @@ static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
 	return container_of(ctrl, struct nvme_dev, ctrl);
 }
 
+/*
+ * At high IOPS, IRQ handling competes with I/O submission for CPU time.
+ * Each non-threaded I/O queue with dedicated MSI-X runs its own DIM
+ * context over two profiles: IRQ and polling every 10 us. Queues sharing
+ * an IRQ CPU share one timer, but measure and choose their modes independently.
+ *
+ * A sample is the queue's completion rate over 4096 CQEs;
+ * dim_decide() steps towards the better profile,
+ * turns when a step does not help, parks on the best one, explores again
+ * when the parked rate changes, and parks tired after too many steps. A
+ * polling sample that exceeds its time limit returns to IRQ without
+ * waiting for CQEs. Samples longer than 1 s reset the DIM context. Within
+ * a search, comparable rates favor IRQ unless both samples reach 100
+ * completions per ms, when they favor polling. After each step the
+ * old profile is measured again, and the step is judged against the mean
+ * of both old samples (A/B/A) to reduce drift bias; a change beyond four
+ * times the threshold needs no recheck. A parked context
+ * follows slow drift and explores again only after three consecutive
+ * samples deviate the same way. Restart measurement on physical mode
+ * changes, so draining polls do not contaminate the IRQ sample.
+ *
+ * Queues join lazily at their IRQ tail after completing the batch and
+ * disabling the Linux IRQ. One pinned HARD timer per CPU
+ * schedules all members each period; a budget of 64 bounds each poll.
+ * Each queue releases irq_poll ownership in its own callback before
+ * leaving and enabling its IRQ; full-budget callbacks defer exit.
+ *
+ * cq_poll_lock protects queue ownership and DIM; the raw lock protects
+ * the CPU's member list and timer state.
+ * Lock order: CQ lock -> group lock. Never take a CQ lock or complete I/O
+ * under the raw lock. A queue cannot rejoin any group until the previous
+ * scheduling ownership is released, even if irq_poll moves to another CPU.
+ * nvme_adaptive_mutex serializes sleeping stops and CPU offline's member
+ * references. Stops synchronize the IRQ, detach, disable irq_poll and
+ * wait for the callback to release cq_poll_lock before proceeding.
+ */
+#define NVME_ADAPTIVE_POLL_PERIOD_NS	(10U * NSEC_PER_USEC)
+#define NVME_ADAPTIVE_SAMPLE_CQES	4096U
+#define NVME_ADAPTIVE_SAMPLE_MAX_NS	NSEC_PER_SEC
+#define NVME_ADAPTIVE_DIM_PCT		3
+#define NVME_ADAPTIVE_POLL_MIN_CPMS	100
+#define NVME_ADAPTIVE_DRIFT_SAMPLES	3
+#define NVME_ADAPTIVE_POLL_BUDGET		64
+
+#define NVME_ADAPTIVE_PROFILES		2
+
+struct nvme_adaptive_poll {
+	struct irq_poll iopoll;
+	struct nvme_queue *nvmeq;
+	struct list_head node;
+	struct dim dim;
+	struct dim_stats probe;
+	u32 cqes;
+	s8 drift;		/* consecutive parked deviations, signed */
+	bool stepped;
+	bool recheck;
+	int cpu;		/* last IRQ CPU; timer's CPU while polling */
+	int irq;
+	bool exit_pending;
+};
+
+struct nvme_adaptive_cpu {
+	raw_spinlock_t lock;
+	struct list_head members;
+	struct hrtimer timer;
+	bool timer_armed;
+	bool online;
+};
+
+static DEFINE_PER_CPU(struct nvme_adaptive_cpu, nvme_adaptive_cpus);
+static DEFINE_MUTEX(nvme_adaptive_mutex);
+static int nvme_adaptive_cpuhp_state;
+
 /*
  * An NVM Express queue.  Each device has at least two (one for admin
  * commands and one for I/O commands).
@@ -367,7 +454,8 @@ struct nvme_queue {
 	struct nvme_descriptor_pools descriptor_pools;
 	spinlock_t sq_lock;
 	void *sq_cmds;
-	 /* only used for poll queues: */
+	struct nvme_adaptive_poll *adaptive;
+	/* Used for poll queues and adaptive interrupt polling. */
 	spinlock_t cq_poll_lock ____cacheline_aligned_in_smp;
 	struct nvme_completion *cqes;
 	dma_addr_t sq_dma_addr;
@@ -386,6 +474,9 @@ struct nvme_queue {
 #define NVMEQ_SQ_CMB		1
 #define NVMEQ_DELETE_ERROR	2
 #define NVMEQ_POLLED		3
+#define NVMEQ_ADAPTIVE_POLLING	4 /* member with IRQ disabled */
+#define NVMEQ_ADAPTIVE_ENABLED	5 /* queue's adaptive policy */
+#define NVMEQ_ADAPTIVE_STALE_IRQ	6 /* swallow one stale IRQ after polling */
 	__le32 *dbbuf_sq_db;
 	__le32 *dbbuf_cq_db;
 	__le32 *dbbuf_sq_ei;
@@ -1606,12 +1697,13 @@ static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
 	}
 }
 
-static inline unsigned int nvme_poll_cq(struct nvme_queue *nvmeq,
-					struct io_comp_batch *iob)
+/* A constant zero limit lets IRQ callers compile out the budget check. */
+static __always_inline unsigned int nvme_poll_cq(struct nvme_queue *nvmeq,
+						 struct io_comp_batch *iob, unsigned int limit)
 {
 	unsigned int found = 0;
 
-	while (nvme_cqe_pending(nvmeq)) {
+	while ((!limit || found < limit) && nvme_cqe_pending(nvmeq)) {
 		/*
 		 * load-load control dependency between phase and the rest of
 		 * the cqe requires a full read memory barrier
@@ -1642,7 +1734,7 @@ static irqreturn_t nvme_irq(int irq, void *data)
 	struct nvme_queue *nvmeq = data;
 	DEFINE_IO_COMP_BATCH(iob);
 
-	return nvme_irq_complete_batch(&iob, nvme_poll_cq(nvmeq, &iob));
+	return nvme_irq_complete_batch(&iob, nvme_poll_cq(nvmeq, &iob, 0));
 }
 
 static irqreturn_t nvme_irq_check(int irq, void *data)
@@ -1654,37 +1746,298 @@ static irqreturn_t nvme_irq_check(int irq, void *data)
 	return IRQ_NONE;
 }
 
-/*
- * Poll for completions for any interrupt driven queue
- * Can be called from any context.
- */
+/* CQ lock held; profile zero returns this queue to IRQ. */
+static void nvme_adaptive_reset(struct nvme_adaptive_poll *adaptive)
+{
+	memset(&adaptive->dim, 0, sizeof(adaptive->dim));
+	adaptive->drift = 0;
+	adaptive->stepped = false;
+	adaptive->recheck = false;
+}
+
+static int nvme_adaptive_compare(struct dim *dim, struct dim_stats *curr,
+				 struct dim_stats *prev)
+{
+	struct nvme_adaptive_poll *adaptive =
+		container_of(dim, struct nvme_adaptive_poll, dim);
+	int delta = curr->cpms - prev->cpms;
+	int res;
+
+	if (!prev->cpms)
+		return curr->cpms ? DIM_STATS_BETTER : DIM_STATS_SAME;
+	if (abs(delta) * 100 <= prev->cpms * NVME_ADAPTIVE_DIM_PCT) {
+		if (dim->tune_state == DIM_GOING_LEFT ||
+		    dim->tune_state == DIM_GOING_RIGHT) {
+			bool prefer_poll = min(curr->cpms, prev->cpms) >=
+				NVME_ADAPTIVE_POLL_MIN_CPMS;
+
+			return dim->profile_ix == prefer_poll ?
+				DIM_STATS_BETTER : DIM_STATS_WORSE;
+		}
+		adaptive->drift = 0;
+		return DIM_STATS_SAME;
+	}
+	res = delta > 0 ? DIM_STATS_BETTER : DIM_STATS_WORSE;
+	if (dim->tune_state != DIM_PARKING_ON_TOP)
+		return res;
+	/* Leave parking only on a change that persists, not on noise. */
+	if (delta * adaptive->drift < 0)
+		adaptive->drift = 0;
+	adaptive->drift += delta > 0 ? 1 : -1;
+	if (abs(adaptive->drift) < NVME_ADAPTIVE_DRIFT_SAMPLES)
+		return DIM_STATS_SAME;
+	adaptive->drift = 0;
+	return res;
+}
+
+/* CQ lock held; ends this queue's sample and lets DIM decide. */
+static void nvme_adaptive_sample(struct nvme_adaptive_poll *adaptive,
+				 const struct dim_sample *end)
+{
+	struct dim *dim = &adaptive->dim;
+	struct dim_stats stats;
+
+	if (!dim_calc_stats(&dim->start_sample, end, &stats))
+		goto out;
+	/*
+	 * After a step, measure the old profile again and judge A/B/A.
+	 * Skip the recheck for differences larger than four thresholds.
+	 */
+	if (adaptive->stepped &&
+	    abs(stats.cpms - dim->prev_stats.cpms) * 100 <=
+	    dim->prev_stats.cpms * 4 * NVME_ADAPTIVE_DIM_PCT) {
+		adaptive->stepped = false;
+		adaptive->probe = stats;
+		dim->profile_ix = !dim->profile_ix;
+		adaptive->recheck = true;
+		goto out;
+	}
+	if (adaptive->recheck) {
+		adaptive->recheck = false;
+		dim->prev_stats.cpms = (dim->prev_stats.cpms + stats.cpms) / 2;
+		dim->profile_ix = !dim->profile_ix;
+		stats = adaptive->probe;
+	}
+	adaptive->stepped = dim_decide(dim, &stats, nvme_adaptive_compare,
+				       NVME_ADAPTIVE_PROFILES);
+	/* While parked, let the reference follow slow drift. */
+	if (dim->tune_state == DIM_PARKING_ON_TOP && !adaptive->stepped)
+		dim->prev_stats.cpms += (stats.cpms - dim->prev_stats.cpms) / 8;
+out:
+	dim->start_sample = *end;
+}
+
+/* CQ lock held, after completing the batch; polling is the physical mode. */
+static void nvme_adaptive_account(struct nvme_adaptive_poll *adaptive,
+				  unsigned int completions, bool polling)
+{
+	struct dim *dim = &adaptive->dim;
+	struct dim_sample end;
+	u64 elapsed_us;
+
+	/* Finish leaving poll before measuring the next IRQ profile. */
+	if (polling && !dim->profile_ix) {
+		dim->state = DIM_START_MEASURE;
+		return;
+	}
+	adaptive->cqes += completions;
+	dim_update_sample_with_comps(0, 0, 0, adaptive->cqes, &end);
+	if (dim->state == DIM_START_MEASURE) {
+		dim->start_sample = end;
+		dim->state = DIM_MEASURE_IN_PROGRESS;
+		return;
+	}
+	elapsed_us = ktime_us_delta(end.time, dim->start_sample.time);
+	/* Check stalled polls even when this callback found no completions. */
+	if (elapsed_us > NVME_ADAPTIVE_SAMPLE_MAX_NS / NSEC_PER_USEC ||
+	    (polling && (!dim->prev_stats.cpms ||
+	     elapsed_us * dim->prev_stats.cpms >
+		2ULL * NVME_ADAPTIVE_SAMPLE_CQES * USEC_PER_MSEC))) {
+		nvme_adaptive_reset(adaptive);
+		return;
+	}
+	if (adaptive->cqes - dim->start_sample.comp_ctr >= NVME_ADAPTIVE_SAMPLE_CQES)
+		nvme_adaptive_sample(adaptive, &end);
+}
+
+/* Hard timer callback; acquires only group->lock. */
+static enum hrtimer_restart nvme_adaptive_poll_timer(struct hrtimer *timer)
+{
+	struct nvme_adaptive_cpu *group =
+		container_of(timer, struct nvme_adaptive_cpu, timer);
+	struct nvme_adaptive_poll *adaptive;
+	unsigned long flags;
+	bool again;
+
+	raw_spin_lock_irqsave(&group->lock, flags);
+	list_for_each_entry(adaptive, &group->members, node)
+		irq_poll_sched(&adaptive->iopoll);
+	again = group->online && !list_empty(&group->members);
+	group->timer_armed = again;
+	if (again)
+		hrtimer_forward_now(timer, ns_to_ktime(NVME_ADAPTIVE_POLL_PERIOD_NS));
+	raw_spin_unlock_irqrestore(&group->lock, flags);
+	return again ? HRTIMER_RESTART : HRTIMER_NORESTART;
+}
+
+/* Hardirq/softirq/task context; CQ lock held, acquires the group lock. */
+static void nvme_adaptive_detach(struct nvme_queue *nvmeq)
+{
+	struct nvme_adaptive_poll *adaptive = nvmeq->adaptive;
+	struct nvme_adaptive_cpu *group;
+	unsigned long flags;
+
+	lockdep_assert_held(&nvmeq->cq_poll_lock);
+	if (!test_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags))
+		return;
+	group = per_cpu_ptr(&nvme_adaptive_cpus, adaptive->cpu);
+	raw_spin_lock_irqsave(&group->lock, flags);
+	list_del_init(&adaptive->node);
+	clear_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags);
+	adaptive->dim.state = DIM_START_MEASURE;
+	set_bit(NVMEQ_ADAPTIVE_STALE_IRQ, &nvmeq->flags);
+	raw_spin_unlock_irqrestore(&group->lock, flags);
+	enable_irq(adaptive->irq);
+}
+
+/* IRQ_POLL softirq; no locks held on entry, takes CQ then group locks. */
+static int nvme_adaptive_irq_poll(struct irq_poll *iop, int budget)
+{
+	struct nvme_adaptive_poll *adaptive =
+		container_of(iop, struct nvme_adaptive_poll, iopoll);
+	struct nvme_queue *nvmeq = adaptive->nvmeq;
+	struct nvme_adaptive_cpu *group;
+	unsigned int found = 0;
+	unsigned long flags, cpu_flags;
+	bool exit;
+	DEFINE_IO_COMP_BATCH(iob);
+
+	spin_lock_irqsave(&nvmeq->cq_poll_lock, flags);
+	if (!test_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags)) {
+		irq_poll_complete(iop);
+		goto out;
+	}
+	if (!adaptive->exit_pending && test_bit(NVMEQ_ENABLED, &nvmeq->flags))
+		found = nvme_poll_cq(nvmeq, &iob, budget);
+	if (!rq_list_empty(&iob.req_list))
+		nvme_pci_complete_batch(&iob);
+	nvme_adaptive_account(adaptive, found, true);
+	group = per_cpu_ptr(&nvme_adaptive_cpus, adaptive->cpu);
+	raw_spin_lock_irqsave(&group->lock, cpu_flags);
+	exit = !group->online || !adaptive->dim.profile_ix ||
+		!test_bit(NVMEQ_ENABLED, &nvmeq->flags);
+	if (found == budget)
+		adaptive->exit_pending = exit;
+	else
+		irq_poll_complete(iop);
+	raw_spin_unlock_irqrestore(&group->lock, cpu_flags);
+	if (exit && found < budget)
+		nvme_adaptive_detach(nvmeq);
+out:
+	spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags);
+	return found;
+}
+
+/* Hardirq; no locks held, takes CQ then group locks. */
+static noinline irqreturn_t nvme_adaptive_irq_enabled(int irq, void *data)
+{
+	struct nvme_queue *nvmeq = data;
+	struct nvme_adaptive_poll *adaptive;
+	struct nvme_adaptive_cpu *group;
+	unsigned int completions;
+	unsigned long flags, cpu_flags;
+	irqreturn_t ret = IRQ_HANDLED;
+	DEFINE_IO_COMP_BATCH(iob);
+
+	spin_lock_irqsave(&nvmeq->cq_poll_lock, flags);
+	if (unlikely(test_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags)))
+		goto out;
+	completions = nvme_poll_cq(nvmeq, &iob, 0);
+	ret = nvme_irq_complete_batch(&iob, completions);
+	adaptive = nvmeq->adaptive;
+	group = this_cpu_ptr(&nvme_adaptive_cpus);
+	if (!test_bit(NVMEQ_ENABLED, &nvmeq->flags))
+		goto out;
+	if (adaptive->cpu != smp_processor_id()) {
+		nvme_adaptive_reset(adaptive);
+		adaptive->cpu = smp_processor_id();
+	}
+	nvme_adaptive_account(adaptive, completions, false);
+	raw_spin_lock_irqsave(&group->lock, cpu_flags);
+	if (group->online && adaptive->dim.profile_ix &&
+	    !test_bit(IRQ_POLL_F_SCHED, &adaptive->iopoll.state)) {
+		disable_irq_nosync(adaptive->irq);
+		adaptive->cpu = smp_processor_id();
+		adaptive->exit_pending = false;
+		adaptive->dim.state = DIM_START_MEASURE;
+		list_add_tail(&adaptive->node, &group->members);
+		set_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags);
+		if (!group->timer_armed) {
+			group->timer_armed = true;
+			hrtimer_start(&group->timer,
+				      ns_to_ktime(ktime_get_ns() +
+						  NVME_ADAPTIVE_POLL_PERIOD_NS),
+				      HRTIMER_MODE_ABS_PINNED_HARD);
+		}
+	}
+	raw_spin_unlock_irqrestore(&group->lock, cpu_flags);
+out:
+	spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags);
+	return ret;
+}
+
+/* Hardirq or forced IRQ thread; no locks held on entry. */
+static irqreturn_t nvme_adaptive_irq(int irq, void *data)
+{
+	struct nvme_queue *nvmeq = data;
+	irqreturn_t ret;
+
+	if (test_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags) &&
+	    !IS_ENABLED(CONFIG_PREEMPT_RT) && in_hardirq() &&
+	    hrtimer_resolution <= NVME_ADAPTIVE_POLL_PERIOD_NS)
+		ret = nvme_adaptive_irq_enabled(irq, data);
+	else
+		ret = nvme_irq(irq, data);
+	if (ret == IRQ_NONE &&
+	    test_and_clear_bit(NVMEQ_ADAPTIVE_STALE_IRQ, &nvmeq->flags))
+		return IRQ_HANDLED;
+	return ret;
+}
+
+/* Timeout task context; synchronizes the IRQ, takes CQ then group locks. */
 static void nvme_poll_irqdisable(struct nvme_queue *nvmeq)
 {
 	struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
+	unsigned long flags;
 	int irq;
 
 	WARN_ON_ONCE(test_bit(NVMEQ_POLLED, &nvmeq->flags));
 
 	irq = pci_irq_vector(pdev, nvmeq->cq_vector);
 	disable_irq(irq);
-	spin_lock(&nvmeq->cq_poll_lock);
-	nvme_poll_cq(nvmeq, NULL);
-	spin_unlock(&nvmeq->cq_poll_lock);
+	spin_lock_irqsave(&nvmeq->cq_poll_lock, flags);
+	nvme_adaptive_detach(nvmeq);
+	if (nvmeq->adaptive)
+		nvme_adaptive_reset(nvmeq->adaptive);
+	nvme_poll_cq(nvmeq, NULL, 0);
+	spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags);
 	enable_irq(irq);
 }
 
 static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
 {
 	struct nvme_queue *nvmeq = hctx->driver_data;
+	unsigned long flags;
 	bool found;
 
 	if (!test_bit(NVMEQ_POLLED, &nvmeq->flags) ||
 	    !nvme_cqe_pending(nvmeq))
 		return 0;
 
-	spin_lock(&nvmeq->cq_poll_lock);
-	found = nvme_poll_cq(nvmeq, iob);
-	spin_unlock(&nvmeq->cq_poll_lock);
+	/* Match the adaptive hardirq handler's lock class. */
+	spin_lock_irqsave(&nvmeq->cq_poll_lock, flags);
+	found = nvme_poll_cq(nvmeq, iob, 0);
+	spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags);
 
 	return found;
 }
@@ -1726,6 +2079,8 @@ static int nvme_pci_subsystem_reset(struct nvme_ctrl *ctrl)
 		goto unlock;
 	}
 
+	/* TODO: stop adaptive polling on the other subsystem controllers. */
+	nvme_adaptive_stop(dev, true);
 	writel(NVME_SUBSYS_RESET, dev->bar + NVME_REG_NSSR);
 
 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING) ||
@@ -2022,8 +2377,7 @@ static void nvme_free_queue(struct nvme_queue *nvmeq)
 	dma_free_coherent(nvmeq->dev->dev, CQ_SIZE(nvmeq),
 				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);
 	if (!nvmeq->sq_cmds)
-		return;
-
+		goto free_adaptive;
 	if (test_and_clear_bit(NVMEQ_SQ_CMB, &nvmeq->flags)) {
 		pci_free_p2pmem(to_pci_dev(nvmeq->dev->dev),
 				nvmeq->sq_cmds, SQ_SIZE(nvmeq));
@@ -2031,6 +2385,9 @@ static void nvme_free_queue(struct nvme_queue *nvmeq)
 		dma_free_coherent(nvmeq->dev->dev, SQ_SIZE(nvmeq),
 				nvmeq->sq_cmds, nvmeq->sq_dma_addr);
 	}
+free_adaptive:
+	kfree(nvmeq->adaptive);
+	nvmeq->adaptive = NULL;
 }
 
 static void nvme_free_queues(struct nvme_dev *dev, int lowest)
@@ -2043,6 +2400,117 @@ static void nvme_free_queues(struct nvme_dev *dev, int lowest)
 	}
 }
 
+/* Sleepable; nvme_adaptive_mutex held, takes CQ then group locks. */
+static void nvme_adaptive_set_queue(struct nvme_queue *nvmeq, bool enable)
+{
+	struct nvme_adaptive_poll *adaptive = nvmeq->adaptive;
+	unsigned long flags;
+
+	lockdep_assert_held(&nvme_adaptive_mutex);
+	if (!adaptive || adaptive->irq < 0) {
+		clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
+		return;
+	}
+	disable_irq(adaptive->irq);
+	clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
+	spin_lock_irqsave(&nvmeq->cq_poll_lock, flags);
+	nvme_adaptive_detach(nvmeq);
+	nvme_adaptive_reset(adaptive);
+	adaptive->cpu = -1;
+	spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags);
+	irq_poll_disable(&adaptive->iopoll);
+	/* irq_poll_complete() can run before the poll callback returns. */
+	spin_lock_irqsave(&nvmeq->cq_poll_lock, flags);
+	spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags);
+	irq_poll_enable(&adaptive->iopoll);
+	if (enable)
+		set_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
+	enable_irq(adaptive->irq);
+}
+
+/* shutdown_lock held; stop synchronously, or restore policy if !stop. */
+static void nvme_adaptive_stop(struct nvme_dev *dev, bool stop)
+{
+	int qid;
+
+	lockdep_assert_held(&dev->shutdown_lock);
+	dev->adaptive_stopped = stop;
+	mutex_lock(&nvme_adaptive_mutex);
+	for (qid = 1; qid < dev->ctrl.queue_count; qid++)
+		nvme_adaptive_set_queue(&dev->queues[qid],
+					!stop && dev->adaptive_irq_polling);
+	mutex_unlock(&nvme_adaptive_mutex);
+}
+
+/* Sleepable hotplug callback; no driver locks held, takes group->lock. */
+static int nvme_adaptive_cpu_online(unsigned int cpu)
+{
+	struct nvme_adaptive_cpu *group = per_cpu_ptr(&nvme_adaptive_cpus, cpu);
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&group->lock, flags);
+	group->online = true;
+	raw_spin_unlock_irqrestore(&group->lock, flags);
+	return 0;
+}
+
+/* Sleepable hotplug callback; no driver locks held, serializes queue stops. */
+static int nvme_adaptive_cpu_offline(unsigned int cpu)
+{
+	struct nvme_adaptive_cpu *group = per_cpu_ptr(&nvme_adaptive_cpus, cpu);
+	struct nvme_adaptive_poll *adaptive;
+	unsigned long flags;
+
+	mutex_lock(&nvme_adaptive_mutex);
+	raw_spin_lock_irqsave(&group->lock, flags);
+	group->online = false;
+	raw_spin_unlock_irqrestore(&group->lock, flags);
+	hrtimer_cancel(&group->timer);
+	for (;;) {
+		raw_spin_lock_irqsave(&group->lock, flags);
+		group->timer_armed = false;
+		adaptive = list_first_entry_or_null(&group->members,
+						    struct nvme_adaptive_poll, node);
+		raw_spin_unlock_irqrestore(&group->lock, flags);
+		if (!adaptive)
+			break;
+		nvme_adaptive_set_queue(adaptive->nvmeq, true);
+	}
+	mutex_unlock(&nvme_adaptive_mutex);
+	return 0;
+}
+
+/* Sleepable sysfs path; no locks held, freezes I/O before shutdown_lock. */
+static int nvme_adaptive_switch(struct nvme_dev *dev, bool enable)
+{
+	int ret = 0;
+
+	mutex_lock(&dev->ctrl.scan_lock);
+	if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_LIVE) {
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+	if (enable == READ_ONCE(dev->adaptive_irq_polling))
+		goto out_unlock;
+
+	nvme_start_freeze(&dev->ctrl);
+	nvme_wait_freeze(&dev->ctrl);
+
+	mutex_lock(&dev->shutdown_lock);
+	if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_LIVE) {
+		ret = -EBUSY;
+	} else {
+		WRITE_ONCE(dev->adaptive_irq_polling, enable);
+		nvme_adaptive_stop(dev, dev->adaptive_stopped);
+	}
+	mutex_unlock(&dev->shutdown_lock);
+
+	nvme_unfreeze(&dev->ctrl);
+out_unlock:
+	mutex_unlock(&dev->ctrl.scan_lock);
+	return ret;
+}
+
 static void nvme_suspend_queue(struct nvme_dev *dev, unsigned int qid)
 {
 	struct nvme_queue *nvmeq = &dev->queues[qid];
@@ -2056,8 +2524,14 @@ static void nvme_suspend_queue(struct nvme_dev *dev, unsigned int qid)
 	nvmeq->dev->online_queues--;
 	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
 		nvme_quiesce_admin_queue(&nvmeq->dev->ctrl);
-	if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags))
+	if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags)) {
+		mutex_lock(&nvme_adaptive_mutex);
+		nvme_adaptive_set_queue(nvmeq, false);
+		if (nvmeq->adaptive)
+			nvmeq->adaptive->irq = -1;
+		mutex_unlock(&nvme_adaptive_mutex);
 		pci_free_irq(to_pci_dev(dev->dev), nvmeq->cq_vector, nvmeq);
+	}
 }
 
 static void nvme_suspend_io_queues(struct nvme_dev *dev)
@@ -2076,12 +2550,13 @@ static void nvme_suspend_io_queues(struct nvme_dev *dev)
  */
 static void nvme_reap_pending_cqes(struct nvme_dev *dev)
 {
+	unsigned long flags;
 	int i;
 
 	for (i = dev->ctrl.queue_count - 1; i > 0; i--) {
-		spin_lock(&dev->queues[i].cq_poll_lock);
-		nvme_poll_cq(&dev->queues[i], NULL);
-		spin_unlock(&dev->queues[i].cq_poll_lock);
+		spin_lock_irqsave(&dev->queues[i].cq_poll_lock, flags);
+		nvme_poll_cq(&dev->queues[i], NULL, 0);
+		spin_unlock_irqrestore(&dev->queues[i].cq_poll_lock, flags);
 	}
 }
 
@@ -2173,16 +2648,47 @@ static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)
 
 static int queue_request_irq(struct nvme_queue *nvmeq)
 {
+	struct nvme_adaptive_poll *adaptive = nvmeq->adaptive;
 	struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
+	irq_handler_t handler = nvme_irq;
 	int nr = nvmeq->dev->ctrl.instance;
+	int irq, ret;
 
-	if (use_threaded_interrupts) {
+	clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
+	if (use_threaded_interrupts)
 		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
 				nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
-	} else {
-		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
-				NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
+	if (!nvmeq->qid || nvmeq->dev->num_vecs <= 1 || !pdev->msix_enabled)
+		goto request_irq;
+	irq = pci_irq_vector(pdev, nvmeq->cq_vector);
+	if (irq < 0)
+		goto request_irq;
+	if (!adaptive) {
+		adaptive = kzalloc_node(sizeof(*adaptive), GFP_KERNEL,
+					dev_to_node(nvmeq->dev->dev));
+		if (!adaptive)
+			goto request_irq;
+		adaptive->nvmeq = nvmeq;
+		INIT_LIST_HEAD(&adaptive->node);
+		adaptive->cpu = -1;
+		irq_poll_init(&adaptive->iopoll, NVME_ADAPTIVE_POLL_BUDGET,
+			      nvme_adaptive_irq_poll);
+		WRITE_ONCE(nvmeq->adaptive, adaptive);
 	}
+	adaptive->irq = irq;
+	handler = nvme_adaptive_irq;
+	if (READ_ONCE(nvmeq->dev->adaptive_irq_polling) &&
+	    !nvmeq->dev->adaptive_stopped)
+		set_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
+request_irq:
+	ret = pci_request_irq(pdev, nvmeq->cq_vector, handler,
+			      NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
+	if (ret) {
+		clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
+		if (nvmeq->adaptive)
+			nvmeq->adaptive->irq = -1;
+	}
+	return ret;
 }
 
 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
@@ -2193,6 +2699,11 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
 	nvmeq->last_sq_tail = 0;
 	nvmeq->cq_head = 0;
 	nvmeq->cq_phase = 1;
+	clear_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags);
+	clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
+	clear_bit(NVMEQ_ADAPTIVE_STALE_IRQ, &nvmeq->flags);
+	if (nvmeq->adaptive)
+		nvmeq->adaptive->irq = -1;
 	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
 	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq));
 	nvme_dbbuf_init(dev, nvmeq, qid);
@@ -2813,6 +3324,33 @@ static ssize_t hmb_store(struct device *dev, struct device_attribute *attr,
 }
 static DEVICE_ATTR_RW(hmb);
 
+/* Sysfs task context; no driver locks held. */
+static ssize_t adaptive_irq_polling_show(struct device *dev,
+					 struct device_attribute *attr,
+					 char *buf)
+{
+	struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
+
+	return sysfs_emit(buf, "%d\n", READ_ONCE(ndev->adaptive_irq_polling));
+}
+
+/* Sysfs task context; no driver locks held. */
+static ssize_t adaptive_irq_polling_store(struct device *dev,
+					  struct device_attribute *attr,
+					  const char *buf, size_t count)
+{
+	struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
+	bool enable;
+	int ret;
+
+	ret = kstrtobool(buf, &enable);
+	if (ret)
+		return ret;
+	ret = nvme_adaptive_switch(ndev, enable);
+	return ret ?: count;
+}
+static DEVICE_ATTR_RW(adaptive_irq_polling);
+
 static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj,
 		struct attribute *a, int n)
 {
@@ -2828,6 +3366,9 @@ static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj,
 	}
 	if (a == &dev_attr_hmb.attr && !ctrl->hmpre)
 		return 0;
+	if (a == &dev_attr_adaptive_irq_polling.attr &&
+	    (use_threaded_interrupts || IS_ENABLED(CONFIG_PREEMPT_RT)))
+		return 0;
 
 	return a->mode;
 }
@@ -2837,6 +3378,7 @@ static struct attribute *nvme_pci_attrs[] = {
 	&dev_attr_cmbloc.attr,
 	&dev_attr_cmbsz.attr,
 	&dev_attr_hmb.attr,
+	&dev_attr_adaptive_irq_polling.attr,
 	NULL,
 };
 
@@ -2997,6 +3539,7 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
 	result = nvme_setup_io_queues_trylock(dev);
 	if (result)
 		return result;
+	dev->adaptive_stopped = false;
 	if (test_and_clear_bit(NVMEQ_ENABLED, &adminq->flags))
 		pci_free_irq(pdev, 0, adminq);
 
@@ -3319,6 +3862,7 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
 	}
 
 	nvme_quiesce_io_queues(&dev->ctrl);
+	nvme_adaptive_stop(dev, true);
 
 	if (!dead && dev->ctrl.queue_count > 0) {
 		nvme_delete_io_queues(dev);
@@ -3690,6 +4234,7 @@ static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev,
 		return ERR_PTR(-ENOMEM);
 	INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
 	mutex_init(&dev->shutdown_lock);
+	dev->adaptive_irq_polling = use_adaptive_irq_polling;
 
 	dev->nr_write_queues = write_queues;
 	dev->nr_poll_queues = poll_queues;
@@ -3936,6 +4481,10 @@ static int nvme_resume(struct device *dev)
 	if (ctrl->hmpre && nvme_setup_host_mem(ndev))
 		goto reset;
 
+	mutex_lock(&ndev->shutdown_lock);
+	if (nvme_ctrl_state(ctrl) == NVME_CTRL_LIVE)
+		nvme_adaptive_stop(ndev, false);
+	mutex_unlock(&ndev->shutdown_lock);
 	return 0;
 reset:
 	return nvme_try_sched_reset(ctrl);
@@ -3973,6 +4522,10 @@ static int nvme_suspend(struct device *dev)
 	nvme_wait_freeze(ctrl);
 	nvme_sync_queues(ctrl);
 
+	mutex_lock(&ndev->shutdown_lock);
+	if (nvme_ctrl_state(ctrl) == NVME_CTRL_LIVE)
+		nvme_adaptive_stop(ndev, true);
+	mutex_unlock(&ndev->shutdown_lock);
 	if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
 		goto unfreeze;
 
@@ -4014,6 +4567,12 @@ static int nvme_suspend(struct device *dev)
 		ctrl->npss = 0;
 	}
 unfreeze:
+	if (ret < 0) {
+		mutex_lock(&ndev->shutdown_lock);
+		if (nvme_ctrl_state(ctrl) == NVME_CTRL_LIVE)
+			nvme_adaptive_stop(ndev, false);
+		mutex_unlock(&ndev->shutdown_lock);
+	}
 	nvme_unfreeze(ctrl);
 	return ret;
 }
@@ -4311,14 +4870,44 @@ static struct pci_driver nvme_driver = {
 	.err_handler	= &nvme_err_handler,
 };
 
+/* Sleepable module teardown; no driver locks held, PCI queues are gone. */
+static void nvme_adaptive_cleanup(void)
+{
+	int cpu;
+
+	cpuhp_remove_state_nocalls(nvme_adaptive_cpuhp_state);
+	for_each_possible_cpu(cpu)
+		hrtimer_cancel(&per_cpu(nvme_adaptive_cpus, cpu).timer);
+}
+
 static int __init nvme_init(void)
 {
+	int cpu, ret;
+
 	BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
 	BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2);
 
-	return pci_register_driver(&nvme_driver);
+	for_each_possible_cpu(cpu) {
+		struct nvme_adaptive_cpu *group = per_cpu_ptr(&nvme_adaptive_cpus, cpu);
+
+		raw_spin_lock_init(&group->lock);
+		INIT_LIST_HEAD(&group->members);
+		group->online = true;
+		hrtimer_setup(&group->timer, nvme_adaptive_poll_timer,
+			      CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
+	}
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "nvme/poll:online",
+					nvme_adaptive_cpu_online,
+					nvme_adaptive_cpu_offline);
+	if (ret < 0)
+		return ret;
+	nvme_adaptive_cpuhp_state = ret;
+	ret = pci_register_driver(&nvme_driver);
+	if (ret)
+		nvme_adaptive_cleanup();
+	return ret;
 }
 
 static void __exit nvme_exit(void)
@@ -4326,6 +4915,7 @@ static void __exit nvme_exit(void)
 	kfree(nvme_pci_quirk_list);
 	pci_unregister_driver(&nvme_driver);
 	flush_workqueue(nvme_wq);
+	nvme_adaptive_cleanup();
 }
 
 MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
-- 
2.39.5

Re: [RFC PATCH 0/3] nvme-pci: dynamic interrupt moderation

[Christoph Hellwig]

On Thu, Oct 22, 2026 at 05:43:54PM +0800, Fengnan wrote:
> Hi Keith:
> 
> I made some modifications to DIM and then implemented a solution
> using the DIM framework. 
> Based on current testing, the results are now comparable to those of
> the previous patch.
> Here are the preliminary test results:
> https://docs.google.com/spreadsheets/d/1e8LQPnaDhmv50XlAM3XxI0TMQkhbk0bTZwlJP1Hzzdk/edit?gid=188992110#gid=188992110
> 
> If you agree with this approach, I’ll continue testing and optimizing.

І can't speak for Keith, but I'd really love to see this going forward!

Re: [RFC PATCH 0/3] nvme-pci: dynamic interrupt moderation

[Keith Busch]

On Wed, Oct 28, 2026 at 04:07:12PM +0100, Christoph Hellwig wrote:
> On Thu, Oct 22, 2026 at 05:43:54PM +0800, Fengnan wrote:
> > Hi Keith:
> > 
> > I made some modifications to DIM and then implemented a solution
> > using the DIM framework. 
> > Based on current testing, the results are now comparable to those of
> > the previous patch.
> > Here are the preliminary test results:
> > https://docs.google.com/spreadsheets/d/1e8LQPnaDhmv50XlAM3XxI0TMQkhbk0bTZwlJP1Hzzdk/edit?gid=188992110#gid=188992110
> > 
> > If you agree with this approach, I’ll continue testing and optimizing.
> 
> І can't speak for Keith, but I'd really love to see this going forward!

Same! This is a great problem to solve, but I'm just sorry I don't have
more cycles to work on the cause. I can quickly respin the msix noflush
pre-patches to address the naming issues tglx mentioned, and we can take
nvme in whichever direction has the best performance.