* [PATCH iwl-next v8 13/15] ixd: add reset checks and initialize the mailbox
From: Larysa Zaremba @ 2026-06-08 14:41 UTC (permalink / raw)
To: intel-wired-lan, Tony Nguyen
Cc: aleksander.lobakin, sridhar.samudrala, Michal Swiatkowski,
Larysa Zaremba, Fijalkowski, Maciej, Emil Tantilov, Madhu Chittim,
Josh Hay, Keller, Jacob E, jayaprakash.shanmugam, Jiri Pirko,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Simon Horman, Jonathan Corbet, Richard Cochran, Przemek Kitszel,
Andrew Lunn, netdev, linux-doc, linux-kernel, Aleksandr Loktionov,
Bharath R
In-Reply-To: <20260608144127.2751230-1-larysa.zaremba@intel.com>
At the end of the probe, trigger hard reset, initialize and schedule the
after-reset task. If the reset is complete in a pre-determined time,
initialize the default mailbox, through which other resources will be
negotiated.
Co-developed-by: Amritha Nambiar <amritha.nambiar@intel.com>
Signed-off-by: Amritha Nambiar <amritha.nambiar@intel.com>
Reviewed-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
Reviewed-by: Aleksandr Loktionov <aleksandr.loktionov@intel.com>
Tested-by: Bharath R <Bharath.r@intel.com>
Signed-off-by: Larysa Zaremba <larysa.zaremba@intel.com>
---
drivers/net/ethernet/intel/ixd/Kconfig | 1 +
drivers/net/ethernet/intel/ixd/Makefile | 2 +
drivers/net/ethernet/intel/ixd/ixd.h | 28 +++-
drivers/net/ethernet/intel/ixd/ixd_dev.c | 89 +++++++++++
drivers/net/ethernet/intel/ixd/ixd_lan_regs.h | 40 +++++
drivers/net/ethernet/intel/ixd/ixd_lib.c | 143 ++++++++++++++++++
drivers/net/ethernet/intel/ixd/ixd_main.c | 32 +++-
7 files changed, 326 insertions(+), 9 deletions(-)
create mode 100644 drivers/net/ethernet/intel/ixd/ixd_dev.c
create mode 100644 drivers/net/ethernet/intel/ixd/ixd_lib.c
diff --git a/drivers/net/ethernet/intel/ixd/Kconfig b/drivers/net/ethernet/intel/ixd/Kconfig
index f5594efe292c..24510c50070e 100644
--- a/drivers/net/ethernet/intel/ixd/Kconfig
+++ b/drivers/net/ethernet/intel/ixd/Kconfig
@@ -5,6 +5,7 @@ config IXD
tristate "Intel(R) Control Plane Function Support"
depends on PCI_MSI
select LIBETH
+ select LIBIE_CP
select LIBIE_PCI
help
This driver supports Intel(R) Control Plane PCI Function
diff --git a/drivers/net/ethernet/intel/ixd/Makefile b/drivers/net/ethernet/intel/ixd/Makefile
index 3849bc240600..164b2c86952f 100644
--- a/drivers/net/ethernet/intel/ixd/Makefile
+++ b/drivers/net/ethernet/intel/ixd/Makefile
@@ -6,3 +6,5 @@
obj-$(CONFIG_IXD) += ixd.o
ixd-y := ixd_main.o
+ixd-y += ixd_dev.o
+ixd-y += ixd_lib.o
diff --git a/drivers/net/ethernet/intel/ixd/ixd.h b/drivers/net/ethernet/intel/ixd/ixd.h
index 1b918c5d31cd..c86c2b05c5b4 100644
--- a/drivers/net/ethernet/intel/ixd/ixd.h
+++ b/drivers/net/ethernet/intel/ixd/ixd.h
@@ -4,14 +4,25 @@
#ifndef _IXD_H_
#define _IXD_H_
-#include <linux/net/intel/libie/pci.h>
+#include <linux/net/intel/libie/controlq.h>
/**
* struct ixd_adapter - Data structure representing a CPF
- * @hw: Device access data
+ * @cp_ctx: Control plane communication context
+ * @init_task: Delayed initialization after reset
+ * @xnm: virtchnl transaction manager
+ * @asq: Send control queue info
+ * @arq: Receive control queue info
*/
struct ixd_adapter {
- struct libie_mmio_info hw;
+ struct libie_ctlq_ctx cp_ctx;
+ struct {
+ struct delayed_work init_work;
+ u8 reset_retries;
+ } init_task;
+ struct libie_ctlq_xn_manager *xnm;
+ struct libie_ctlq_info *asq;
+ struct libie_ctlq_info *arq;
};
/**
@@ -22,7 +33,16 @@ struct ixd_adapter {
*/
static inline struct device *ixd_to_dev(struct ixd_adapter *adapter)
{
- return &adapter->hw.pdev->dev;
+ return &adapter->cp_ctx.mmio_info.pdev->dev;
}
+void ixd_ctlq_reg_init(struct ixd_adapter *adapter,
+ struct libie_ctlq_reg *ctlq_reg_tx,
+ struct libie_ctlq_reg *ctlq_reg_rx);
+void ixd_trigger_reset(struct ixd_adapter *adapter);
+bool ixd_check_reset_complete(struct ixd_adapter *adapter);
+void ixd_init_task(struct work_struct *work);
+int ixd_init_dflt_mbx(struct ixd_adapter *adapter);
+void ixd_deinit_dflt_mbx(struct ixd_adapter *adapter);
+
#endif /* _IXD_H_ */
diff --git a/drivers/net/ethernet/intel/ixd/ixd_dev.c b/drivers/net/ethernet/intel/ixd/ixd_dev.c
new file mode 100644
index 000000000000..cdd5477cc1f4
--- /dev/null
+++ b/drivers/net/ethernet/intel/ixd/ixd_dev.c
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2025 Intel Corporation */
+
+#include "ixd.h"
+#include "ixd_lan_regs.h"
+
+/**
+ * ixd_ctlq_reg_init - Initialize default mailbox registers
+ * @adapter: PCI device driver-specific private data
+ * @ctlq_reg_tx: Transmit queue registers info to be filled
+ * @ctlq_reg_rx: Receive queue registers info to be filled
+ */
+void ixd_ctlq_reg_init(struct ixd_adapter *adapter,
+ struct libie_ctlq_reg *ctlq_reg_tx,
+ struct libie_ctlq_reg *ctlq_reg_rx)
+{
+ struct libie_mmio_info *mmio_info = &adapter->cp_ctx.mmio_info;
+ *ctlq_reg_tx = (struct libie_ctlq_reg) {
+ .head = libie_pci_get_mmio_addr(mmio_info, PF_FW_ATQH),
+ .tail = libie_pci_get_mmio_addr(mmio_info, PF_FW_ATQT),
+ .len = libie_pci_get_mmio_addr(mmio_info, PF_FW_ATQLEN),
+ .addr_high = libie_pci_get_mmio_addr(mmio_info, PF_FW_ATQBAH),
+ .addr_low = libie_pci_get_mmio_addr(mmio_info, PF_FW_ATQBAL),
+ .len_mask = PF_FW_ATQLEN_ATQLEN_M,
+ .len_ena_mask = PF_FW_ATQLEN_ATQENABLE_M,
+ .head_mask = PF_FW_ATQH_ATQH_M,
+ };
+
+ *ctlq_reg_rx = (struct libie_ctlq_reg) {
+ .head = libie_pci_get_mmio_addr(mmio_info, PF_FW_ARQH),
+ .tail = libie_pci_get_mmio_addr(mmio_info, PF_FW_ARQT),
+ .len = libie_pci_get_mmio_addr(mmio_info, PF_FW_ARQLEN),
+ .addr_high = libie_pci_get_mmio_addr(mmio_info, PF_FW_ARQBAH),
+ .addr_low = libie_pci_get_mmio_addr(mmio_info, PF_FW_ARQBAL),
+ .len_mask = PF_FW_ARQLEN_ARQLEN_M,
+ .len_ena_mask = PF_FW_ARQLEN_ARQENABLE_M,
+ .head_mask = PF_FW_ARQH_ARQH_M,
+ };
+}
+
+static const struct ixd_reset_reg ixd_reset_reg = {
+ .rstat = PFGEN_RSTAT,
+ .rstat_m = PFGEN_RSTAT_PFR_STATE_M,
+ .rstat_ok_v = 0b01,
+ .rtrigger = PFGEN_CTRL,
+ .rtrigger_m = PFGEN_CTRL_PFSWR,
+};
+
+/**
+ * ixd_trigger_reset - Trigger PFR reset
+ * @adapter: the device with mapped reset register
+ */
+void ixd_trigger_reset(struct ixd_adapter *adapter)
+{
+ void __iomem *addr;
+ u32 reg_val;
+
+ addr = libie_pci_get_mmio_addr(&adapter->cp_ctx.mmio_info,
+ ixd_reset_reg.rtrigger);
+ reg_val = readl(addr);
+ writel(reg_val | ixd_reset_reg.rtrigger_m, addr);
+}
+
+/**
+ * ixd_check_reset_complete - Check if the PFR reset is completed
+ * @adapter: CPF being reset
+ *
+ * Return: %true if the register read indicates reset has been finished,
+ * %false otherwise
+ */
+bool ixd_check_reset_complete(struct ixd_adapter *adapter)
+{
+ u32 reg_val, reset_status;
+ void __iomem *addr;
+
+ addr = libie_pci_get_mmio_addr(&adapter->cp_ctx.mmio_info,
+ ixd_reset_reg.rstat);
+ reg_val = readl(addr);
+ reset_status = reg_val & ixd_reset_reg.rstat_m;
+
+ /* 0xFFFFFFFF might be read if the other side hasn't cleared
+ * the register for us yet.
+ */
+ if (reg_val != GENMASK(31, 0) &&
+ reset_status == ixd_reset_reg.rstat_ok_v)
+ return true;
+
+ return false;
+}
diff --git a/drivers/net/ethernet/intel/ixd/ixd_lan_regs.h b/drivers/net/ethernet/intel/ixd/ixd_lan_regs.h
index fbb88929d0de..58e58c75981b 100644
--- a/drivers/net/ethernet/intel/ixd/ixd_lan_regs.h
+++ b/drivers/net/ethernet/intel/ixd/ixd_lan_regs.h
@@ -11,9 +11,33 @@
#define PF_FW_MBX_REG_LEN 4096
#define PF_FW_MBX 0x08400000
+#define PF_FW_ARQBAL (PF_FW_MBX)
+#define PF_FW_ARQBAH (PF_FW_MBX + 0x4)
+#define PF_FW_ARQLEN (PF_FW_MBX + 0x8)
+#define PF_FW_ARQLEN_ARQLEN_M GENMASK(12, 0)
+#define PF_FW_ARQLEN_ARQENABLE_S 31
+#define PF_FW_ARQLEN_ARQENABLE_M BIT(PF_FW_ARQLEN_ARQENABLE_S)
+#define PF_FW_ARQH_ARQH_M GENMASK(12, 0)
+#define PF_FW_ARQH (PF_FW_MBX + 0xC)
+#define PF_FW_ARQT (PF_FW_MBX + 0x10)
+
+#define PF_FW_ATQBAL (PF_FW_MBX + 0x14)
+#define PF_FW_ATQBAH (PF_FW_MBX + 0x18)
+#define PF_FW_ATQLEN (PF_FW_MBX + 0x1C)
+#define PF_FW_ATQLEN_ATQLEN_M GENMASK(9, 0)
+#define PF_FW_ATQLEN_ATQENABLE_S 31
+#define PF_FW_ATQLEN_ATQENABLE_M BIT(PF_FW_ATQLEN_ATQENABLE_S)
+#define PF_FW_ATQH_ATQH_M GENMASK(9, 0)
+#define PF_FW_ATQH (PF_FW_MBX + 0x20)
+#define PF_FW_ATQT (PF_FW_MBX + 0x24)
+
/* Reset registers */
#define PFGEN_RTRIG_REG_LEN 2048
#define PFGEN_RTRIG 0x08407000 /* Device resets */
+#define PFGEN_RSTAT 0x08407008 /* PFR status */
+#define PFGEN_RSTAT_PFR_STATE_M GENMASK(1, 0)
+#define PFGEN_CTRL 0x0840700C /* PFR trigger */
+#define PFGEN_CTRL_PFSWR BIT(0)
/**
* struct ixd_bar_region - BAR region description
@@ -25,4 +49,20 @@ struct ixd_bar_region {
resource_size_t size;
};
+/**
+ * struct ixd_reset_reg - structure for reset registers
+ * @rstat: offset of status in register
+ * @rstat_m: status mask
+ * @rstat_ok_v: value that indicates PFR completed status
+ * @rtrigger: offset of reset trigger in register
+ * @rtrigger_m: reset trigger mask
+ */
+struct ixd_reset_reg {
+ u32 rstat;
+ u32 rstat_m;
+ u32 rstat_ok_v;
+ u32 rtrigger;
+ u32 rtrigger_m;
+};
+
#endif /* _IXD_LAN_REGS_H_ */
diff --git a/drivers/net/ethernet/intel/ixd/ixd_lib.c b/drivers/net/ethernet/intel/ixd/ixd_lib.c
new file mode 100644
index 000000000000..afc413d3650f
--- /dev/null
+++ b/drivers/net/ethernet/intel/ixd/ixd_lib.c
@@ -0,0 +1,143 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2025 Intel Corporation */
+
+#include "ixd.h"
+
+#define IXD_DFLT_MBX_Q_LEN 64
+
+/**
+ * ixd_init_ctlq_create_info - Initialize control queue info for creation
+ * @info: destination
+ * @type: type of the queue to create
+ * @ctlq_reg: register assigned to the control queue
+ */
+static void ixd_init_ctlq_create_info(struct libie_ctlq_create_info *info,
+ enum virtchnl2_queue_type type,
+ const struct libie_ctlq_reg *ctlq_reg)
+{
+ *info = (struct libie_ctlq_create_info) {
+ .type = type,
+ .id = -1,
+ .reg = *ctlq_reg,
+ .len = IXD_DFLT_MBX_Q_LEN,
+ };
+}
+
+/**
+ * ixd_init_libie_xn_params - Initialize xn transaction manager creation info
+ * @params: destination
+ * @adapter: adapter info struct
+ * @ctlqs: list of the managed queues to create
+ * @num_queues: length of the queue list
+ */
+static void ixd_init_libie_xn_params(struct libie_ctlq_xn_init_params *params,
+ struct ixd_adapter *adapter,
+ struct libie_ctlq_create_info *ctlqs,
+ uint num_queues)
+{
+ *params = (struct libie_ctlq_xn_init_params){
+ .cctlq_info = ctlqs,
+ .ctx = &adapter->cp_ctx,
+ .num_qs = num_queues,
+ };
+}
+
+/**
+ * ixd_adapter_fill_dflt_ctlqs - Find default control queues and store them
+ * @adapter: adapter info struct
+ */
+static void ixd_adapter_fill_dflt_ctlqs(struct ixd_adapter *adapter)
+{
+ guard(spinlock)(&adapter->cp_ctx.ctlqs_lock);
+ struct libie_ctlq_info *cq;
+
+ list_for_each_entry(cq, &adapter->cp_ctx.ctlqs, list) {
+ if (cq->qid != -1)
+ continue;
+ if (cq->type == LIBIE_CTLQ_TYPE_RX)
+ adapter->arq = cq;
+ else if (cq->type == LIBIE_CTLQ_TYPE_TX)
+ adapter->asq = cq;
+ }
+}
+
+/**
+ * ixd_deinit_dflt_mbx - Deinitialize default mailbox
+ * @adapter: adapter info struct
+ */
+void ixd_deinit_dflt_mbx(struct ixd_adapter *adapter)
+{
+ if (adapter->xnm)
+ libie_ctlq_xn_deinit(adapter->xnm, &adapter->cp_ctx);
+
+ adapter->arq = NULL;
+ adapter->asq = NULL;
+ adapter->xnm = NULL;
+}
+
+/**
+ * ixd_init_dflt_mbx - Setup default mailbox parameters and make request
+ * @adapter: adapter info struct
+ *
+ * Return: %0 on success, negative errno code on failure
+ */
+int ixd_init_dflt_mbx(struct ixd_adapter *adapter)
+{
+ struct libie_ctlq_create_info ctlqs_info[2];
+ struct libie_ctlq_xn_init_params xn_params;
+ struct libie_ctlq_reg ctlq_reg_tx;
+ struct libie_ctlq_reg ctlq_reg_rx;
+ int err;
+
+ ixd_ctlq_reg_init(adapter, &ctlq_reg_tx, &ctlq_reg_rx);
+ ixd_init_ctlq_create_info(&ctlqs_info[0], LIBIE_CTLQ_TYPE_TX,
+ &ctlq_reg_tx);
+ ixd_init_ctlq_create_info(&ctlqs_info[1], LIBIE_CTLQ_TYPE_RX,
+ &ctlq_reg_rx);
+ ixd_init_libie_xn_params(&xn_params, adapter, ctlqs_info,
+ ARRAY_SIZE(ctlqs_info));
+ err = libie_ctlq_xn_init(&xn_params);
+ if (err)
+ return err;
+ adapter->xnm = xn_params.xnm;
+
+ ixd_adapter_fill_dflt_ctlqs(adapter);
+
+ if (!adapter->asq || !adapter->arq) {
+ ixd_deinit_dflt_mbx(adapter);
+ return -ENOENT;
+ }
+
+ return 0;
+}
+
+/**
+ * ixd_init_task - Initialize after reset
+ * @work: init work struct
+ */
+void ixd_init_task(struct work_struct *work)
+{
+ struct ixd_adapter *adapter;
+ int err;
+
+ adapter = container_of(work, struct ixd_adapter,
+ init_task.init_work.work);
+
+ if (!ixd_check_reset_complete(adapter)) {
+ if (++adapter->init_task.reset_retries < 10)
+ queue_delayed_work(system_unbound_wq,
+ &adapter->init_task.init_work,
+ msecs_to_jiffies(500));
+ else
+ dev_err(ixd_to_dev(adapter),
+ "Device reset failed. The driver was unable to contact the device's firmware. Check that the FW is running.\n");
+ return;
+ }
+
+ adapter->init_task.reset_retries = 0;
+ err = ixd_init_dflt_mbx(adapter);
+ if (err)
+ dev_err(ixd_to_dev(adapter),
+ "Failed to initialize the default mailbox: %pe\n",
+ ERR_PTR(err));
+}
diff --git a/drivers/net/ethernet/intel/ixd/ixd_main.c b/drivers/net/ethernet/intel/ixd/ixd_main.c
index 75ee53152e61..b4d4000b63ed 100644
--- a/drivers/net/ethernet/intel/ixd/ixd_main.c
+++ b/drivers/net/ethernet/intel/ixd/ixd_main.c
@@ -5,6 +5,7 @@
#include "ixd_lan_regs.h"
MODULE_DESCRIPTION("Intel(R) Control Plane Function Device Driver");
+MODULE_IMPORT_NS("LIBIE_CP");
MODULE_IMPORT_NS("LIBIE_PCI");
MODULE_LICENSE("GPL");
@@ -16,7 +17,13 @@ static void ixd_remove(struct pci_dev *pdev)
{
struct ixd_adapter *adapter = pci_get_drvdata(pdev);
- libie_pci_unmap_all_mmio_regions(&adapter->hw);
+ /* Do not mix removal with (re)initialization */
+ cancel_delayed_work_sync(&adapter->init_task.init_work);
+ /* Leave the device clean on exit */
+ ixd_trigger_reset(adapter);
+ ixd_deinit_dflt_mbx(adapter);
+
+ libie_pci_unmap_all_mmio_regions(&adapter->cp_ctx.mmio_info);
}
/**
@@ -51,7 +58,7 @@ static int ixd_iomap_regions(struct ixd_adapter *adapter)
};
for (int i = 0; i < ARRAY_SIZE(regions); i++) {
- struct libie_mmio_info *mmio_info = &adapter->hw;
+ struct libie_mmio_info *mmio_info = &adapter->cp_ctx.mmio_info;
bool map_ok;
map_ok = libie_pci_map_mmio_region(mmio_info,
@@ -81,11 +88,15 @@ static int ixd_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
struct ixd_adapter *adapter;
int err;
+ if (WARN_ON(ent->device != IXD_DEV_ID_CPF))
+ return -EINVAL;
+
adapter = devm_kzalloc(&pdev->dev, sizeof(*adapter), GFP_KERNEL);
if (!adapter)
return -ENOMEM;
- adapter->hw.pdev = pdev;
- INIT_LIST_HEAD(&adapter->hw.mmio_list);
+
+ adapter->cp_ctx.mmio_info.pdev = pdev;
+ INIT_LIST_HEAD(&adapter->cp_ctx.mmio_info.mmio_list);
err = libie_pci_init_dev(pdev);
if (err)
@@ -93,7 +104,18 @@ static int ixd_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
pci_set_drvdata(pdev, adapter);
- return ixd_iomap_regions(adapter);
+ err = ixd_iomap_regions(adapter);
+ if (err)
+ return err;
+
+ INIT_DELAYED_WORK(&adapter->init_task.init_work,
+ ixd_init_task);
+
+ ixd_trigger_reset(adapter);
+ queue_delayed_work(system_unbound_wq, &adapter->init_task.init_work,
+ msecs_to_jiffies(500));
+
+ return 0;
}
static const struct pci_device_id ixd_pci_tbl[] = {
--
2.47.0
^ permalink raw reply related
* [PATCH iwl-next v8 09/15] idpf: refactor idpf to use libie control queues
From: Larysa Zaremba @ 2026-06-08 14:41 UTC (permalink / raw)
To: intel-wired-lan, Tony Nguyen
Cc: aleksander.lobakin, sridhar.samudrala, Michal Swiatkowski,
Larysa Zaremba, Fijalkowski, Maciej, Emil Tantilov, Madhu Chittim,
Josh Hay, Keller, Jacob E, jayaprakash.shanmugam, Jiri Pirko,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Simon Horman, Jonathan Corbet, Richard Cochran, Przemek Kitszel,
Andrew Lunn, netdev, linux-doc, linux-kernel, Aleksandr Loktionov,
Samuel Salin
In-Reply-To: <20260608144127.2751230-1-larysa.zaremba@intel.com>
From: Pavan Kumar Linga <pavan.kumar.linga@intel.com>
Support to initialize and configure controlqs, and manage their
transactions was introduced in libie. As part of it, most of the existing
controlq structures are renamed and modified. Use those APIs in idpf and
make all the necessary changes.
Previously for the send and receive virtchnl messages, there used to be a
memcpy involved in controlq code to copy the buffer info passed by the send
function into the controlq specific buffers. There was no restriction to
use automatic memory in that case. The new implementation in libie removed
copying of the send buffer info and introduced DMA mapping of the send
buffer itself. To accommodate it, use dynamic memory for the larger send
buffers. For smaller ones (<= 128 bytes) libie still can copy them into the
pre-allocated message memory.
In case of receive, idpf receives a page pool buffer allocated by the libie
and care should be taken to release it after use in the idpf.
The changes are fairly trivial and localized, with a notable exception
being the consolidation of idpf_vc_xn_shutdown and idpf_deinit_dflt_mbx
under the latter name. This has some additional consequences that are
addressed in the following patches.
This refactoring introduces roughly additional 40KB of module storage used
for systems that only run idpf, so idpf + libie_cp + libie_pci takes about
7% more storage than just idpf before refactoring.
We now pre-allocate small TX buffers, so that does increase the memory
usage, but reduces the need to allocate. This results in additional 256 *
128B of memory permanently used, increasing the worst-case memory usage by
32KB but our ctlq RX buffers need to be of size 4096B anyway (not changed
by the patchset), so this is hardly noticeable.
As for the timings, the fact that we are mostly limited by the HW response
time which is far from instant, is not changed by this refactor.
Reviewed-by: Aleksandr Loktionov <aleksandr.loktionov@intel.com>
Signed-off-by: Pavan Kumar Linga <pavan.kumar.linga@intel.com>
Tested-by: Samuel Salin <Samuel.salin@intel.com>
Co-developed-by: Larysa Zaremba <larysa.zaremba@intel.com>
Signed-off-by: Larysa Zaremba <larysa.zaremba@intel.com>
---
drivers/net/ethernet/intel/idpf/Makefile | 2 -
drivers/net/ethernet/intel/idpf/idpf.h | 28 +-
.../net/ethernet/intel/idpf/idpf_controlq.c | 631 -------
.../net/ethernet/intel/idpf/idpf_controlq.h | 142 --
.../ethernet/intel/idpf/idpf_controlq_api.h | 177 --
.../ethernet/intel/idpf/idpf_controlq_setup.c | 169 --
drivers/net/ethernet/intel/idpf/idpf_dev.c | 60 +-
.../net/ethernet/intel/idpf/idpf_ethtool.c | 28 +-
drivers/net/ethernet/intel/idpf/idpf_lib.c | 51 +-
drivers/net/ethernet/intel/idpf/idpf_main.c | 5 -
drivers/net/ethernet/intel/idpf/idpf_mem.h | 20 -
drivers/net/ethernet/intel/idpf/idpf_txrx.h | 2 +-
drivers/net/ethernet/intel/idpf/idpf_vf_dev.c | 64 +-
.../net/ethernet/intel/idpf/idpf_virtchnl.c | 1570 +++++++----------
.../net/ethernet/intel/idpf/idpf_virtchnl.h | 94 +-
.../ethernet/intel/idpf/idpf_virtchnl_ptp.c | 251 ++-
16 files changed, 806 insertions(+), 2488 deletions(-)
delete mode 100644 drivers/net/ethernet/intel/idpf/idpf_controlq.c
delete mode 100644 drivers/net/ethernet/intel/idpf/idpf_controlq.h
delete mode 100644 drivers/net/ethernet/intel/idpf/idpf_controlq_api.h
delete mode 100644 drivers/net/ethernet/intel/idpf/idpf_controlq_setup.c
delete mode 100644 drivers/net/ethernet/intel/idpf/idpf_mem.h
diff --git a/drivers/net/ethernet/intel/idpf/Makefile b/drivers/net/ethernet/intel/idpf/Makefile
index 651ddee942bd..4aaafa175ec3 100644
--- a/drivers/net/ethernet/intel/idpf/Makefile
+++ b/drivers/net/ethernet/intel/idpf/Makefile
@@ -6,8 +6,6 @@
obj-$(CONFIG_IDPF) += idpf.o
idpf-y := \
- idpf_controlq.o \
- idpf_controlq_setup.o \
idpf_dev.o \
idpf_ethtool.o \
idpf_idc.o \
diff --git a/drivers/net/ethernet/intel/idpf/idpf.h b/drivers/net/ethernet/intel/idpf/idpf.h
index 92a120aadfcd..d7d751e2a781 100644
--- a/drivers/net/ethernet/intel/idpf/idpf.h
+++ b/drivers/net/ethernet/intel/idpf/idpf.h
@@ -27,7 +27,6 @@ struct idpf_rss_data;
#include <linux/net/intel/virtchnl2.h>
#include "idpf_txrx.h"
-#include "idpf_controlq.h"
#define GETMAXVAL(num_bits) GENMASK((num_bits) - 1, 0)
@@ -37,11 +36,10 @@ struct idpf_rss_data;
#define IDPF_NUM_FILTERS_PER_MSG 20
#define IDPF_NUM_DFLT_MBX_Q 2 /* includes both TX and RX */
#define IDPF_DFLT_MBX_Q_LEN 64
-#define IDPF_DFLT_MBX_ID -1
/* maximum number of times to try before resetting mailbox */
#define IDPF_MB_MAX_ERR 20
#define IDPF_NUM_CHUNKS_PER_MSG(struct_sz, chunk_sz) \
- ((IDPF_CTLQ_MAX_BUF_LEN - (struct_sz)) / (chunk_sz))
+ ((LIBIE_CTLQ_MAX_BUF_LEN - (struct_sz)) / (chunk_sz))
#define IDPF_WAIT_FOR_MARKER_TIMEO 500
#define IDPF_MAX_WAIT 500
@@ -202,8 +200,8 @@ struct idpf_vport_max_q {
* @ptp_reg_init: PTP register initialization
*/
struct idpf_reg_ops {
- void (*ctlq_reg_init)(struct idpf_adapter *adapter,
- struct idpf_ctlq_create_info *cq);
+ void (*ctlq_reg_init)(struct libie_mmio_info *mmio,
+ struct libie_ctlq_create_info *cctlq_info);
int (*intr_reg_init)(struct idpf_vport *vport,
struct idpf_q_vec_rsrc *rsrc);
void (*mb_intr_reg_init)(struct idpf_adapter *adapter);
@@ -606,8 +604,6 @@ struct idpf_vport_config {
DECLARE_BITMAP(flags, IDPF_VPORT_CONFIG_FLAGS_NBITS);
};
-struct idpf_vc_xn_manager;
-
#define idpf_for_each_vport(adapter, iter) \
for (struct idpf_vport **__##iter = &(adapter)->vports[0], \
*iter = (adapter)->max_vports ? *__##iter : NULL; \
@@ -625,8 +621,10 @@ struct idpf_vc_xn_manager;
* @state: Init state machine
* @flags: See enum idpf_flags
* @reset_reg: See struct idpf_reset_reg
- * @hw: Device access data
* @ctlq_ctx: controlq context
+ * @asq: Send control queue info
+ * @arq: Receive control queue info
+ * @xnm: Xn transaction manager
* @num_avail_msix: Available number of MSIX vectors
* @num_msix_entries: Number of entries in MSIX table
* @msix_entries: MSIX table
@@ -659,7 +657,6 @@ struct idpf_vc_xn_manager;
* @stats_task: Periodic statistics retrieval task
* @stats_wq: Workqueue for statistics task
* @caps: Negotiated capabilities with device
- * @vcxn_mngr: Virtchnl transaction manager
* @dev_ops: See idpf_dev_ops
* @cdev_info: IDC core device info pointer
* @num_vfs: Number of allocated VFs through sysfs. PF does not directly talk
@@ -683,8 +680,10 @@ struct idpf_adapter {
enum idpf_state state;
DECLARE_BITMAP(flags, IDPF_FLAGS_NBITS);
struct idpf_reset_reg reset_reg;
- struct idpf_hw hw;
struct libie_ctlq_ctx ctlq_ctx;
+ struct libie_ctlq_info *asq;
+ struct libie_ctlq_info *arq;
+ struct libie_ctlq_xn_manager *xnm;
u16 num_avail_msix;
u16 num_msix_entries;
struct msix_entry *msix_entries;
@@ -721,7 +720,6 @@ struct idpf_adapter {
struct delayed_work stats_task;
struct workqueue_struct *stats_wq;
struct virtchnl2_get_capabilities caps;
- struct idpf_vc_xn_manager *vcxn_mngr;
struct idpf_dev_ops dev_ops;
struct iidc_rdma_core_dev_info *cdev_info;
@@ -881,12 +879,12 @@ static inline u8 idpf_get_min_tx_pkt_len(struct idpf_adapter *adapter)
*/
static inline bool idpf_is_reset_detected(struct idpf_adapter *adapter)
{
- if (!adapter->hw.arq)
+ struct libie_ctlq_info *arq = adapter->arq;
+
+ if (!arq)
return true;
- return !(readl(libie_pci_get_mmio_addr(&adapter->ctlq_ctx.mmio_info,
- adapter->hw.arq->reg.len)) &
- adapter->hw.arq->reg.len_mask);
+ return !(readl(arq->reg.len) & arq->reg.len_mask);
}
/**
diff --git a/drivers/net/ethernet/intel/idpf/idpf_controlq.c b/drivers/net/ethernet/intel/idpf/idpf_controlq.c
deleted file mode 100644
index 020b08367e18..000000000000
--- a/drivers/net/ethernet/intel/idpf/idpf_controlq.c
+++ /dev/null
@@ -1,631 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/* Copyright (C) 2023 Intel Corporation */
-
-#include "idpf.h"
-
-/**
- * idpf_ctlq_setup_regs - initialize control queue registers
- * @cq: pointer to the specific control queue
- * @q_create_info: structs containing info for each queue to be initialized
- */
-static void idpf_ctlq_setup_regs(struct idpf_ctlq_info *cq,
- struct idpf_ctlq_create_info *q_create_info)
-{
- /* set control queue registers in our local struct */
- cq->reg.head = q_create_info->reg.head;
- cq->reg.tail = q_create_info->reg.tail;
- cq->reg.len = q_create_info->reg.len;
- cq->reg.bah = q_create_info->reg.bah;
- cq->reg.bal = q_create_info->reg.bal;
- cq->reg.len_mask = q_create_info->reg.len_mask;
- cq->reg.len_ena_mask = q_create_info->reg.len_ena_mask;
- cq->reg.head_mask = q_create_info->reg.head_mask;
-}
-
-/**
- * idpf_ctlq_init_regs - Initialize control queue registers
- * @hw: pointer to hw struct
- * @cq: pointer to the specific Control queue
- * @is_rxq: true if receive control queue, false otherwise
- *
- * Initialize registers. The caller is expected to have already initialized the
- * descriptor ring memory and buffer memory
- */
-static void idpf_ctlq_init_regs(struct idpf_hw *hw, struct idpf_ctlq_info *cq,
- bool is_rxq)
-{
- struct libie_mmio_info *mmio = &hw->back->ctlq_ctx.mmio_info;
-
- /* Update tail to post pre-allocated buffers for rx queues */
- if (is_rxq)
- writel((u32)(cq->ring_size - 1),
- libie_pci_get_mmio_addr(mmio, cq->reg.tail));
-
- /* For non-Mailbox control queues only TAIL need to be set */
- if (cq->q_id != -1)
- return;
-
- /* Clear Head for both send or receive */
- writel(0, libie_pci_get_mmio_addr(mmio, cq->reg.head));
-
- /* set starting point */
- writel(lower_32_bits(cq->desc_ring.pa),
- libie_pci_get_mmio_addr(mmio, cq->reg.bal));
- writel(upper_32_bits(cq->desc_ring.pa),
- libie_pci_get_mmio_addr(mmio, cq->reg.bah));
- writel((cq->ring_size | cq->reg.len_ena_mask),
- libie_pci_get_mmio_addr(mmio, cq->reg.len));
-}
-
-/**
- * idpf_ctlq_init_rxq_bufs - populate receive queue descriptors with buf
- * @cq: pointer to the specific Control queue
- *
- * Record the address of the receive queue DMA buffers in the descriptors.
- * The buffers must have been previously allocated.
- */
-static void idpf_ctlq_init_rxq_bufs(struct idpf_ctlq_info *cq)
-{
- int i;
-
- for (i = 0; i < cq->ring_size; i++) {
- struct idpf_ctlq_desc *desc = IDPF_CTLQ_DESC(cq, i);
- struct idpf_dma_mem *bi = cq->bi.rx_buff[i];
-
- /* No buffer to post to descriptor, continue */
- if (!bi)
- continue;
-
- desc->flags =
- cpu_to_le16(IDPF_CTLQ_FLAG_BUF | IDPF_CTLQ_FLAG_RD);
- desc->opcode = 0;
- desc->datalen = cpu_to_le16(bi->size);
- desc->ret_val = 0;
- desc->v_opcode_dtype = 0;
- desc->v_retval = 0;
- desc->params.indirect.addr_high =
- cpu_to_le32(upper_32_bits(bi->pa));
- desc->params.indirect.addr_low =
- cpu_to_le32(lower_32_bits(bi->pa));
- desc->params.indirect.param0 = 0;
- desc->params.indirect.sw_cookie = 0;
- desc->params.indirect.v_flags = 0;
- }
-}
-
-/**
- * idpf_ctlq_shutdown - shutdown the CQ
- * @hw: pointer to hw struct
- * @cq: pointer to the specific Control queue
- *
- * The main shutdown routine for any controq queue
- */
-static void idpf_ctlq_shutdown(struct idpf_hw *hw, struct idpf_ctlq_info *cq)
-{
- spin_lock(&cq->cq_lock);
-
- /* free ring buffers and the ring itself */
- idpf_ctlq_dealloc_ring_res(hw, cq);
-
- /* Set ring_size to 0 to indicate uninitialized queue */
- cq->ring_size = 0;
-
- spin_unlock(&cq->cq_lock);
-}
-
-/**
- * idpf_ctlq_add - add one control queue
- * @hw: pointer to hardware struct
- * @qinfo: info for queue to be created
- * @cq_out: (output) double pointer to control queue to be created
- *
- * Allocate and initialize a control queue and add it to the control queue list.
- * The cq parameter will be allocated/initialized and passed back to the caller
- * if no errors occur.
- *
- * Note: idpf_ctlq_init must be called prior to any calls to idpf_ctlq_add
- */
-int idpf_ctlq_add(struct idpf_hw *hw,
- struct idpf_ctlq_create_info *qinfo,
- struct idpf_ctlq_info **cq_out)
-{
- struct idpf_ctlq_info *cq;
- bool is_rxq = false;
- int err;
-
- cq = kzalloc_obj(*cq);
- if (!cq)
- return -ENOMEM;
-
- cq->cq_type = qinfo->type;
- cq->q_id = qinfo->id;
- cq->buf_size = qinfo->buf_size;
- cq->ring_size = qinfo->len;
-
- cq->next_to_use = 0;
- cq->next_to_clean = 0;
- cq->next_to_post = cq->ring_size - 1;
-
- switch (qinfo->type) {
- case IDPF_CTLQ_TYPE_MAILBOX_RX:
- is_rxq = true;
- fallthrough;
- case IDPF_CTLQ_TYPE_MAILBOX_TX:
- err = idpf_ctlq_alloc_ring_res(hw, cq);
- break;
- default:
- err = -EBADR;
- break;
- }
-
- if (err)
- goto init_free_q;
-
- if (is_rxq) {
- idpf_ctlq_init_rxq_bufs(cq);
- } else {
- /* Allocate the array of msg pointers for TX queues */
- cq->bi.tx_msg = kzalloc_objs(struct idpf_ctlq_msg *, qinfo->len);
- if (!cq->bi.tx_msg) {
- err = -ENOMEM;
- goto init_dealloc_q_mem;
- }
- }
-
- idpf_ctlq_setup_regs(cq, qinfo);
-
- idpf_ctlq_init_regs(hw, cq, is_rxq);
-
- spin_lock_init(&cq->cq_lock);
-
- list_add(&cq->cq_list, &hw->cq_list_head);
-
- *cq_out = cq;
-
- return 0;
-
-init_dealloc_q_mem:
- /* free ring buffers and the ring itself */
- idpf_ctlq_dealloc_ring_res(hw, cq);
-init_free_q:
- kfree(cq);
-
- return err;
-}
-
-/**
- * idpf_ctlq_remove - deallocate and remove specified control queue
- * @hw: pointer to hardware struct
- * @cq: pointer to control queue to be removed
- */
-void idpf_ctlq_remove(struct idpf_hw *hw,
- struct idpf_ctlq_info *cq)
-{
- list_del(&cq->cq_list);
- idpf_ctlq_shutdown(hw, cq);
- kfree(cq);
-}
-
-/**
- * idpf_ctlq_init - main initialization routine for all control queues
- * @hw: pointer to hardware struct
- * @num_q: number of queues to initialize
- * @q_info: array of structs containing info for each queue to be initialized
- *
- * This initializes any number and any type of control queues. This is an all
- * or nothing routine; if one fails, all previously allocated queues will be
- * destroyed. This must be called prior to using the individual add/remove
- * APIs.
- */
-int idpf_ctlq_init(struct idpf_hw *hw, u8 num_q,
- struct idpf_ctlq_create_info *q_info)
-{
- struct idpf_ctlq_info *cq, *tmp;
- int err;
- int i;
-
- INIT_LIST_HEAD(&hw->cq_list_head);
-
- for (i = 0; i < num_q; i++) {
- struct idpf_ctlq_create_info *qinfo = q_info + i;
-
- err = idpf_ctlq_add(hw, qinfo, &cq);
- if (err)
- goto init_destroy_qs;
- }
-
- return 0;
-
-init_destroy_qs:
- list_for_each_entry_safe(cq, tmp, &hw->cq_list_head, cq_list)
- idpf_ctlq_remove(hw, cq);
-
- return err;
-}
-
-/**
- * idpf_ctlq_deinit - destroy all control queues
- * @hw: pointer to hw struct
- */
-void idpf_ctlq_deinit(struct idpf_hw *hw)
-{
- struct idpf_ctlq_info *cq, *tmp;
-
- list_for_each_entry_safe(cq, tmp, &hw->cq_list_head, cq_list)
- idpf_ctlq_remove(hw, cq);
-}
-
-/**
- * idpf_ctlq_send - send command to Control Queue (CTQ)
- * @hw: pointer to hw struct
- * @cq: handle to control queue struct to send on
- * @num_q_msg: number of messages to send on control queue
- * @q_msg: pointer to array of queue messages to be sent
- *
- * The caller is expected to allocate DMAable buffers and pass them to the
- * send routine via the q_msg struct / control queue specific data struct.
- * The control queue will hold a reference to each send message until
- * the completion for that message has been cleaned.
- */
-int idpf_ctlq_send(struct idpf_hw *hw, struct idpf_ctlq_info *cq,
- u16 num_q_msg, struct idpf_ctlq_msg q_msg[])
-{
- struct idpf_ctlq_desc *desc;
- int num_desc_avail;
- int err = 0;
- int i;
-
- spin_lock(&cq->cq_lock);
-
- /* Ensure there are enough descriptors to send all messages */
- num_desc_avail = IDPF_CTLQ_DESC_UNUSED(cq);
- if (num_desc_avail == 0 || num_desc_avail < num_q_msg) {
- err = -ENOSPC;
- goto err_unlock;
- }
-
- for (i = 0; i < num_q_msg; i++) {
- struct idpf_ctlq_msg *msg = &q_msg[i];
-
- desc = IDPF_CTLQ_DESC(cq, cq->next_to_use);
-
- desc->opcode = cpu_to_le16(msg->opcode);
- desc->pfid_vfid = cpu_to_le16(msg->func_id);
-
- desc->v_opcode_dtype = cpu_to_le32(msg->cookie.mbx.chnl_opcode);
- desc->v_retval = cpu_to_le32(msg->cookie.mbx.chnl_retval);
-
- desc->flags = cpu_to_le16((msg->host_id & IDPF_HOST_ID_MASK) <<
- IDPF_CTLQ_FLAG_HOST_ID_S);
- if (msg->data_len) {
- struct idpf_dma_mem *buff = msg->ctx.indirect.payload;
-
- desc->datalen |= cpu_to_le16(msg->data_len);
- desc->flags |= cpu_to_le16(IDPF_CTLQ_FLAG_BUF);
- desc->flags |= cpu_to_le16(IDPF_CTLQ_FLAG_RD);
-
- /* Update the address values in the desc with the pa
- * value for respective buffer
- */
- desc->params.indirect.addr_high =
- cpu_to_le32(upper_32_bits(buff->pa));
- desc->params.indirect.addr_low =
- cpu_to_le32(lower_32_bits(buff->pa));
-
- memcpy(&desc->params, msg->ctx.indirect.context,
- IDPF_INDIRECT_CTX_SIZE);
- } else {
- memcpy(&desc->params, msg->ctx.direct,
- IDPF_DIRECT_CTX_SIZE);
- }
-
- /* Store buffer info */
- cq->bi.tx_msg[cq->next_to_use] = msg;
-
- (cq->next_to_use)++;
- if (cq->next_to_use == cq->ring_size)
- cq->next_to_use = 0;
- }
-
- /* Force memory write to complete before letting hardware
- * know that there are new descriptors to fetch.
- */
- dma_wmb();
-
- writel(cq->next_to_use,
- libie_pci_get_mmio_addr(&hw->back->ctlq_ctx.mmio_info,
- cq->reg.tail));
-
-err_unlock:
- spin_unlock(&cq->cq_lock);
-
- return err;
-}
-
-/**
- * idpf_ctlq_clean_sq - reclaim send descriptors on HW write back for the
- * requested queue
- * @cq: pointer to the specific Control queue
- * @clean_count: (input|output) number of descriptors to clean as input, and
- * number of descriptors actually cleaned as output
- * @msg_status: (output) pointer to msg pointer array to be populated; needs
- * to be allocated by caller
- *
- * Returns an array of message pointers associated with the cleaned
- * descriptors. The pointers are to the original ctlq_msgs sent on the cleaned
- * descriptors. The status will be returned for each; any messages that failed
- * to send will have a non-zero status. The caller is expected to free original
- * ctlq_msgs and free or reuse the DMA buffers.
- */
-int idpf_ctlq_clean_sq(struct idpf_ctlq_info *cq, u16 *clean_count,
- struct idpf_ctlq_msg *msg_status[])
-{
- struct idpf_ctlq_desc *desc;
- u16 i, num_to_clean;
- u16 ntc, desc_err;
-
- if (*clean_count == 0)
- return 0;
- if (*clean_count > cq->ring_size)
- return -EBADR;
-
- spin_lock(&cq->cq_lock);
-
- ntc = cq->next_to_clean;
-
- num_to_clean = *clean_count;
-
- for (i = 0; i < num_to_clean; i++) {
- /* Fetch next descriptor and check if marked as done */
- desc = IDPF_CTLQ_DESC(cq, ntc);
- if (!(le16_to_cpu(desc->flags) & IDPF_CTLQ_FLAG_DD))
- break;
-
- /* Ensure no other fields are read until DD flag is checked */
- dma_rmb();
-
- /* strip off FW internal code */
- desc_err = le16_to_cpu(desc->ret_val) & 0xff;
-
- msg_status[i] = cq->bi.tx_msg[ntc];
- msg_status[i]->status = desc_err;
-
- cq->bi.tx_msg[ntc] = NULL;
-
- /* Zero out any stale data */
- memset(desc, 0, sizeof(*desc));
-
- ntc++;
- if (ntc == cq->ring_size)
- ntc = 0;
- }
-
- cq->next_to_clean = ntc;
-
- spin_unlock(&cq->cq_lock);
-
- /* Return number of descriptors actually cleaned */
- *clean_count = i;
-
- return 0;
-}
-
-/**
- * idpf_ctlq_post_rx_buffs - post buffers to descriptor ring
- * @hw: pointer to hw struct
- * @cq: pointer to control queue handle
- * @buff_count: (input|output) input is number of buffers caller is trying to
- * return; output is number of buffers that were not posted
- * @buffs: array of pointers to dma mem structs to be given to hardware
- *
- * Caller uses this function to return DMA buffers to the descriptor ring after
- * consuming them; buff_count will be the number of buffers.
- *
- * Note: this function needs to be called after a receive call even
- * if there are no DMA buffers to be returned, i.e. buff_count = 0,
- * buffs = NULL to support direct commands
- */
-int idpf_ctlq_post_rx_buffs(struct idpf_hw *hw, struct idpf_ctlq_info *cq,
- u16 *buff_count, struct idpf_dma_mem **buffs)
-{
- struct idpf_ctlq_desc *desc;
- u16 ntp = cq->next_to_post;
- bool buffs_avail = false;
- u16 tbp = ntp + 1;
- int i = 0;
-
- if (*buff_count > cq->ring_size)
- return -EBADR;
-
- if (*buff_count > 0)
- buffs_avail = true;
-
- spin_lock(&cq->cq_lock);
-
- if (tbp >= cq->ring_size)
- tbp = 0;
-
- if (tbp == cq->next_to_clean)
- /* Nothing to do */
- goto post_buffs_out;
-
- /* Post buffers for as many as provided or up until the last one used */
- while (ntp != cq->next_to_clean) {
- desc = IDPF_CTLQ_DESC(cq, ntp);
-
- if (cq->bi.rx_buff[ntp])
- goto fill_desc;
- if (!buffs_avail) {
- /* If the caller hasn't given us any buffers or
- * there are none left, search the ring itself
- * for an available buffer to move to this
- * entry starting at the next entry in the ring
- */
- tbp = ntp + 1;
-
- /* Wrap ring if necessary */
- if (tbp >= cq->ring_size)
- tbp = 0;
-
- while (tbp != cq->next_to_clean) {
- if (cq->bi.rx_buff[tbp]) {
- cq->bi.rx_buff[ntp] =
- cq->bi.rx_buff[tbp];
- cq->bi.rx_buff[tbp] = NULL;
-
- /* Found a buffer, no need to
- * search anymore
- */
- break;
- }
-
- /* Wrap ring if necessary */
- tbp++;
- if (tbp >= cq->ring_size)
- tbp = 0;
- }
-
- if (tbp == cq->next_to_clean)
- goto post_buffs_out;
- } else {
- /* Give back pointer to DMA buffer */
- cq->bi.rx_buff[ntp] = buffs[i];
- i++;
-
- if (i >= *buff_count)
- buffs_avail = false;
- }
-
-fill_desc:
- desc->flags =
- cpu_to_le16(IDPF_CTLQ_FLAG_BUF | IDPF_CTLQ_FLAG_RD);
-
- /* Post buffers to descriptor */
- desc->datalen = cpu_to_le16(cq->bi.rx_buff[ntp]->size);
- desc->params.indirect.addr_high =
- cpu_to_le32(upper_32_bits(cq->bi.rx_buff[ntp]->pa));
- desc->params.indirect.addr_low =
- cpu_to_le32(lower_32_bits(cq->bi.rx_buff[ntp]->pa));
-
- ntp++;
- if (ntp == cq->ring_size)
- ntp = 0;
- }
-
-post_buffs_out:
- /* Only update tail if buffers were actually posted */
- if (cq->next_to_post != ntp) {
- if (ntp)
- /* Update next_to_post to ntp - 1 since current ntp
- * will not have a buffer
- */
- cq->next_to_post = ntp - 1;
- else
- /* Wrap to end of end ring since current ntp is 0 */
- cq->next_to_post = cq->ring_size - 1;
-
- dma_wmb();
-
- writel(cq->next_to_post,
- libie_pci_get_mmio_addr(&hw->back->ctlq_ctx.mmio_info,
- cq->reg.tail));
- }
-
- spin_unlock(&cq->cq_lock);
-
- /* return the number of buffers that were not posted */
- *buff_count = *buff_count - i;
-
- return 0;
-}
-
-/**
- * idpf_ctlq_recv - receive control queue message call back
- * @cq: pointer to control queue handle to receive on
- * @num_q_msg: (input|output) input number of messages that should be received;
- * output number of messages actually received
- * @q_msg: (output) array of received control queue messages on this q;
- * needs to be pre-allocated by caller for as many messages as requested
- *
- * Called by interrupt handler or polling mechanism. Caller is expected
- * to free buffers
- */
-int idpf_ctlq_recv(struct idpf_ctlq_info *cq, u16 *num_q_msg,
- struct idpf_ctlq_msg *q_msg)
-{
- u16 num_to_clean, ntc, flags;
- struct idpf_ctlq_desc *desc;
- int err = 0;
- u16 i;
-
- /* take the lock before we start messing with the ring */
- spin_lock(&cq->cq_lock);
-
- ntc = cq->next_to_clean;
-
- num_to_clean = *num_q_msg;
-
- for (i = 0; i < num_to_clean; i++) {
- /* Fetch next descriptor and check if marked as done */
- desc = IDPF_CTLQ_DESC(cq, ntc);
- flags = le16_to_cpu(desc->flags);
-
- if (!(flags & IDPF_CTLQ_FLAG_DD))
- break;
-
- /* Ensure no other fields are read until DD flag is checked */
- dma_rmb();
-
- q_msg[i].vmvf_type = (flags &
- (IDPF_CTLQ_FLAG_FTYPE_VM |
- IDPF_CTLQ_FLAG_FTYPE_PF)) >>
- IDPF_CTLQ_FLAG_FTYPE_S;
-
- if (flags & IDPF_CTLQ_FLAG_ERR)
- err = -EBADMSG;
-
- q_msg[i].cookie.mbx.chnl_opcode =
- le32_to_cpu(desc->v_opcode_dtype);
- q_msg[i].cookie.mbx.chnl_retval =
- le32_to_cpu(desc->v_retval);
-
- q_msg[i].opcode = le16_to_cpu(desc->opcode);
- q_msg[i].data_len = le16_to_cpu(desc->datalen);
- q_msg[i].status = le16_to_cpu(desc->ret_val);
-
- if (desc->datalen) {
- memcpy(q_msg[i].ctx.indirect.context,
- &desc->params.indirect, IDPF_INDIRECT_CTX_SIZE);
-
- /* Assign pointer to dma buffer to ctlq_msg array
- * to be given to upper layer
- */
- q_msg[i].ctx.indirect.payload = cq->bi.rx_buff[ntc];
-
- /* Zero out pointer to DMA buffer info;
- * will be repopulated by post buffers API
- */
- cq->bi.rx_buff[ntc] = NULL;
- } else {
- memcpy(q_msg[i].ctx.direct, desc->params.raw,
- IDPF_DIRECT_CTX_SIZE);
- }
-
- /* Zero out stale data in descriptor */
- memset(desc, 0, sizeof(struct idpf_ctlq_desc));
-
- ntc++;
- if (ntc == cq->ring_size)
- ntc = 0;
- }
-
- cq->next_to_clean = ntc;
-
- spin_unlock(&cq->cq_lock);
-
- *num_q_msg = i;
- if (*num_q_msg == 0)
- err = -ENOMSG;
-
- return err;
-}
diff --git a/drivers/net/ethernet/intel/idpf/idpf_controlq.h b/drivers/net/ethernet/intel/idpf/idpf_controlq.h
deleted file mode 100644
index acf595e9265f..000000000000
--- a/drivers/net/ethernet/intel/idpf/idpf_controlq.h
+++ /dev/null
@@ -1,142 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (C) 2023 Intel Corporation */
-
-#ifndef _IDPF_CONTROLQ_H_
-#define _IDPF_CONTROLQ_H_
-
-#include <linux/slab.h>
-
-#include "idpf_controlq_api.h"
-
-/* Maximum buffer length for all control queue types */
-#define IDPF_CTLQ_MAX_BUF_LEN 4096
-
-#define IDPF_CTLQ_DESC(R, i) \
- (&(((struct idpf_ctlq_desc *)((R)->desc_ring.va))[i]))
-
-#define IDPF_CTLQ_DESC_UNUSED(R) \
- ((u16)((((R)->next_to_clean > (R)->next_to_use) ? 0 : (R)->ring_size) + \
- (R)->next_to_clean - (R)->next_to_use - 1))
-
-/* Control Queue default settings */
-#define IDPF_CTRL_SQ_CMD_TIMEOUT 250 /* msecs */
-
-struct idpf_ctlq_desc {
- /* Control queue descriptor flags */
- __le16 flags;
- /* Control queue message opcode */
- __le16 opcode;
- __le16 datalen; /* 0 for direct commands */
- union {
- __le16 ret_val;
- __le16 pfid_vfid;
-#define IDPF_CTLQ_DESC_VF_ID_S 0
-#define IDPF_CTLQ_DESC_VF_ID_M (0x7FF << IDPF_CTLQ_DESC_VF_ID_S)
-#define IDPF_CTLQ_DESC_PF_ID_S 11
-#define IDPF_CTLQ_DESC_PF_ID_M (0x1F << IDPF_CTLQ_DESC_PF_ID_S)
- };
-
- /* Virtchnl message opcode and virtchnl descriptor type
- * v_opcode=[27:0], v_dtype=[31:28]
- */
- __le32 v_opcode_dtype;
- /* Virtchnl return value */
- __le32 v_retval;
- union {
- struct {
- __le32 param0;
- __le32 param1;
- __le32 param2;
- __le32 param3;
- } direct;
- struct {
- __le32 param0;
- __le16 sw_cookie;
- /* Virtchnl flags */
- __le16 v_flags;
- __le32 addr_high;
- __le32 addr_low;
- } indirect;
- u8 raw[16];
- } params;
-};
-
-/* Flags sub-structure
- * |0 |1 |2 |3 |4 |5 |6 |7 |8 |9 |10 |11 |12 |13 |14 |15 |
- * |DD |CMP|ERR| * RSV * |FTYPE | *RSV* |RD |VFC|BUF| HOST_ID |
- */
-/* command flags and offsets */
-#define IDPF_CTLQ_FLAG_DD_S 0
-#define IDPF_CTLQ_FLAG_CMP_S 1
-#define IDPF_CTLQ_FLAG_ERR_S 2
-#define IDPF_CTLQ_FLAG_FTYPE_S 6
-#define IDPF_CTLQ_FLAG_RD_S 10
-#define IDPF_CTLQ_FLAG_VFC_S 11
-#define IDPF_CTLQ_FLAG_BUF_S 12
-#define IDPF_CTLQ_FLAG_HOST_ID_S 13
-
-#define IDPF_CTLQ_FLAG_DD BIT(IDPF_CTLQ_FLAG_DD_S) /* 0x1 */
-#define IDPF_CTLQ_FLAG_CMP BIT(IDPF_CTLQ_FLAG_CMP_S) /* 0x2 */
-#define IDPF_CTLQ_FLAG_ERR BIT(IDPF_CTLQ_FLAG_ERR_S) /* 0x4 */
-#define IDPF_CTLQ_FLAG_FTYPE_VM BIT(IDPF_CTLQ_FLAG_FTYPE_S) /* 0x40 */
-#define IDPF_CTLQ_FLAG_FTYPE_PF BIT(IDPF_CTLQ_FLAG_FTYPE_S + 1) /* 0x80 */
-#define IDPF_CTLQ_FLAG_RD BIT(IDPF_CTLQ_FLAG_RD_S) /* 0x400 */
-#define IDPF_CTLQ_FLAG_VFC BIT(IDPF_CTLQ_FLAG_VFC_S) /* 0x800 */
-#define IDPF_CTLQ_FLAG_BUF BIT(IDPF_CTLQ_FLAG_BUF_S) /* 0x1000 */
-
-/* Host ID is a special field that has 3b and not a 1b flag */
-#define IDPF_CTLQ_FLAG_HOST_ID_M MAKE_MASK(0x7000UL, IDPF_CTLQ_FLAG_HOST_ID_S)
-
-struct idpf_mbxq_desc {
- u8 pad[8]; /* CTLQ flags/opcode/len/retval fields */
- u32 chnl_opcode; /* avoid confusion with desc->opcode */
- u32 chnl_retval; /* ditto for desc->retval */
- u32 pf_vf_id; /* used by CP when sending to PF */
-};
-
-/* Max number of MMIO regions not including the mailbox and rstat regions in
- * the fallback case when the whole bar is mapped.
- */
-#define IDPF_MMIO_MAP_FALLBACK_MAX_REMAINING 3
-
-struct idpf_mmio_reg {
- void __iomem *vaddr;
- resource_size_t addr_start;
- resource_size_t addr_len;
-};
-
-/* Define the driver hardware struct to replace other control structs as needed
- * Align to ctlq_hw_info
- */
-struct idpf_hw {
- /* Array of remaining LAN BAR regions */
- int num_lan_regs;
- struct idpf_mmio_reg *lan_regs;
-
- struct idpf_adapter *back;
-
- /* control queue - send and receive */
- struct idpf_ctlq_info *asq;
- struct idpf_ctlq_info *arq;
-
- /* pci info */
- u16 device_id;
- u16 vendor_id;
- u16 subsystem_device_id;
- u16 subsystem_vendor_id;
- u8 revision_id;
- bool adapter_stopped;
-
- struct list_head cq_list_head;
-};
-
-int idpf_ctlq_alloc_ring_res(struct idpf_hw *hw,
- struct idpf_ctlq_info *cq);
-
-void idpf_ctlq_dealloc_ring_res(struct idpf_hw *hw, struct idpf_ctlq_info *cq);
-
-/* prototype for functions used for dynamic memory allocation */
-void *idpf_alloc_dma_mem(struct idpf_hw *hw, struct idpf_dma_mem *mem,
- u64 size);
-void idpf_free_dma_mem(struct idpf_hw *hw, struct idpf_dma_mem *mem);
-#endif /* _IDPF_CONTROLQ_H_ */
diff --git a/drivers/net/ethernet/intel/idpf/idpf_controlq_api.h b/drivers/net/ethernet/intel/idpf/idpf_controlq_api.h
deleted file mode 100644
index 3414c5f9a831..000000000000
--- a/drivers/net/ethernet/intel/idpf/idpf_controlq_api.h
+++ /dev/null
@@ -1,177 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (C) 2023 Intel Corporation */
-
-#ifndef _IDPF_CONTROLQ_API_H_
-#define _IDPF_CONTROLQ_API_H_
-
-#include "idpf_mem.h"
-
-struct idpf_hw;
-
-/* Used for queue init, response and events */
-enum idpf_ctlq_type {
- IDPF_CTLQ_TYPE_MAILBOX_TX = 0,
- IDPF_CTLQ_TYPE_MAILBOX_RX = 1,
- IDPF_CTLQ_TYPE_CONFIG_TX = 2,
- IDPF_CTLQ_TYPE_CONFIG_RX = 3,
- IDPF_CTLQ_TYPE_EVENT_RX = 4,
- IDPF_CTLQ_TYPE_RDMA_TX = 5,
- IDPF_CTLQ_TYPE_RDMA_RX = 6,
- IDPF_CTLQ_TYPE_RDMA_COMPL = 7
-};
-
-/* Generic Control Queue Structures */
-struct idpf_ctlq_reg {
- /* used for queue tracking */
- u32 head;
- u32 tail;
- /* Below applies only to default mb (if present) */
- u32 len;
- u32 bah;
- u32 bal;
- u32 len_mask;
- u32 len_ena_mask;
- u32 head_mask;
-};
-
-/* Generic queue msg structure */
-struct idpf_ctlq_msg {
- u8 vmvf_type; /* represents the source of the message on recv */
-#define IDPF_VMVF_TYPE_VF 0
-#define IDPF_VMVF_TYPE_VM 1
-#define IDPF_VMVF_TYPE_PF 2
- u8 host_id;
- /* 3b field used only when sending a message to CP - to be used in
- * combination with target func_id to route the message
- */
-#define IDPF_HOST_ID_MASK 0x7
-
- u16 opcode;
- u16 data_len; /* data_len = 0 when no payload is attached */
- union {
- u16 func_id; /* when sending a message */
- u16 status; /* when receiving a message */
- };
- union {
- struct {
- u32 chnl_opcode;
- u32 chnl_retval;
- } mbx;
- } cookie;
- union {
-#define IDPF_DIRECT_CTX_SIZE 16
-#define IDPF_INDIRECT_CTX_SIZE 8
- /* 16 bytes of context can be provided or 8 bytes of context
- * plus the address of a DMA buffer
- */
- u8 direct[IDPF_DIRECT_CTX_SIZE];
- struct {
- u8 context[IDPF_INDIRECT_CTX_SIZE];
- struct idpf_dma_mem *payload;
- } indirect;
- struct {
- u32 rsvd;
- u16 data;
- u16 flags;
- } sw_cookie;
- } ctx;
-};
-
-/* Generic queue info structures */
-/* MB, CONFIG and EVENT q do not have extended info */
-struct idpf_ctlq_create_info {
- enum idpf_ctlq_type type;
- int id; /* absolute queue offset passed as input
- * -1 for default mailbox if present
- */
- u16 len; /* Queue length passed as input */
- u16 buf_size; /* buffer size passed as input */
- u64 base_address; /* output, HPA of the Queue start */
- struct idpf_ctlq_reg reg; /* registers accessed by ctlqs */
-
- int ext_info_size;
- void *ext_info; /* Specific to q type */
-};
-
-/* Control Queue information */
-struct idpf_ctlq_info {
- struct list_head cq_list;
-
- enum idpf_ctlq_type cq_type;
- int q_id;
- spinlock_t cq_lock; /* control queue lock */
- /* used for interrupt processing */
- u16 next_to_use;
- u16 next_to_clean;
- u16 next_to_post; /* starting descriptor to post buffers
- * to after recev
- */
-
- struct idpf_dma_mem desc_ring; /* descriptor ring memory
- * idpf_dma_mem is defined in OSdep.h
- */
- union {
- struct idpf_dma_mem **rx_buff;
- struct idpf_ctlq_msg **tx_msg;
- } bi;
-
- u16 buf_size; /* queue buffer size */
- u16 ring_size; /* Number of descriptors */
- struct idpf_ctlq_reg reg; /* registers accessed by ctlqs */
-};
-
-/**
- * enum idpf_mbx_opc - PF/VF mailbox commands
- * @idpf_mbq_opc_send_msg_to_cp: used by PF or VF to send a message to its CP
- * @idpf_mbq_opc_send_msg_to_peer_drv: used by PF or VF to send a message to
- * any peer driver
- */
-enum idpf_mbx_opc {
- idpf_mbq_opc_send_msg_to_cp = 0x0801,
- idpf_mbq_opc_send_msg_to_peer_drv = 0x0804,
-};
-
-/* API supported for control queue management */
-/* Will init all required q including default mb. "q_info" is an array of
- * create_info structs equal to the number of control queues to be created.
- */
-int idpf_ctlq_init(struct idpf_hw *hw, u8 num_q,
- struct idpf_ctlq_create_info *q_info);
-
-/* Allocate and initialize a single control queue, which will be added to the
- * control queue list; returns a handle to the created control queue
- */
-int idpf_ctlq_add(struct idpf_hw *hw,
- struct idpf_ctlq_create_info *qinfo,
- struct idpf_ctlq_info **cq);
-
-/* Deinitialize and deallocate a single control queue */
-void idpf_ctlq_remove(struct idpf_hw *hw,
- struct idpf_ctlq_info *cq);
-
-/* Sends messages to HW and will also free the buffer*/
-int idpf_ctlq_send(struct idpf_hw *hw,
- struct idpf_ctlq_info *cq,
- u16 num_q_msg,
- struct idpf_ctlq_msg q_msg[]);
-
-/* Receives messages and called by interrupt handler/polling
- * initiated by app/process. Also caller is supposed to free the buffers
- */
-int idpf_ctlq_recv(struct idpf_ctlq_info *cq, u16 *num_q_msg,
- struct idpf_ctlq_msg *q_msg);
-
-/* Reclaims send descriptors on HW write back */
-int idpf_ctlq_clean_sq(struct idpf_ctlq_info *cq, u16 *clean_count,
- struct idpf_ctlq_msg *msg_status[]);
-
-/* Indicate RX buffers are done being processed */
-int idpf_ctlq_post_rx_buffs(struct idpf_hw *hw,
- struct idpf_ctlq_info *cq,
- u16 *buff_count,
- struct idpf_dma_mem **buffs);
-
-/* Will destroy all q including the default mb */
-void idpf_ctlq_deinit(struct idpf_hw *hw);
-
-#endif /* _IDPF_CONTROLQ_API_H_ */
diff --git a/drivers/net/ethernet/intel/idpf/idpf_controlq_setup.c b/drivers/net/ethernet/intel/idpf/idpf_controlq_setup.c
deleted file mode 100644
index d4d488c7cfd6..000000000000
--- a/drivers/net/ethernet/intel/idpf/idpf_controlq_setup.c
+++ /dev/null
@@ -1,169 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/* Copyright (C) 2023 Intel Corporation */
-
-#include "idpf_controlq.h"
-
-/**
- * idpf_ctlq_alloc_desc_ring - Allocate Control Queue (CQ) rings
- * @hw: pointer to hw struct
- * @cq: pointer to the specific Control queue
- */
-static int idpf_ctlq_alloc_desc_ring(struct idpf_hw *hw,
- struct idpf_ctlq_info *cq)
-{
- size_t size = cq->ring_size * sizeof(struct idpf_ctlq_desc);
-
- cq->desc_ring.va = idpf_alloc_dma_mem(hw, &cq->desc_ring, size);
- if (!cq->desc_ring.va)
- return -ENOMEM;
-
- return 0;
-}
-
-/**
- * idpf_ctlq_alloc_bufs - Allocate Control Queue (CQ) buffers
- * @hw: pointer to hw struct
- * @cq: pointer to the specific Control queue
- *
- * Allocate the buffer head for all control queues, and if it's a receive
- * queue, allocate DMA buffers
- */
-static int idpf_ctlq_alloc_bufs(struct idpf_hw *hw,
- struct idpf_ctlq_info *cq)
-{
- int i;
-
- /* Do not allocate DMA buffers for transmit queues */
- if (cq->cq_type == IDPF_CTLQ_TYPE_MAILBOX_TX)
- return 0;
-
- /* We'll be allocating the buffer info memory first, then we can
- * allocate the mapped buffers for the event processing
- */
- cq->bi.rx_buff = kzalloc_objs(struct idpf_dma_mem *, cq->ring_size);
- if (!cq->bi.rx_buff)
- return -ENOMEM;
-
- /* allocate the mapped buffers (except for the last one) */
- for (i = 0; i < cq->ring_size - 1; i++) {
- struct idpf_dma_mem *bi;
- int num = 1; /* number of idpf_dma_mem to be allocated */
-
- cq->bi.rx_buff[i] = kzalloc_objs(struct idpf_dma_mem, num);
- if (!cq->bi.rx_buff[i])
- goto unwind_alloc_cq_bufs;
-
- bi = cq->bi.rx_buff[i];
-
- bi->va = idpf_alloc_dma_mem(hw, bi, cq->buf_size);
- if (!bi->va) {
- /* unwind will not free the failed entry */
- kfree(cq->bi.rx_buff[i]);
- goto unwind_alloc_cq_bufs;
- }
- }
-
- return 0;
-
-unwind_alloc_cq_bufs:
- /* don't try to free the one that failed... */
- i--;
- for (; i >= 0; i--) {
- idpf_free_dma_mem(hw, cq->bi.rx_buff[i]);
- kfree(cq->bi.rx_buff[i]);
- }
- kfree(cq->bi.rx_buff);
-
- return -ENOMEM;
-}
-
-/**
- * idpf_ctlq_free_desc_ring - Free Control Queue (CQ) rings
- * @hw: pointer to hw struct
- * @cq: pointer to the specific Control queue
- *
- * This assumes the posted send buffers have already been cleaned
- * and de-allocated
- */
-static void idpf_ctlq_free_desc_ring(struct idpf_hw *hw,
- struct idpf_ctlq_info *cq)
-{
- idpf_free_dma_mem(hw, &cq->desc_ring);
-}
-
-/**
- * idpf_ctlq_free_bufs - Free CQ buffer info elements
- * @hw: pointer to hw struct
- * @cq: pointer to the specific Control queue
- *
- * Free the DMA buffers for RX queues, and DMA buffer header for both RX and TX
- * queues. The upper layers are expected to manage freeing of TX DMA buffers
- */
-static void idpf_ctlq_free_bufs(struct idpf_hw *hw, struct idpf_ctlq_info *cq)
-{
- void *bi;
-
- if (cq->cq_type == IDPF_CTLQ_TYPE_MAILBOX_RX) {
- int i;
-
- /* free DMA buffers for rx queues*/
- for (i = 0; i < cq->ring_size; i++) {
- if (cq->bi.rx_buff[i]) {
- idpf_free_dma_mem(hw, cq->bi.rx_buff[i]);
- kfree(cq->bi.rx_buff[i]);
- }
- }
-
- bi = (void *)cq->bi.rx_buff;
- } else {
- bi = (void *)cq->bi.tx_msg;
- }
-
- /* free the buffer header */
- kfree(bi);
-}
-
-/**
- * idpf_ctlq_dealloc_ring_res - Free memory allocated for control queue
- * @hw: pointer to hw struct
- * @cq: pointer to the specific Control queue
- *
- * Free the memory used by the ring, buffers and other related structures
- */
-void idpf_ctlq_dealloc_ring_res(struct idpf_hw *hw, struct idpf_ctlq_info *cq)
-{
- /* free ring buffers and the ring itself */
- idpf_ctlq_free_bufs(hw, cq);
- idpf_ctlq_free_desc_ring(hw, cq);
-}
-
-/**
- * idpf_ctlq_alloc_ring_res - allocate memory for descriptor ring and bufs
- * @hw: pointer to hw struct
- * @cq: pointer to control queue struct
- *
- * Do *NOT* hold cq_lock when calling this as the memory allocation routines
- * called are not going to be atomic context safe
- */
-int idpf_ctlq_alloc_ring_res(struct idpf_hw *hw, struct idpf_ctlq_info *cq)
-{
- int err;
-
- /* allocate the ring memory */
- err = idpf_ctlq_alloc_desc_ring(hw, cq);
- if (err)
- return err;
-
- /* allocate buffers in the rings */
- err = idpf_ctlq_alloc_bufs(hw, cq);
- if (err)
- goto idpf_init_cq_free_ring;
-
- /* success! */
- return 0;
-
-idpf_init_cq_free_ring:
- idpf_free_dma_mem(hw, &cq->desc_ring);
-
- return err;
-}
diff --git a/drivers/net/ethernet/intel/idpf/idpf_dev.c b/drivers/net/ethernet/intel/idpf/idpf_dev.c
index e36b0017186f..3a357d5dea20 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_dev.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_dev.c
@@ -10,44 +10,32 @@
/**
* idpf_ctlq_reg_init - initialize default mailbox registers
- * @adapter: adapter structure
- * @cq: pointer to the array of create control queues
+ * @mmio: struct that contains MMIO region info
+ * @cci: struct where the register offset pointer to be copied to
*/
-static void idpf_ctlq_reg_init(struct idpf_adapter *adapter,
- struct idpf_ctlq_create_info *cq)
+static void idpf_ctlq_reg_init(struct libie_mmio_info *mmio,
+ struct libie_ctlq_create_info *cci)
{
- int i;
-
- for (i = 0; i < IDPF_NUM_DFLT_MBX_Q; i++) {
- struct idpf_ctlq_create_info *ccq = cq + i;
-
- switch (ccq->type) {
- case IDPF_CTLQ_TYPE_MAILBOX_TX:
- /* set head and tail registers in our local struct */
- ccq->reg.head = PF_FW_ATQH;
- ccq->reg.tail = PF_FW_ATQT;
- ccq->reg.len = PF_FW_ATQLEN;
- ccq->reg.bah = PF_FW_ATQBAH;
- ccq->reg.bal = PF_FW_ATQBAL;
- ccq->reg.len_mask = PF_FW_ATQLEN_ATQLEN_M;
- ccq->reg.len_ena_mask = PF_FW_ATQLEN_ATQENABLE_M;
- ccq->reg.head_mask = PF_FW_ATQH_ATQH_M;
- break;
- case IDPF_CTLQ_TYPE_MAILBOX_RX:
- /* set head and tail registers in our local struct */
- ccq->reg.head = PF_FW_ARQH;
- ccq->reg.tail = PF_FW_ARQT;
- ccq->reg.len = PF_FW_ARQLEN;
- ccq->reg.bah = PF_FW_ARQBAH;
- ccq->reg.bal = PF_FW_ARQBAL;
- ccq->reg.len_mask = PF_FW_ARQLEN_ARQLEN_M;
- ccq->reg.len_ena_mask = PF_FW_ARQLEN_ARQENABLE_M;
- ccq->reg.head_mask = PF_FW_ARQH_ARQH_M;
- break;
- default:
- break;
- }
- }
+ struct libie_ctlq_reg *tx_reg = &cci[LIBIE_CTLQ_TYPE_TX].reg;
+ struct libie_ctlq_reg *rx_reg = &cci[LIBIE_CTLQ_TYPE_RX].reg;
+
+ tx_reg->head = libie_pci_get_mmio_addr(mmio, PF_FW_ATQH);
+ tx_reg->tail = libie_pci_get_mmio_addr(mmio, PF_FW_ATQT);
+ tx_reg->len = libie_pci_get_mmio_addr(mmio, PF_FW_ATQLEN);
+ tx_reg->addr_high = libie_pci_get_mmio_addr(mmio, PF_FW_ATQBAH);
+ tx_reg->addr_low = libie_pci_get_mmio_addr(mmio, PF_FW_ATQBAL);
+ tx_reg->len_mask = PF_FW_ATQLEN_ATQLEN_M;
+ tx_reg->len_ena_mask = PF_FW_ATQLEN_ATQENABLE_M;
+ tx_reg->head_mask = PF_FW_ATQH_ATQH_M;
+
+ rx_reg->head = libie_pci_get_mmio_addr(mmio, PF_FW_ARQH);
+ rx_reg->tail = libie_pci_get_mmio_addr(mmio, PF_FW_ARQT);
+ rx_reg->len = libie_pci_get_mmio_addr(mmio, PF_FW_ARQLEN);
+ rx_reg->addr_high = libie_pci_get_mmio_addr(mmio, PF_FW_ARQBAH);
+ rx_reg->addr_low = libie_pci_get_mmio_addr(mmio, PF_FW_ARQBAL);
+ rx_reg->len_mask = PF_FW_ARQLEN_ARQLEN_M;
+ rx_reg->len_ena_mask = PF_FW_ARQLEN_ARQENABLE_M;
+ rx_reg->head_mask = PF_FW_ARQH_ARQH_M;
}
/**
diff --git a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c
index bb99d9e7c65d..95c45f12b0f9 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c
@@ -225,7 +225,7 @@ static int idpf_add_flow_steer(struct net_device *netdev,
spin_unlock_bh(&vport_config->flow_steer_list_lock);
if (err)
- goto out;
+ goto out_free_fltr;
rule->vport_id = cpu_to_le32(vport->vport_id);
rule->count = cpu_to_le32(1);
@@ -252,17 +252,15 @@ static int idpf_add_flow_steer(struct net_device *netdev,
break;
default:
err = -EINVAL;
- goto out;
+ goto out_free_fltr;
}
err = idpf_add_del_fsteer_filters(vport->adapter, rule,
VIRTCHNL2_OP_ADD_FLOW_RULE);
- if (err)
- goto out;
-
- if (info->status != cpu_to_le32(VIRTCHNL2_FLOW_RULE_SUCCESS)) {
- err = -EIO;
- goto out;
+ if (err) {
+ /* virtchnl2 rule is already consumed */
+ kfree(fltr);
+ return err;
}
/* Save a copy of the user's flow spec so ethtool can later retrieve it */
@@ -274,9 +272,10 @@ static int idpf_add_flow_steer(struct net_device *netdev,
user_config->num_fsteer_fltrs++;
spin_unlock_bh(&vport_config->flow_steer_list_lock);
- goto out_free_rule;
-out:
+ return 0;
+
+out_free_fltr:
kfree(fltr);
out_free_rule:
kfree(rule);
@@ -319,12 +318,7 @@ static int idpf_del_flow_steer(struct net_device *netdev,
err = idpf_add_del_fsteer_filters(vport->adapter, rule,
VIRTCHNL2_OP_DEL_FLOW_RULE);
if (err)
- goto out;
-
- if (info->status != cpu_to_le32(VIRTCHNL2_FLOW_RULE_SUCCESS)) {
- err = -EIO;
- goto out;
- }
+ return err;
spin_lock_bh(&vport_config->flow_steer_list_lock);
list_for_each_entry_safe(f, iter,
@@ -340,8 +334,6 @@ static int idpf_del_flow_steer(struct net_device *netdev,
out_unlock:
spin_unlock_bh(&vport_config->flow_steer_list_lock);
-out:
- kfree(rule);
return err;
}
diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c
index 875472ae77fd..0d131bf0993e 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_lib.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c
@@ -1371,6 +1371,7 @@ void idpf_statistics_task(struct work_struct *work)
*/
void idpf_mbx_task(struct work_struct *work)
{
+ struct libie_ctlq_xn_recv_params xn_params;
struct idpf_adapter *adapter;
adapter = container_of(work, struct idpf_adapter, mbx_task.work);
@@ -1381,7 +1382,14 @@ void idpf_mbx_task(struct work_struct *work)
queue_delayed_work(adapter->mbx_wq, &adapter->mbx_task,
usecs_to_jiffies(300));
- idpf_recv_mb_msg(adapter, adapter->hw.arq);
+ xn_params = (struct libie_ctlq_xn_recv_params) {
+ .xnm = adapter->xnm,
+ .ctlq = adapter->arq,
+ .ctlq_msg_handler = idpf_recv_event_msg,
+ .budget = LIBIE_CTLQ_MAX_XN_ENTRIES,
+ };
+
+ libie_ctlq_xn_recv(&xn_params);
}
/**
@@ -1909,7 +1917,6 @@ static void idpf_init_hard_reset(struct idpf_adapter *adapter)
idpf_vc_core_deinit(adapter);
if (!is_reset)
reg_ops->trigger_reset(adapter, IDPF_HR_FUNC_RESET);
- idpf_deinit_dflt_mbx(adapter);
} else {
dev_err(dev, "Unhandled hard reset cause\n");
err = -EBADRQC;
@@ -1984,7 +1991,7 @@ void idpf_vc_event_task(struct work_struct *work)
return;
func_reset:
- idpf_vc_xn_shutdown(adapter->vcxn_mngr);
+ libie_ctlq_xn_shutdown(adapter->xnm);
drv_load:
set_bit(IDPF_HR_RESET_IN_PROG, adapter->flags);
idpf_init_hard_reset(adapter);
@@ -2567,44 +2574,6 @@ static int idpf_set_mac(struct net_device *netdev, void *p)
return err;
}
-/**
- * idpf_alloc_dma_mem - Allocate dma memory
- * @hw: pointer to hw struct
- * @mem: pointer to dma_mem struct
- * @size: size of the memory to allocate
- */
-void *idpf_alloc_dma_mem(struct idpf_hw *hw, struct idpf_dma_mem *mem, u64 size)
-{
- struct idpf_adapter *adapter = hw->back;
- size_t sz = ALIGN(size, 4096);
-
- /* The control queue resources are freed under a spinlock, contiguous
- * pages will avoid IOMMU remapping and the use vmap (and vunmap in
- * dma_free_*() path.
- */
- mem->va = dma_alloc_attrs(&adapter->pdev->dev, sz, &mem->pa,
- GFP_KERNEL, DMA_ATTR_FORCE_CONTIGUOUS);
- mem->size = sz;
-
- return mem->va;
-}
-
-/**
- * idpf_free_dma_mem - Free the allocated dma memory
- * @hw: pointer to hw struct
- * @mem: pointer to dma_mem struct
- */
-void idpf_free_dma_mem(struct idpf_hw *hw, struct idpf_dma_mem *mem)
-{
- struct idpf_adapter *adapter = hw->back;
-
- dma_free_attrs(&adapter->pdev->dev, mem->size,
- mem->va, mem->pa, DMA_ATTR_FORCE_CONTIGUOUS);
- mem->size = 0;
- mem->va = NULL;
- mem->pa = 0;
-}
-
static int idpf_hwtstamp_set(struct net_device *netdev,
struct kernel_hwtstamp_config *config,
struct netlink_ext_ack *extack)
diff --git a/drivers/net/ethernet/intel/idpf/idpf_main.c b/drivers/net/ethernet/intel/idpf/idpf_main.c
index 93b11fb1609f..db91039c54d0 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_main.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_main.c
@@ -133,7 +133,6 @@ static void idpf_remove(struct pci_dev *pdev)
/* Be a good citizen and leave the device clean on exit */
adapter->dev_ops.reg_ops.trigger_reset(adapter, IDPF_HR_FUNC_RESET);
- idpf_deinit_dflt_mbx(adapter);
if (!adapter->netdevs)
goto destroy_wqs;
@@ -170,8 +169,6 @@ static void idpf_remove(struct pci_dev *pdev)
adapter->vport_config = NULL;
kfree(adapter->netdevs);
adapter->netdevs = NULL;
- kfree(adapter->vcxn_mngr);
- adapter->vcxn_mngr = NULL;
mutex_destroy(&adapter->vport_ctrl_lock);
mutex_destroy(&adapter->vector_lock);
@@ -194,7 +191,6 @@ static void idpf_shutdown(struct pci_dev *pdev)
cancel_delayed_work_sync(&adapter->serv_task);
cancel_delayed_work_sync(&adapter->vc_event_task);
idpf_vc_core_deinit(adapter);
- idpf_deinit_dflt_mbx(adapter);
if (system_state == SYSTEM_POWER_OFF)
pci_set_power_state(pdev, PCI_D3hot);
@@ -239,7 +235,6 @@ static int idpf_cfg_device(struct idpf_adapter *adapter)
pci_dbg(pdev, "PCIe PTM is not supported by PCIe bus/controller\n");
pci_set_drvdata(pdev, adapter);
- adapter->hw.back = adapter;
return 0;
}
diff --git a/drivers/net/ethernet/intel/idpf/idpf_mem.h b/drivers/net/ethernet/intel/idpf/idpf_mem.h
deleted file mode 100644
index 2aaabdc02dd2..000000000000
--- a/drivers/net/ethernet/intel/idpf/idpf_mem.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (C) 2023 Intel Corporation */
-
-#ifndef _IDPF_MEM_H_
-#define _IDPF_MEM_H_
-
-#include <linux/io.h>
-
-struct idpf_dma_mem {
- void *va;
- dma_addr_t pa;
- size_t size;
-};
-
-#define idpf_mbx_wr32(a, reg, value) writel((value), ((a)->mbx.vaddr + (reg)))
-#define idpf_mbx_rd32(a, reg) readl((a)->mbx.vaddr + (reg))
-#define idpf_mbx_wr64(a, reg, value) writeq((value), ((a)->mbx.vaddr + (reg)))
-#define idpf_mbx_rd64(a, reg) readq((a)->mbx.vaddr + (reg))
-
-#endif /* _IDPF_MEM_H_ */
diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h
index 08b048664029..068c432ee833 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h
+++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h
@@ -236,7 +236,7 @@ enum idpf_tx_ctx_desc_eipt_offload {
(sizeof(u16) * IDPF_RX_MAX_PTYPE_PROTO_IDS))
#define IDPF_RX_PTYPE_HDR_SZ sizeof(struct virtchnl2_get_ptype_info)
#define IDPF_RX_MAX_PTYPES_PER_BUF \
- DIV_ROUND_DOWN_ULL((IDPF_CTLQ_MAX_BUF_LEN - IDPF_RX_PTYPE_HDR_SZ), \
+ DIV_ROUND_DOWN_ULL(LIBIE_CTLQ_MAX_BUF_LEN - IDPF_RX_PTYPE_HDR_SZ, \
IDPF_RX_MAX_PTYPE_SZ)
#define IDPF_GET_PTYPE_SIZE(p) struct_size((p), proto_id, (p)->proto_id_count)
diff --git a/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c b/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c
index 98b8f678bd9a..3dafe680b701 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c
@@ -9,42 +9,32 @@
/**
* idpf_vf_ctlq_reg_init - initialize default mailbox registers
- * @adapter: adapter structure
- * @cq: pointer to the array of create control queues
+ * @mmio: struct that contains MMIO region info
+ * @cci: struct where the register offset pointer to be copied to
*/
-static void idpf_vf_ctlq_reg_init(struct idpf_adapter *adapter,
- struct idpf_ctlq_create_info *cq)
+static void idpf_vf_ctlq_reg_init(struct libie_mmio_info *mmio,
+ struct libie_ctlq_create_info *cci)
{
- for (int i = 0; i < IDPF_NUM_DFLT_MBX_Q; i++) {
- struct idpf_ctlq_create_info *ccq = cq + i;
-
- switch (ccq->type) {
- case IDPF_CTLQ_TYPE_MAILBOX_TX:
- /* set head and tail registers in our local struct */
- ccq->reg.head = VF_ATQH;
- ccq->reg.tail = VF_ATQT;
- ccq->reg.len = VF_ATQLEN;
- ccq->reg.bah = VF_ATQBAH;
- ccq->reg.bal = VF_ATQBAL;
- ccq->reg.len_mask = VF_ATQLEN_ATQLEN_M;
- ccq->reg.len_ena_mask = VF_ATQLEN_ATQENABLE_M;
- ccq->reg.head_mask = VF_ATQH_ATQH_M;
- break;
- case IDPF_CTLQ_TYPE_MAILBOX_RX:
- /* set head and tail registers in our local struct */
- ccq->reg.head = VF_ARQH;
- ccq->reg.tail = VF_ARQT;
- ccq->reg.len = VF_ARQLEN;
- ccq->reg.bah = VF_ARQBAH;
- ccq->reg.bal = VF_ARQBAL;
- ccq->reg.len_mask = VF_ARQLEN_ARQLEN_M;
- ccq->reg.len_ena_mask = VF_ARQLEN_ARQENABLE_M;
- ccq->reg.head_mask = VF_ARQH_ARQH_M;
- break;
- default:
- break;
- }
- }
+ struct libie_ctlq_reg *tx_reg = &cci[LIBIE_CTLQ_TYPE_TX].reg;
+ struct libie_ctlq_reg *rx_reg = &cci[LIBIE_CTLQ_TYPE_RX].reg;
+
+ tx_reg->head = libie_pci_get_mmio_addr(mmio, VF_ATQH);
+ tx_reg->tail = libie_pci_get_mmio_addr(mmio, VF_ATQT);
+ tx_reg->len = libie_pci_get_mmio_addr(mmio, VF_ATQLEN);
+ tx_reg->addr_high = libie_pci_get_mmio_addr(mmio, VF_ATQBAH);
+ tx_reg->addr_low = libie_pci_get_mmio_addr(mmio, VF_ATQBAL);
+ tx_reg->len_mask = VF_ATQLEN_ATQLEN_M;
+ tx_reg->len_ena_mask = VF_ATQLEN_ATQENABLE_M;
+ tx_reg->head_mask = VF_ATQH_ATQH_M;
+
+ rx_reg->head = libie_pci_get_mmio_addr(mmio, VF_ARQH);
+ rx_reg->tail = libie_pci_get_mmio_addr(mmio, VF_ARQT);
+ rx_reg->len = libie_pci_get_mmio_addr(mmio, VF_ARQLEN);
+ rx_reg->addr_high = libie_pci_get_mmio_addr(mmio, VF_ARQBAH);
+ rx_reg->addr_low = libie_pci_get_mmio_addr(mmio, VF_ARQBAL);
+ rx_reg->len_mask = VF_ARQLEN_ARQLEN_M;
+ rx_reg->len_ena_mask = VF_ARQLEN_ARQENABLE_M;
+ rx_reg->head_mask = VF_ARQH_ARQH_M;
}
/**
@@ -157,11 +147,13 @@ static void idpf_vf_reset_reg_init(struct idpf_adapter *adapter)
static void idpf_vf_trigger_reset(struct idpf_adapter *adapter,
enum idpf_flags trig_cause)
{
+ struct libie_ctlq_xn_send_params xn_params = {
+ .chnl_opcode = VIRTCHNL2_OP_RESET_VF,
+ };
/* Do not send VIRTCHNL2_OP_RESET_VF message on driver unload */
if (trig_cause == IDPF_HR_FUNC_RESET &&
!test_bit(IDPF_REMOVE_IN_PROG, adapter->flags))
- idpf_send_mb_msg(adapter, adapter->hw.asq,
- VIRTCHNL2_OP_RESET_VF, 0, NULL, 0);
+ idpf_send_mb_msg(adapter, &xn_params, NULL, 0);
}
/**
diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c
index 357ff3e86b2e..dff719c4b1a2 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c
@@ -9,20 +9,6 @@
#include "idpf_virtchnl.h"
#include "idpf_ptp.h"
-/**
- * struct idpf_vc_xn_manager - Manager for tracking transactions
- * @ring: backing and lookup for transactions
- * @free_xn_bm: bitmap for free transactions
- * @xn_bm_lock: make bitmap access synchronous where necessary
- * @salt: used to make cookie unique every message
- */
-struct idpf_vc_xn_manager {
- struct idpf_vc_xn ring[IDPF_VC_XN_RING_LEN];
- DECLARE_BITMAP(free_xn_bm, IDPF_VC_XN_RING_LEN);
- spinlock_t xn_bm_lock;
- u8 salt;
-};
-
/**
* idpf_vid_to_vport - Translate vport id to vport pointer
* @adapter: private data struct
@@ -83,79 +69,65 @@ static void idpf_handle_event_link(struct idpf_adapter *adapter,
/**
* idpf_recv_event_msg - Receive virtchnl event message
- * @adapter: Driver specific private structure
+ * @ctx: control queue context
* @ctlq_msg: message to copy from
*
* Receive virtchnl event message
*/
-static void idpf_recv_event_msg(struct idpf_adapter *adapter,
- struct idpf_ctlq_msg *ctlq_msg)
+void idpf_recv_event_msg(struct libie_ctlq_ctx *ctx,
+ struct libie_ctlq_msg *ctlq_msg)
{
- int payload_size = ctlq_msg->ctx.indirect.payload->size;
+ struct kvec *buff = &ctlq_msg->recv_mem;
+ int payload_size = buff->iov_len;
+ struct idpf_adapter *adapter;
struct virtchnl2_event *v2e;
u32 event;
+ adapter = container_of(ctx, struct idpf_adapter, ctlq_ctx);
if (payload_size < sizeof(*v2e)) {
dev_err_ratelimited(&adapter->pdev->dev, "Failed to receive valid payload for event msg (op %d len %d)\n",
- ctlq_msg->cookie.mbx.chnl_opcode,
+ ctlq_msg->chnl_opcode,
payload_size);
- return;
+ goto free_rx_buf;
}
- v2e = (struct virtchnl2_event *)ctlq_msg->ctx.indirect.payload->va;
+ v2e = (struct virtchnl2_event *)buff->iov_base;
event = le32_to_cpu(v2e->event);
switch (event) {
case VIRTCHNL2_EVENT_LINK_CHANGE:
idpf_handle_event_link(adapter, v2e);
- return;
+ break;
default:
dev_err(&adapter->pdev->dev,
"Unknown event %d from PF\n", event);
break;
}
+
+free_rx_buf:
+ libie_ctlq_release_rx_buf(buff);
}
/**
* idpf_mb_clean - Reclaim the send mailbox queue entries
* @adapter: driver specific private structure
* @asq: send control queue info
+ * @deinit: release all buffers before destroying the queue
*
- * Reclaim the send mailbox queue entries to be used to send further messages
- *
- * Return: 0 on success, negative on failure
+ * This is a helper function to clean the send mailbox queue entries.
*/
-static int idpf_mb_clean(struct idpf_adapter *adapter,
- struct idpf_ctlq_info *asq)
+static void idpf_mb_clean(struct idpf_adapter *adapter,
+ struct libie_ctlq_info *asq, bool deinit)
{
- u16 i, num_q_msg = IDPF_DFLT_MBX_Q_LEN;
- struct idpf_ctlq_msg **q_msg;
- struct idpf_dma_mem *dma_mem;
- int err;
-
- q_msg = kzalloc_objs(struct idpf_ctlq_msg *, num_q_msg, GFP_ATOMIC);
- if (!q_msg)
- return -ENOMEM;
-
- err = idpf_ctlq_clean_sq(asq, &num_q_msg, q_msg);
- if (err)
- goto err_kfree;
-
- for (i = 0; i < num_q_msg; i++) {
- if (!q_msg[i])
- continue;
- dma_mem = q_msg[i]->ctx.indirect.payload;
- if (dma_mem)
- dma_free_coherent(&adapter->pdev->dev, dma_mem->size,
- dma_mem->va, dma_mem->pa);
- kfree(q_msg[i]);
- kfree(dma_mem);
- }
-
-err_kfree:
- kfree(q_msg);
+ struct libie_ctlq_xn_clean_params clean_params = {
+ .ctx = &adapter->ctlq_ctx,
+ .ctlq = asq,
+ .rel_tx_buf = kfree,
+ .num_msgs = IDPF_DFLT_MBX_Q_LEN,
+ .force = deinit,
+ };
- return err;
+ libie_ctlq_xn_send_clean(&clean_params);
}
#if IS_ENABLED(CONFIG_PTP_1588_CLOCK)
@@ -189,7 +161,7 @@ static bool idpf_ptp_is_mb_msg(u32 op)
* @ctlq_msg: Corresponding control queue message
*/
static void idpf_prepare_ptp_mb_msg(struct idpf_adapter *adapter, u32 op,
- struct idpf_ctlq_msg *ctlq_msg)
+ struct libie_ctlq_msg *ctlq_msg)
{
/* If the message is PTP-related and the secondary mailbox is available,
* send the message through the secondary mailbox.
@@ -197,528 +169,83 @@ static void idpf_prepare_ptp_mb_msg(struct idpf_adapter *adapter, u32 op,
if (!idpf_ptp_is_mb_msg(op) || !adapter->ptp->secondary_mbx.valid)
return;
- ctlq_msg->opcode = idpf_mbq_opc_send_msg_to_peer_drv;
+ ctlq_msg->opcode = LIBIE_CTLQ_SEND_MSG_TO_PEER;
ctlq_msg->func_id = adapter->ptp->secondary_mbx.peer_mbx_q_id;
- ctlq_msg->host_id = adapter->ptp->secondary_mbx.peer_id;
+ ctlq_msg->flags = FIELD_PREP(LIBIE_CTLQ_DESC_FLAG_HOST_ID,
+ adapter->ptp->secondary_mbx.peer_id);
}
#else /* !CONFIG_PTP_1588_CLOCK */
static void idpf_prepare_ptp_mb_msg(struct idpf_adapter *adapter, u32 op,
- struct idpf_ctlq_msg *ctlq_msg)
+ struct libie_ctlq_msg *ctlq_msg)
{ }
#endif /* CONFIG_PTP_1588_CLOCK */
/**
- * idpf_send_mb_msg - Send message over mailbox
+ * idpf_send_mb_msg - send mailbox message to the device control plane
* @adapter: driver specific private structure
- * @asq: control queue to send message to
- * @op: virtchnl opcode
- * @msg_size: size of the payload
- * @msg: pointer to buffer holding the payload
- * @cookie: unique SW generated cookie per message
- *
- * Will prepare the control queue message and initiates the send api
+ * @xn_params: Xn send parameters to fill
+ * @send_buf: buffer to send
+ * @send_buf_size: size of the send buffer
*
- * Return: 0 on success, negative on failure
- */
-int idpf_send_mb_msg(struct idpf_adapter *adapter, struct idpf_ctlq_info *asq,
- u32 op, u16 msg_size, u8 *msg, u16 cookie)
-{
- struct idpf_ctlq_msg *ctlq_msg;
- struct idpf_dma_mem *dma_mem;
- int err;
-
- /* If we are here and a reset is detected nothing much can be
- * done. This thread should silently abort and expected to
- * be corrected with a new run either by user or driver
- * flows after reset
- */
- if (idpf_is_reset_detected(adapter))
- return 0;
-
- err = idpf_mb_clean(adapter, asq);
- if (err)
- return err;
-
- ctlq_msg = kzalloc_obj(*ctlq_msg, GFP_ATOMIC);
- if (!ctlq_msg)
- return -ENOMEM;
-
- dma_mem = kzalloc_obj(*dma_mem, GFP_ATOMIC);
- if (!dma_mem) {
- err = -ENOMEM;
- goto dma_mem_error;
- }
-
- ctlq_msg->opcode = idpf_mbq_opc_send_msg_to_cp;
- ctlq_msg->func_id = 0;
-
- idpf_prepare_ptp_mb_msg(adapter, op, ctlq_msg);
-
- ctlq_msg->data_len = msg_size;
- ctlq_msg->cookie.mbx.chnl_opcode = op;
- ctlq_msg->cookie.mbx.chnl_retval = 0;
- dma_mem->size = IDPF_CTLQ_MAX_BUF_LEN;
- dma_mem->va = dma_alloc_coherent(&adapter->pdev->dev, dma_mem->size,
- &dma_mem->pa, GFP_ATOMIC);
- if (!dma_mem->va) {
- err = -ENOMEM;
- goto dma_alloc_error;
- }
-
- /* It's possible we're just sending an opcode but no buffer */
- if (msg && msg_size)
- memcpy(dma_mem->va, msg, msg_size);
- ctlq_msg->ctx.indirect.payload = dma_mem;
- ctlq_msg->ctx.sw_cookie.data = cookie;
-
- err = idpf_ctlq_send(&adapter->hw, asq, 1, ctlq_msg);
- if (err)
- goto send_error;
-
- return 0;
-
-send_error:
- dma_free_coherent(&adapter->pdev->dev, dma_mem->size, dma_mem->va,
- dma_mem->pa);
-dma_alloc_error:
- kfree(dma_mem);
-dma_mem_error:
- kfree(ctlq_msg);
-
- return err;
-}
-
-/* API for virtchnl "transaction" support ("xn" for short). */
-
-/**
- * idpf_vc_xn_lock - Request exclusive access to vc transaction
- * @xn: struct idpf_vc_xn* to access
- */
-#define idpf_vc_xn_lock(xn) \
- spin_lock(&(xn)->lock)
-
-/**
- * idpf_vc_xn_unlock - Release exclusive access to vc transaction
- * @xn: struct idpf_vc_xn* to access
- */
-#define idpf_vc_xn_unlock(xn) \
- spin_unlock(&(xn)->lock)
-
-/**
- * idpf_vc_xn_release_bufs - Release reference to reply buffer(s) and
- * reset the transaction state.
- * @xn: struct idpf_vc_xn to update
- */
-static void idpf_vc_xn_release_bufs(struct idpf_vc_xn *xn)
-{
- xn->reply.iov_base = NULL;
- xn->reply.iov_len = 0;
-
- if (xn->state != IDPF_VC_XN_SHUTDOWN)
- xn->state = IDPF_VC_XN_IDLE;
-}
-
-/**
- * idpf_vc_xn_init - Initialize virtchnl transaction object
- * @vcxn_mngr: pointer to vc transaction manager struct
- */
-static void idpf_vc_xn_init(struct idpf_vc_xn_manager *vcxn_mngr)
-{
- int i;
-
- spin_lock_init(&vcxn_mngr->xn_bm_lock);
-
- for (i = 0; i < ARRAY_SIZE(vcxn_mngr->ring); i++) {
- struct idpf_vc_xn *xn = &vcxn_mngr->ring[i];
-
- xn->state = IDPF_VC_XN_IDLE;
- xn->idx = i;
- idpf_vc_xn_release_bufs(xn);
- spin_lock_init(&xn->lock);
- init_completion(&xn->completed);
- }
-
- bitmap_fill(vcxn_mngr->free_xn_bm, IDPF_VC_XN_RING_LEN);
-}
-
-/**
- * idpf_vc_xn_shutdown - Uninitialize virtchnl transaction object
- * @vcxn_mngr: pointer to vc transaction manager struct
- *
- * All waiting threads will be woken-up and their transaction aborted. Further
- * operations on that object will fail.
- */
-void idpf_vc_xn_shutdown(struct idpf_vc_xn_manager *vcxn_mngr)
-{
- int i;
-
- spin_lock_bh(&vcxn_mngr->xn_bm_lock);
- bitmap_zero(vcxn_mngr->free_xn_bm, IDPF_VC_XN_RING_LEN);
- spin_unlock_bh(&vcxn_mngr->xn_bm_lock);
-
- for (i = 0; i < ARRAY_SIZE(vcxn_mngr->ring); i++) {
- struct idpf_vc_xn *xn = &vcxn_mngr->ring[i];
-
- idpf_vc_xn_lock(xn);
- xn->state = IDPF_VC_XN_SHUTDOWN;
- idpf_vc_xn_release_bufs(xn);
- idpf_vc_xn_unlock(xn);
- complete_all(&xn->completed);
- }
-}
-
-/**
- * idpf_vc_xn_pop_free - Pop a free transaction from free list
- * @vcxn_mngr: transaction manager to pop from
- *
- * Returns NULL if no free transactions
- */
-static
-struct idpf_vc_xn *idpf_vc_xn_pop_free(struct idpf_vc_xn_manager *vcxn_mngr)
-{
- struct idpf_vc_xn *xn = NULL;
- unsigned long free_idx;
-
- spin_lock_bh(&vcxn_mngr->xn_bm_lock);
- free_idx = find_first_bit(vcxn_mngr->free_xn_bm, IDPF_VC_XN_RING_LEN);
- if (free_idx == IDPF_VC_XN_RING_LEN)
- goto do_unlock;
-
- clear_bit(free_idx, vcxn_mngr->free_xn_bm);
- xn = &vcxn_mngr->ring[free_idx];
- xn->salt = vcxn_mngr->salt++;
-
-do_unlock:
- spin_unlock_bh(&vcxn_mngr->xn_bm_lock);
-
- return xn;
-}
-
-/**
- * idpf_vc_xn_push_free - Push a free transaction to free list
- * @vcxn_mngr: transaction manager to push to
- * @xn: transaction to push
- */
-static void idpf_vc_xn_push_free(struct idpf_vc_xn_manager *vcxn_mngr,
- struct idpf_vc_xn *xn)
-{
- idpf_vc_xn_release_bufs(xn);
- spin_lock_bh(&vcxn_mngr->xn_bm_lock);
- set_bit(xn->idx, vcxn_mngr->free_xn_bm);
- spin_unlock_bh(&vcxn_mngr->xn_bm_lock);
-}
-
-/**
- * idpf_vc_xn_exec - Perform a send/recv virtchnl transaction
- * @adapter: driver specific private structure with vcxn_mngr
- * @params: parameters for this particular transaction including
- * -vc_op: virtchannel operation to send
- * -send_buf: kvec iov for send buf and len
- * -recv_buf: kvec iov for recv buf and len (ignored if NULL)
- * -timeout_ms: timeout waiting for a reply (milliseconds)
- * -async: don't wait for message reply, will lose caller context
- * -async_handler: callback to handle async replies
+ * Fill the Xn parameters with the required info to send a virtchnl message.
+ * The send buffer is DMA mapped in the libie to avoid memcpy.
*
- * @returns >= 0 for success, the size of the initial reply (may or may not be
- * >= @recv_buf.iov_len, but we never overflow @@recv_buf_iov_base). < 0 for
- * error.
- */
-ssize_t idpf_vc_xn_exec(struct idpf_adapter *adapter,
- const struct idpf_vc_xn_params *params)
-{
- const struct kvec *send_buf = ¶ms->send_buf;
- struct idpf_vc_xn *xn;
- ssize_t retval;
- u16 cookie;
-
- xn = idpf_vc_xn_pop_free(adapter->vcxn_mngr);
- /* no free transactions available */
- if (!xn)
- return -ENOSPC;
-
- idpf_vc_xn_lock(xn);
- if (xn->state == IDPF_VC_XN_SHUTDOWN) {
- retval = -ENXIO;
- goto only_unlock;
- } else if (xn->state != IDPF_VC_XN_IDLE) {
- /* We're just going to clobber this transaction even though
- * it's not IDLE. If we don't reuse it we could theoretically
- * eventually leak all the free transactions and not be able to
- * send any messages. At least this way we make an attempt to
- * remain functional even though something really bad is
- * happening that's corrupting what was supposed to be free
- * transactions.
- */
- WARN_ONCE(1, "There should only be idle transactions in free list (idx %d op %d)\n",
- xn->idx, xn->vc_op);
- }
-
- xn->reply = params->recv_buf;
- xn->reply_sz = 0;
- xn->state = params->async ? IDPF_VC_XN_ASYNC : IDPF_VC_XN_WAITING;
- xn->vc_op = params->vc_op;
- xn->async_handler = params->async_handler;
- idpf_vc_xn_unlock(xn);
-
- if (!params->async)
- reinit_completion(&xn->completed);
- cookie = FIELD_PREP(IDPF_VC_XN_SALT_M, xn->salt) |
- FIELD_PREP(IDPF_VC_XN_IDX_M, xn->idx);
-
- retval = idpf_send_mb_msg(adapter, adapter->hw.asq, params->vc_op,
- send_buf->iov_len, send_buf->iov_base,
- cookie);
- if (retval) {
- idpf_vc_xn_lock(xn);
- goto release_and_unlock;
- }
-
- if (params->async)
- return 0;
-
- wait_for_completion_timeout(&xn->completed,
- msecs_to_jiffies(params->timeout_ms));
-
- /* No need to check the return value; we check the final state of the
- * transaction below. It's possible the transaction actually gets more
- * timeout than specified if we get preempted here but after
- * wait_for_completion_timeout returns. This should be non-issue
- * however.
- */
- idpf_vc_xn_lock(xn);
- switch (xn->state) {
- case IDPF_VC_XN_SHUTDOWN:
- retval = -ENXIO;
- goto only_unlock;
- case IDPF_VC_XN_WAITING:
- dev_notice_ratelimited(&adapter->pdev->dev,
- "Transaction timed-out (op:%d cookie:%04x vc_op:%d salt:%02x timeout:%dms)\n",
- params->vc_op, cookie, xn->vc_op,
- xn->salt, params->timeout_ms);
- retval = -ETIME;
- break;
- case IDPF_VC_XN_COMPLETED_SUCCESS:
- retval = xn->reply_sz;
- break;
- case IDPF_VC_XN_COMPLETED_FAILED:
- dev_notice_ratelimited(&adapter->pdev->dev, "Transaction failed (op %d)\n",
- params->vc_op);
- retval = -EIO;
- break;
- default:
- /* Invalid state. */
- WARN_ON_ONCE(1);
- retval = -EIO;
- break;
- }
-
-release_and_unlock:
- idpf_vc_xn_push_free(adapter->vcxn_mngr, xn);
- /* If we receive a VC reply after here, it will be dropped. */
-only_unlock:
- idpf_vc_xn_unlock(xn);
-
- return retval;
-}
-
-/**
- * idpf_vc_xn_forward_async - Handle async reply receives
- * @adapter: private data struct
- * @xn: transaction to handle
- * @ctlq_msg: corresponding ctlq_msg
+ * Cleanup the mailbox queue entries of the previously sent message to
+ * unmap and release the buffer.
*
- * For async sends we're going to lose the caller's context so, if an
- * async_handler was provided, it can deal with the reply, otherwise we'll just
- * check and report if there is an error.
+ * Return: 0 if the request was successful, -%EBUSY if reset is detected
+ * or Tx control queue is full, other negative error code on failure.
*/
-static int
-idpf_vc_xn_forward_async(struct idpf_adapter *adapter, struct idpf_vc_xn *xn,
- const struct idpf_ctlq_msg *ctlq_msg)
+int idpf_send_mb_msg(struct idpf_adapter *adapter,
+ struct libie_ctlq_xn_send_params *xn_params,
+ void *send_buf, size_t send_buf_size)
{
- int err = 0;
+ struct libie_ctlq_msg ctlq_msg = {};
- if (ctlq_msg->cookie.mbx.chnl_opcode != xn->vc_op) {
- dev_err_ratelimited(&adapter->pdev->dev, "Async message opcode does not match transaction opcode (msg: %d) (xn: %d)\n",
- ctlq_msg->cookie.mbx.chnl_opcode, xn->vc_op);
- xn->reply_sz = 0;
- err = -EINVAL;
- goto release_bufs;
- }
+ if (idpf_is_reset_detected(adapter)) {
+ if (!libie_cp_can_send_onstack(send_buf_size))
+ kfree(send_buf);
- if (xn->async_handler) {
- err = xn->async_handler(adapter, xn, ctlq_msg);
- goto release_bufs;
+ return -EBUSY;
}
- if (ctlq_msg->cookie.mbx.chnl_retval) {
- xn->reply_sz = 0;
- dev_err_ratelimited(&adapter->pdev->dev, "Async message failure (op %d)\n",
- ctlq_msg->cookie.mbx.chnl_opcode);
- err = -EINVAL;
- }
+ idpf_prepare_ptp_mb_msg(adapter, xn_params->chnl_opcode, &ctlq_msg);
+ xn_params->ctlq_msg = ctlq_msg.opcode ? &ctlq_msg : NULL;
-release_bufs:
- idpf_vc_xn_push_free(adapter->vcxn_mngr, xn);
-
- return err;
-}
-
-/**
- * idpf_vc_xn_forward_reply - copy a reply back to receiving thread
- * @adapter: driver specific private structure with vcxn_mngr
- * @ctlq_msg: controlq message to send back to receiving thread
- */
-static int
-idpf_vc_xn_forward_reply(struct idpf_adapter *adapter,
- const struct idpf_ctlq_msg *ctlq_msg)
-{
- const void *payload = NULL;
- size_t payload_size = 0;
- struct idpf_vc_xn *xn;
- u16 msg_info;
- int err = 0;
- u16 xn_idx;
- u16 salt;
-
- msg_info = ctlq_msg->ctx.sw_cookie.data;
- xn_idx = FIELD_GET(IDPF_VC_XN_IDX_M, msg_info);
- if (xn_idx >= ARRAY_SIZE(adapter->vcxn_mngr->ring)) {
- dev_err_ratelimited(&adapter->pdev->dev, "Out of bounds cookie received: %02x\n",
- xn_idx);
- return -EINVAL;
- }
- xn = &adapter->vcxn_mngr->ring[xn_idx];
- idpf_vc_xn_lock(xn);
- salt = FIELD_GET(IDPF_VC_XN_SALT_M, msg_info);
- if (xn->salt != salt) {
- dev_err_ratelimited(&adapter->pdev->dev, "Transaction salt does not match (exp:%d@%02x(%d) != got:%d@%02x)\n",
- xn->vc_op, xn->salt, xn->state,
- ctlq_msg->cookie.mbx.chnl_opcode, salt);
- idpf_vc_xn_unlock(xn);
- return -EINVAL;
- }
+ xn_params->send_buf.iov_base = send_buf;
+ xn_params->send_buf.iov_len = send_buf_size;
+ xn_params->xnm = adapter->xnm;
+ xn_params->ctlq = xn_params->ctlq ? xn_params->ctlq : adapter->asq;
+ xn_params->rel_tx_buf = kfree;
- switch (xn->state) {
- case IDPF_VC_XN_WAITING:
- /* success */
- break;
- case IDPF_VC_XN_IDLE:
- dev_err_ratelimited(&adapter->pdev->dev, "Unexpected or belated VC reply (op %d)\n",
- ctlq_msg->cookie.mbx.chnl_opcode);
- err = -EINVAL;
- goto out_unlock;
- case IDPF_VC_XN_SHUTDOWN:
- /* ENXIO is a bit special here as the recv msg loop uses that
- * know if it should stop trying to clean the ring if we lost
- * the virtchnl. We need to stop playing with registers and
- * yield.
- */
- err = -ENXIO;
- goto out_unlock;
- case IDPF_VC_XN_ASYNC:
- /* Set reply_sz from the actual payload so that async_handler
- * can evaluate the response.
- */
- xn->reply_sz = ctlq_msg->data_len;
- err = idpf_vc_xn_forward_async(adapter, xn, ctlq_msg);
- idpf_vc_xn_unlock(xn);
- return err;
- default:
- dev_err_ratelimited(&adapter->pdev->dev, "Overwriting VC reply (op %d)\n",
- ctlq_msg->cookie.mbx.chnl_opcode);
- err = -EBUSY;
- goto out_unlock;
- }
+ idpf_mb_clean(adapter, xn_params->ctlq, false);
- if (ctlq_msg->cookie.mbx.chnl_opcode != xn->vc_op) {
- dev_err_ratelimited(&adapter->pdev->dev, "Message opcode does not match transaction opcode (msg: %d) (xn: %d)\n",
- ctlq_msg->cookie.mbx.chnl_opcode, xn->vc_op);
- xn->reply_sz = 0;
- xn->state = IDPF_VC_XN_COMPLETED_FAILED;
- err = -EINVAL;
- goto out_unlock;
- }
-
- if (ctlq_msg->cookie.mbx.chnl_retval) {
- xn->reply_sz = 0;
- xn->state = IDPF_VC_XN_COMPLETED_FAILED;
- err = -EINVAL;
- goto out_unlock;
- }
-
- if (ctlq_msg->data_len) {
- payload = ctlq_msg->ctx.indirect.payload->va;
- payload_size = ctlq_msg->data_len;
- }
-
- xn->reply_sz = payload_size;
- xn->state = IDPF_VC_XN_COMPLETED_SUCCESS;
-
- if (xn->reply.iov_base && xn->reply.iov_len && payload_size)
- memcpy(xn->reply.iov_base, payload,
- min_t(size_t, xn->reply.iov_len, payload_size));
-
-out_unlock:
- idpf_vc_xn_unlock(xn);
- /* we _cannot_ hold lock while calling complete */
- complete(&xn->completed);
-
- return err;
+ return libie_ctlq_xn_send(xn_params);
}
/**
- * idpf_recv_mb_msg - Receive message over mailbox
+ * idpf_send_mb_msg_kfree - send mailbox message and free the send buffer
* @adapter: driver specific private structure
- * @arq: control queue to receive message from
+ * @xn_params: Xn send parameters to fill
+ * @send_buf: buffer to send, can be released with kfree()
+ * @send_buf_size: size of the send buffer
*
- * Will receive control queue message and posts the receive buffer.
+ * libie_cp functions consume only buffers above certain size,
+ * smaller buffers are assumed to be on the stack. However, for some
+ * commands with variable message size it makes sense to always use kzalloc(),
+ * which means we have to free smaller buffers ourselves.
*
- * Return: 0 on success and negative on failure.
+ * Return: 0 if no unexpected errors were encountered,
+ * negative error code otherwise.
*/
-int idpf_recv_mb_msg(struct idpf_adapter *adapter, struct idpf_ctlq_info *arq)
+int idpf_send_mb_msg_kfree(struct idpf_adapter *adapter,
+ struct libie_ctlq_xn_send_params *xn_params,
+ void *send_buf, size_t send_buf_size)
{
- struct idpf_ctlq_msg ctlq_msg;
- struct idpf_dma_mem *dma_mem;
- int post_err, err;
- u16 num_recv;
-
- while (1) {
- /* This will get <= num_recv messages and output how many
- * actually received on num_recv.
- */
- num_recv = 1;
- err = idpf_ctlq_recv(arq, &num_recv, &ctlq_msg);
- if (err || !num_recv)
- break;
-
- if (ctlq_msg.data_len) {
- dma_mem = ctlq_msg.ctx.indirect.payload;
- } else {
- dma_mem = NULL;
- num_recv = 0;
- }
-
- if (ctlq_msg.cookie.mbx.chnl_opcode == VIRTCHNL2_OP_EVENT)
- idpf_recv_event_msg(adapter, &ctlq_msg);
- else
- err = idpf_vc_xn_forward_reply(adapter, &ctlq_msg);
+ int err = idpf_send_mb_msg(adapter, xn_params, send_buf, send_buf_size);
- post_err = idpf_ctlq_post_rx_buffs(&adapter->hw, arq,
- &num_recv, &dma_mem);
-
- /* If post failed clear the only buffer we supplied */
- if (post_err) {
- if (dma_mem)
- dma_free_coherent(&adapter->pdev->dev,
- dma_mem->size, dma_mem->va,
- dma_mem->pa);
- break;
- }
-
- /* virtchnl trying to shutdown, stop cleaning */
- if (err == -ENXIO)
- break;
- }
+ if (libie_cp_can_send_onstack(send_buf_size))
+ kfree(send_buf);
return err;
}
@@ -768,45 +295,43 @@ struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_adapter *adapter,
static int idpf_send_chunked_msg(struct idpf_adapter *adapter,
const struct idpf_chunked_msg_params *params)
{
- struct idpf_vc_xn_params xn_params = {
- .vc_op = params->vc_op,
+ struct libie_ctlq_xn_send_params xn_params = {
.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = params->vc_op,
};
const void *pos = params->chunks;
- u32 num_chunks, num_msgs, buf_sz;
- void *buf __free(kfree) = NULL;
u32 totqs = params->num_chunks;
u32 vid = params->vport_id;
+ u32 num_chunks, num_msgs;
- num_chunks = min(IDPF_NUM_CHUNKS_PER_MSG(params->config_sz,
- params->chunk_sz), totqs);
+ num_chunks = IDPF_NUM_CHUNKS_PER_MSG(params->config_sz,
+ params->chunk_sz);
num_msgs = DIV_ROUND_UP(totqs, num_chunks);
- buf_sz = params->config_sz + num_chunks * params->chunk_sz;
- buf = kzalloc(buf_sz, GFP_KERNEL);
- if (!buf)
- return -ENOMEM;
-
- xn_params.send_buf.iov_base = buf;
-
for (u32 i = 0; i < num_msgs; i++) {
- ssize_t reply_sz;
+ u32 buf_sz;
+ void *buf;
+ int err;
- memset(buf, 0, buf_sz);
- xn_params.send_buf.iov_len = buf_sz;
+ num_chunks = min(num_chunks, totqs);
+ buf_sz = params->config_sz + num_chunks * params->chunk_sz;
+ buf = kzalloc(buf_sz, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
- if (params->prepare_msg(vid, buf, pos, num_chunks) != buf_sz)
+ if (params->prepare_msg(vid, buf, pos, num_chunks) != buf_sz) {
+ kfree(buf);
return -EINVAL;
+ }
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
+ err = idpf_send_mb_msg_kfree(adapter, &xn_params, buf, buf_sz);
+ if (err)
+ return err;
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+ xn_params.recv_mem = (struct kvec) {};
pos += num_chunks * params->chunk_sz;
totqs -= num_chunks;
-
- num_chunks = min(num_chunks, totqs);
- buf_sz = params->config_sz + num_chunks * params->chunk_sz;
}
return 0;
@@ -881,11 +406,14 @@ static int idpf_wait_for_marker_event(struct idpf_vport *vport)
*/
static int idpf_send_ver_msg(struct idpf_adapter *adapter)
{
- struct idpf_vc_xn_params xn_params = {};
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_VERSION,
+ };
+ struct virtchnl2_version_info *vvi_recv;
struct virtchnl2_version_info vvi;
- ssize_t reply_sz;
u32 major, minor;
- int err = 0;
+ int err;
if (adapter->virt_ver_maj) {
vvi.major = cpu_to_le32(adapter->virt_ver_maj);
@@ -895,24 +423,23 @@ static int idpf_send_ver_msg(struct idpf_adapter *adapter)
vvi.minor = cpu_to_le32(IDPF_VIRTCHNL_VERSION_MINOR);
}
- xn_params.vc_op = VIRTCHNL2_OP_VERSION;
- xn_params.send_buf.iov_base = &vvi;
- xn_params.send_buf.iov_len = sizeof(vvi);
- xn_params.recv_buf = xn_params.send_buf;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
+ err = idpf_send_mb_msg(adapter, &xn_params, &vvi, sizeof(vvi));
+ if (err)
+ return err;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
- if (reply_sz < sizeof(vvi))
- return -EIO;
+ if (xn_params.recv_mem.iov_len < sizeof(*vvi_recv)) {
+ err = -EIO;
+ goto free_rx_buf;
+ }
- major = le32_to_cpu(vvi.major);
- minor = le32_to_cpu(vvi.minor);
+ vvi_recv = xn_params.recv_mem.iov_base;
+ major = le32_to_cpu(vvi_recv->major);
+ minor = le32_to_cpu(vvi_recv->minor);
if (major > IDPF_VIRTCHNL_VERSION_MAJOR) {
dev_warn(&adapter->pdev->dev, "Virtchnl major version greater than supported\n");
- return -EINVAL;
+ err = -EINVAL;
+ goto free_rx_buf;
}
if (major == IDPF_VIRTCHNL_VERSION_MAJOR &&
@@ -930,6 +457,9 @@ static int idpf_send_ver_msg(struct idpf_adapter *adapter)
adapter->virt_ver_maj = major;
adapter->virt_ver_min = minor;
+free_rx_buf:
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+
return err;
}
@@ -942,9 +472,12 @@ static int idpf_send_ver_msg(struct idpf_adapter *adapter)
*/
static int idpf_send_get_caps_msg(struct idpf_adapter *adapter)
{
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_GET_CAPS,
+ };
struct virtchnl2_get_capabilities caps = {};
- struct idpf_vc_xn_params xn_params = {};
- ssize_t reply_sz;
+ int err;
caps.csum_caps =
cpu_to_le32(VIRTCHNL2_CAP_TX_CSUM_L3_IPV4 |
@@ -1004,20 +537,22 @@ static int idpf_send_get_caps_msg(struct idpf_adapter *adapter)
VIRTCHNL2_CAP_LOOPBACK |
VIRTCHNL2_CAP_PTP);
- xn_params.vc_op = VIRTCHNL2_OP_GET_CAPS;
- xn_params.send_buf.iov_base = ∩︀
- xn_params.send_buf.iov_len = sizeof(caps);
- xn_params.recv_buf.iov_base = &adapter->caps;
- xn_params.recv_buf.iov_len = sizeof(adapter->caps);
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
+ err = idpf_send_mb_msg(adapter, &xn_params, &caps, sizeof(caps));
+ if (err)
+ return err;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
- if (reply_sz < sizeof(adapter->caps))
- return -EIO;
+ if (xn_params.recv_mem.iov_len < sizeof(adapter->caps)) {
+ err = -EIO;
+ goto free_rx_buf;
+ }
- return 0;
+ memcpy(&adapter->caps, xn_params.recv_mem.iov_base,
+ sizeof(adapter->caps));
+
+free_rx_buf:
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+
+ return err;
}
/**
@@ -1062,37 +597,39 @@ static void idpf_decfg_lan_memory_regions(struct idpf_adapter *adapter)
*/
static int idpf_cfg_lan_memory_regions(struct idpf_adapter *adapter)
{
- struct virtchnl2_get_lan_memory_regions *rcvd_regions __free(kfree);
- struct idpf_vc_xn_params xn_params = {
- .vc_op = VIRTCHNL2_OP_GET_LAN_MEMORY_REGIONS,
- .recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN,
- .send_buf.iov_len =
- sizeof(struct virtchnl2_get_lan_memory_regions) +
- sizeof(struct virtchnl2_mem_region),
+ struct virtchnl2_get_lan_memory_regions *send_regions, *rcvd_regions;
+ struct libie_ctlq_xn_send_params xn_params = {
+ .chnl_opcode = VIRTCHNL2_OP_GET_LAN_MEMORY_REGIONS,
.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
};
- int num_regions, size;
- ssize_t reply_sz;
+ size_t send_sz, reply_sz, size;
+ int num_regions;
int err = 0;
- rcvd_regions = kzalloc(IDPF_CTLQ_MAX_BUF_LEN, GFP_KERNEL);
- if (!rcvd_regions)
+ send_sz = sizeof(struct virtchnl2_get_lan_memory_regions) +
+ sizeof(struct virtchnl2_mem_region);
+ send_regions = kzalloc(send_sz, GFP_KERNEL);
+ if (!send_regions)
return -ENOMEM;
- xn_params.recv_buf.iov_base = rcvd_regions;
- rcvd_regions->num_memory_regions = cpu_to_le16(1);
- xn_params.send_buf.iov_base = rcvd_regions;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
+ send_regions->num_memory_regions = cpu_to_le16(1);
+ err = idpf_send_mb_msg_kfree(adapter, &xn_params, send_regions,
+ send_sz);
+ if (err)
+ return err;
+ rcvd_regions = xn_params.recv_mem.iov_base;
+ reply_sz = xn_params.recv_mem.iov_len;
+ if (reply_sz < sizeof(*rcvd_regions)) {
+ err = -EIO;
+ goto rel_rx_buf;
+ }
num_regions = le16_to_cpu(rcvd_regions->num_memory_regions);
size = struct_size(rcvd_regions, mem_reg, num_regions);
- if (reply_sz < size)
- return -EIO;
-
- if (size > IDPF_CTLQ_MAX_BUF_LEN)
- return -EINVAL;
+ if (reply_sz < size) {
+ err = -EIO;
+ goto rel_rx_buf;
+ }
for (int i = 0; i < num_regions; i++) {
struct libie_mmio_info *mmio = &adapter->ctlq_ctx.mmio_info;
@@ -1102,10 +639,14 @@ static int idpf_cfg_lan_memory_regions(struct idpf_adapter *adapter)
len = le64_to_cpu(rcvd_regions->mem_reg[i].size);
if (len && !libie_pci_map_mmio_region(mmio, offset, len)) {
idpf_decfg_lan_memory_regions(adapter);
- return -EIO;
+ err = -EIO;
+ goto rel_rx_buf;
}
}
+rel_rx_buf:
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+
return err;
}
@@ -1164,24 +705,43 @@ int idpf_add_del_fsteer_filters(struct idpf_adapter *adapter,
struct virtchnl2_flow_rule_add_del *rule,
enum virtchnl2_op opcode)
{
+ struct libie_ctlq_xn_send_params xn_params = {
+ .chnl_opcode = opcode,
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ };
+ struct virtchnl2_flow_rule_add_del *rx_rule;
int rule_count = le32_to_cpu(rule->count);
- struct idpf_vc_xn_params xn_params = {};
- ssize_t reply_sz;
+ size_t send_sz;
+ int err;
if (opcode != VIRTCHNL2_OP_ADD_FLOW_RULE &&
- opcode != VIRTCHNL2_OP_DEL_FLOW_RULE)
+ opcode != VIRTCHNL2_OP_DEL_FLOW_RULE) {
+ kfree(rule);
return -EINVAL;
+ }
- xn_params.vc_op = opcode;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- xn_params.async = false;
- xn_params.send_buf.iov_base = rule;
- xn_params.send_buf.iov_len = struct_size(rule, rule_info, rule_count);
- xn_params.recv_buf.iov_base = rule;
- xn_params.recv_buf.iov_len = struct_size(rule, rule_info, rule_count);
+ send_sz = struct_size(rule, rule_info, rule_count);
+ err = idpf_send_mb_msg_kfree(adapter, &xn_params, rule, send_sz);
+ if (err)
+ return err;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- return reply_sz < 0 ? reply_sz : 0;
+ if (xn_params.recv_mem.iov_len < send_sz) {
+ err = -EIO;
+ goto rel_rx;
+ }
+
+ rx_rule = xn_params.recv_mem.iov_base;
+ for (int i = 0; i < rule_count; i++) {
+ if (rx_rule->rule_info[i].status !=
+ cpu_to_le32(VIRTCHNL2_FLOW_RULE_SUCCESS)) {
+ err = -EIO;
+ goto rel_rx;
+ }
+ }
+
+rel_rx:
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+ return err;
}
/**
@@ -1555,11 +1115,13 @@ int idpf_queue_reg_init(struct idpf_vport *vport,
int idpf_send_create_vport_msg(struct idpf_adapter *adapter,
struct idpf_vport_max_q *max_q)
{
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_CREATE_VPORT,
+ };
struct virtchnl2_create_vport *vport_msg;
- struct idpf_vc_xn_params xn_params = {};
u16 idx = adapter->next_vport;
int err, buf_size;
- ssize_t reply_sz;
buf_size = sizeof(struct virtchnl2_create_vport);
vport_msg = kzalloc(buf_size, GFP_KERNEL);
@@ -1586,33 +1148,29 @@ int idpf_send_create_vport_msg(struct idpf_adapter *adapter,
}
if (!adapter->vport_params_recvd[idx]) {
- adapter->vport_params_recvd[idx] = kzalloc(IDPF_CTLQ_MAX_BUF_LEN,
- GFP_KERNEL);
+ adapter->vport_params_recvd[idx] =
+ kzalloc(LIBIE_CTLQ_MAX_BUF_LEN, GFP_KERNEL);
if (!adapter->vport_params_recvd[idx]) {
err = -ENOMEM;
goto rel_buf;
}
}
- xn_params.vc_op = VIRTCHNL2_OP_CREATE_VPORT;
- xn_params.send_buf.iov_base = vport_msg;
- xn_params.send_buf.iov_len = buf_size;
- xn_params.recv_buf.iov_base = adapter->vport_params_recvd[idx];
- xn_params.recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0) {
- err = reply_sz;
- goto free_vport_params;
+ err = idpf_send_mb_msg_kfree(adapter, &xn_params, vport_msg,
+ sizeof(*vport_msg));
+ if (err) {
+ kfree(adapter->vport_params_recvd[idx]);
+ adapter->vport_params_recvd[idx] = NULL;
+ return err;
}
- kfree(vport_msg);
+ memcpy(adapter->vport_params_recvd[idx], xn_params.recv_mem.iov_base,
+ xn_params.recv_mem.iov_len);
+
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
return 0;
-free_vport_params:
- kfree(adapter->vport_params_recvd[idx]);
- adapter->vport_params_recvd[idx] = NULL;
rel_buf:
kfree(vport_msg);
@@ -1674,19 +1232,22 @@ int idpf_check_supported_desc_ids(struct idpf_vport *vport)
*/
int idpf_send_destroy_vport_msg(struct idpf_adapter *adapter, u32 vport_id)
{
- struct idpf_vc_xn_params xn_params = {};
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_DESTROY_VPORT,
+ };
struct virtchnl2_vport v_id;
- ssize_t reply_sz;
+ int err;
v_id.vport_id = cpu_to_le32(vport_id);
- xn_params.vc_op = VIRTCHNL2_OP_DESTROY_VPORT;
- xn_params.send_buf.iov_base = &v_id;
- xn_params.send_buf.iov_len = sizeof(v_id);
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
+ err = idpf_send_mb_msg(adapter, &xn_params, &v_id, sizeof(v_id));
+ if (err)
+ return err;
+
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
- return reply_sz < 0 ? reply_sz : 0;
+ return 0;
}
/**
@@ -1698,19 +1259,22 @@ int idpf_send_destroy_vport_msg(struct idpf_adapter *adapter, u32 vport_id)
*/
int idpf_send_enable_vport_msg(struct idpf_adapter *adapter, u32 vport_id)
{
- struct idpf_vc_xn_params xn_params = {};
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_ENABLE_VPORT,
+ };
struct virtchnl2_vport v_id;
- ssize_t reply_sz;
+ int err;
v_id.vport_id = cpu_to_le32(vport_id);
- xn_params.vc_op = VIRTCHNL2_OP_ENABLE_VPORT;
- xn_params.send_buf.iov_base = &v_id;
- xn_params.send_buf.iov_len = sizeof(v_id);
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
+ err = idpf_send_mb_msg(adapter, &xn_params, &v_id, sizeof(v_id));
+ if (err)
+ return err;
+
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
- return reply_sz < 0 ? reply_sz : 0;
+ return 0;
}
/**
@@ -1722,19 +1286,22 @@ int idpf_send_enable_vport_msg(struct idpf_adapter *adapter, u32 vport_id)
*/
int idpf_send_disable_vport_msg(struct idpf_adapter *adapter, u32 vport_id)
{
- struct idpf_vc_xn_params xn_params = {};
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_DISABLE_VPORT,
+ };
struct virtchnl2_vport v_id;
- ssize_t reply_sz;
+ int err;
v_id.vport_id = cpu_to_le32(vport_id);
- xn_params.vc_op = VIRTCHNL2_OP_DISABLE_VPORT;
- xn_params.send_buf.iov_base = &v_id;
- xn_params.send_buf.iov_len = sizeof(v_id);
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
+ err = idpf_send_mb_msg(adapter, &xn_params, &v_id, sizeof(v_id));
+ if (err)
+ return err;
- return reply_sz < 0 ? reply_sz : 0;
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+
+ return 0;
}
/**
@@ -2573,11 +2140,14 @@ int idpf_send_delete_queues_msg(struct idpf_adapter *adapter,
struct idpf_queue_id_reg_info *chunks,
u32 vport_id)
{
- struct virtchnl2_del_ena_dis_queues *eq __free(kfree) = NULL;
- struct idpf_vc_xn_params xn_params = {};
- ssize_t reply_sz;
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_DEL_QUEUES,
+ };
+ struct virtchnl2_del_ena_dis_queues *eq;
+ ssize_t buf_size;
u16 num_chunks;
- int buf_size;
+ int err;
num_chunks = chunks->num_chunks;
buf_size = struct_size(eq, chunks.chunks, num_chunks);
@@ -2592,13 +2162,13 @@ int idpf_send_delete_queues_msg(struct idpf_adapter *adapter,
idpf_convert_reg_to_queue_chunks(eq->chunks.chunks, chunks->queue_chunks,
num_chunks);
- xn_params.vc_op = VIRTCHNL2_OP_DEL_QUEUES;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- xn_params.send_buf.iov_base = eq;
- xn_params.send_buf.iov_len = buf_size;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
+ err = idpf_send_mb_msg_kfree(adapter, &xn_params, eq, buf_size);
+ if (err)
+ return err;
+
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
- return reply_sz < 0 ? reply_sz : 0;
+ return 0;
}
/**
@@ -2636,15 +2206,14 @@ int idpf_send_add_queues_msg(struct idpf_adapter *adapter,
struct idpf_q_vec_rsrc *rsrc,
u32 vport_id)
{
- struct virtchnl2_add_queues *vc_msg __free(kfree) = NULL;
- struct idpf_vc_xn_params xn_params = {};
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_ADD_QUEUES,
+ };
+ struct virtchnl2_add_queues *vc_msg;
struct virtchnl2_add_queues aq = {};
- ssize_t reply_sz;
- int size;
-
- vc_msg = kzalloc(IDPF_CTLQ_MAX_BUF_LEN, GFP_KERNEL);
- if (!vc_msg)
- return -ENOMEM;
+ size_t size;
+ int err;
aq.vport_id = cpu_to_le32(vport_id);
aq.num_tx_q = cpu_to_le16(rsrc->num_txq);
@@ -2652,29 +2221,38 @@ int idpf_send_add_queues_msg(struct idpf_adapter *adapter,
aq.num_rx_q = cpu_to_le16(rsrc->num_rxq);
aq.num_rx_bufq = cpu_to_le16(rsrc->num_bufq);
- xn_params.vc_op = VIRTCHNL2_OP_ADD_QUEUES;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- xn_params.send_buf.iov_base = &aq;
- xn_params.send_buf.iov_len = sizeof(aq);
- xn_params.recv_buf.iov_base = vc_msg;
- xn_params.recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
+ err = idpf_send_mb_msg(adapter, &xn_params, &aq, sizeof(aq));
+ if (err)
+ return err;
+
+ vc_msg = xn_params.recv_mem.iov_base;
+ if (xn_params.recv_mem.iov_len < sizeof(*vc_msg)) {
+ err = -EIO;
+ goto free_rx_buf;
+ }
/* compare vc_msg num queues with vport num queues */
if (le16_to_cpu(vc_msg->num_tx_q) != rsrc->num_txq ||
le16_to_cpu(vc_msg->num_rx_q) != rsrc->num_rxq ||
le16_to_cpu(vc_msg->num_tx_complq) != rsrc->num_complq ||
- le16_to_cpu(vc_msg->num_rx_bufq) != rsrc->num_bufq)
- return -EINVAL;
+ le16_to_cpu(vc_msg->num_rx_bufq) != rsrc->num_bufq) {
+ err = -EINVAL;
+ goto free_rx_buf;
+ }
size = struct_size(vc_msg, chunks.chunks,
le16_to_cpu(vc_msg->chunks.num_chunks));
- if (reply_sz < size)
- return -EIO;
+ if (xn_params.recv_mem.iov_len < size) {
+ err = -EIO;
+ goto free_rx_buf;
+ }
- return idpf_vport_init_queue_reg_chunks(vport_config, &vc_msg->chunks);
+ err = idpf_vport_init_queue_reg_chunks(vport_config, &vc_msg->chunks);
+
+free_rx_buf:
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+
+ return err;
}
/**
@@ -2686,49 +2264,51 @@ int idpf_send_add_queues_msg(struct idpf_adapter *adapter,
*/
int idpf_send_alloc_vectors_msg(struct idpf_adapter *adapter, u16 num_vectors)
{
- struct virtchnl2_alloc_vectors *rcvd_vec __free(kfree) = NULL;
- struct idpf_vc_xn_params xn_params = {};
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_ALLOC_VECTORS,
+ };
+ struct virtchnl2_alloc_vectors *rcvd_vec;
struct virtchnl2_alloc_vectors ac = {};
- ssize_t reply_sz;
u16 num_vchunks;
- int size;
+ int size, err;
ac.num_vectors = cpu_to_le16(num_vectors);
- rcvd_vec = kzalloc(IDPF_CTLQ_MAX_BUF_LEN, GFP_KERNEL);
- if (!rcvd_vec)
- return -ENOMEM;
+ err = idpf_send_mb_msg(adapter, &xn_params, &ac, sizeof(ac));
+ if (err)
+ return err;
- xn_params.vc_op = VIRTCHNL2_OP_ALLOC_VECTORS;
- xn_params.send_buf.iov_base = ∾
- xn_params.send_buf.iov_len = sizeof(ac);
- xn_params.recv_buf.iov_base = rcvd_vec;
- xn_params.recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
+ rcvd_vec = xn_params.recv_mem.iov_base;
+ if (xn_params.recv_mem.iov_len < sizeof(*rcvd_vec)) {
+ err = -EIO;
+ goto free_rx_buf;
+ }
num_vchunks = le16_to_cpu(rcvd_vec->vchunks.num_vchunks);
size = struct_size(rcvd_vec, vchunks.vchunks, num_vchunks);
- if (reply_sz < size)
- return -EIO;
-
- if (size > IDPF_CTLQ_MAX_BUF_LEN)
- return -EINVAL;
+ if (xn_params.recv_mem.iov_len < size) {
+ err = -EIO;
+ goto free_rx_buf;
+ }
kfree(adapter->req_vec_chunks);
adapter->req_vec_chunks = kmemdup(rcvd_vec, size, GFP_KERNEL);
- if (!adapter->req_vec_chunks)
- return -ENOMEM;
+ if (!adapter->req_vec_chunks) {
+ err = -ENOMEM;
+ goto free_rx_buf;
+ }
if (le16_to_cpu(adapter->req_vec_chunks->num_vectors) < num_vectors) {
kfree(adapter->req_vec_chunks);
adapter->req_vec_chunks = NULL;
- return -EINVAL;
+ err = -EINVAL;
}
- return 0;
+free_rx_buf:
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+
+ return err;
}
/**
@@ -2740,24 +2320,28 @@ int idpf_send_alloc_vectors_msg(struct idpf_adapter *adapter, u16 num_vectors)
int idpf_send_dealloc_vectors_msg(struct idpf_adapter *adapter)
{
struct virtchnl2_alloc_vectors *ac = adapter->req_vec_chunks;
- struct virtchnl2_vector_chunks *vcs = &ac->vchunks;
- struct idpf_vc_xn_params xn_params = {};
- ssize_t reply_sz;
- int buf_size;
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_DEALLOC_VECTORS,
+ };
+ struct virtchnl2_vector_chunks *vcs;
+ int buf_size, err;
- buf_size = struct_size(vcs, vchunks, le16_to_cpu(vcs->num_vchunks));
+ buf_size = struct_size(&ac->vchunks, vchunks,
+ le16_to_cpu(ac->vchunks.num_vchunks));
+ vcs = kmemdup(&ac->vchunks, buf_size, GFP_KERNEL);
+ if (!vcs)
+ return -ENOMEM;
- xn_params.vc_op = VIRTCHNL2_OP_DEALLOC_VECTORS;
- xn_params.send_buf.iov_base = vcs;
- xn_params.send_buf.iov_len = buf_size;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
+ err = idpf_send_mb_msg_kfree(adapter, &xn_params, vcs, buf_size);
+ if (err)
+ return err;
kfree(adapter->req_vec_chunks);
adapter->req_vec_chunks = NULL;
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+
return 0;
}
@@ -2781,18 +2365,22 @@ static int idpf_get_max_vfs(struct idpf_adapter *adapter)
*/
int idpf_send_set_sriov_vfs_msg(struct idpf_adapter *adapter, u16 num_vfs)
{
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_SET_SRIOV_VFS,
+ };
struct virtchnl2_sriov_vfs_info svi = {};
- struct idpf_vc_xn_params xn_params = {};
- ssize_t reply_sz;
+ int err;
svi.num_vfs = cpu_to_le16(num_vfs);
- xn_params.vc_op = VIRTCHNL2_OP_SET_SRIOV_VFS;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- xn_params.send_buf.iov_base = &svi;
- xn_params.send_buf.iov_len = sizeof(svi);
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- return reply_sz < 0 ? reply_sz : 0;
+ err = idpf_send_mb_msg(adapter, &xn_params, &svi, sizeof(svi));
+ if (err)
+ return err;
+
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+
+ return 0;
}
/**
@@ -2805,10 +2393,14 @@ int idpf_send_set_sriov_vfs_msg(struct idpf_adapter *adapter, u16 num_vfs)
int idpf_send_get_stats_msg(struct idpf_netdev_priv *np,
struct idpf_port_stats *port_stats)
{
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_GET_STATS,
+ };
struct rtnl_link_stats64 *netstats = &np->netstats;
+ struct virtchnl2_vport_stats *stats_recv;
struct virtchnl2_vport_stats stats_msg = {};
- struct idpf_vc_xn_params xn_params = {};
- ssize_t reply_sz;
+ int err;
/* Don't send get_stats message if the link is down */
@@ -2817,38 +2409,41 @@ int idpf_send_get_stats_msg(struct idpf_netdev_priv *np,
stats_msg.vport_id = cpu_to_le32(np->vport_id);
- xn_params.vc_op = VIRTCHNL2_OP_GET_STATS;
- xn_params.send_buf.iov_base = &stats_msg;
- xn_params.send_buf.iov_len = sizeof(stats_msg);
- xn_params.recv_buf = xn_params.send_buf;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
+ err = idpf_send_mb_msg(np->adapter, &xn_params, &stats_msg,
+ sizeof(stats_msg));
+ if (err)
+ return err;
- reply_sz = idpf_vc_xn_exec(np->adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
- if (reply_sz < sizeof(stats_msg))
- return -EIO;
+ if (xn_params.recv_mem.iov_len < sizeof(*stats_recv)) {
+ err = -EIO;
+ goto free_rx_buf;
+ }
+
+ stats_recv = xn_params.recv_mem.iov_base;
spin_lock_bh(&np->stats_lock);
- netstats->rx_packets = le64_to_cpu(stats_msg.rx_unicast) +
- le64_to_cpu(stats_msg.rx_multicast) +
- le64_to_cpu(stats_msg.rx_broadcast);
- netstats->tx_packets = le64_to_cpu(stats_msg.tx_unicast) +
- le64_to_cpu(stats_msg.tx_multicast) +
- le64_to_cpu(stats_msg.tx_broadcast);
- netstats->rx_bytes = le64_to_cpu(stats_msg.rx_bytes);
- netstats->tx_bytes = le64_to_cpu(stats_msg.tx_bytes);
- netstats->rx_errors = le64_to_cpu(stats_msg.rx_errors);
- netstats->tx_errors = le64_to_cpu(stats_msg.tx_errors);
- netstats->rx_dropped = le64_to_cpu(stats_msg.rx_discards);
- netstats->tx_dropped = le64_to_cpu(stats_msg.tx_discards);
-
- port_stats->vport_stats = stats_msg;
+ netstats->rx_packets = le64_to_cpu(stats_recv->rx_unicast) +
+ le64_to_cpu(stats_recv->rx_multicast) +
+ le64_to_cpu(stats_recv->rx_broadcast);
+ netstats->tx_packets = le64_to_cpu(stats_recv->tx_unicast) +
+ le64_to_cpu(stats_recv->tx_multicast) +
+ le64_to_cpu(stats_recv->tx_broadcast);
+ netstats->rx_bytes = le64_to_cpu(stats_recv->rx_bytes);
+ netstats->tx_bytes = le64_to_cpu(stats_recv->tx_bytes);
+ netstats->rx_errors = le64_to_cpu(stats_recv->rx_errors);
+ netstats->tx_errors = le64_to_cpu(stats_recv->tx_errors);
+ netstats->rx_dropped = le64_to_cpu(stats_recv->rx_discards);
+ netstats->tx_dropped = le64_to_cpu(stats_recv->tx_discards);
+
+ port_stats->vport_stats = *stats_recv;
spin_unlock_bh(&np->stats_lock);
- return 0;
+free_rx_buf:
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+
+ return err;
}
/**
@@ -2866,13 +2461,14 @@ int idpf_send_get_stats_msg(struct idpf_netdev_priv *np,
int idpf_send_set_rss_lut_msg(struct idpf_adapter *adapter,
struct idpf_rss_data *rss_data, u32 vport_id)
{
- struct virtchnl2_rss_lut *rl __free(kfree) = NULL;
- struct idpf_vc_xn_params xn_params = {};
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_SET_RSS_LUT,
+ };
+ struct virtchnl2_rss_lut *rl;
struct idpf_vport *vport;
- ssize_t reply_sz;
+ int buf_size, i, err;
bool rxhash_ena;
- int buf_size;
- int i;
vport = idpf_vid_to_vport(adapter, vport_id);
if (!vport)
@@ -2886,21 +2482,17 @@ int idpf_send_set_rss_lut_msg(struct idpf_adapter *adapter,
return -ENOMEM;
rl->vport_id = cpu_to_le32(vport_id);
-
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- xn_params.send_buf.iov_base = rl;
- xn_params.send_buf.iov_len = buf_size;
- xn_params.vc_op = VIRTCHNL2_OP_SET_RSS_LUT;
-
rl->lut_entries = cpu_to_le16(rss_data->rss_lut_size);
for (i = 0; i < rss_data->rss_lut_size; i++)
rl->lut[i] = rxhash_ena ? cpu_to_le32(rss_data->rss_lut[i]) : 0;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
+ err = idpf_send_mb_msg_kfree(adapter, &xn_params, rl, buf_size);
+ if (err)
+ return err;
- return 0;
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+
+ return err;
}
/**
@@ -2914,10 +2506,12 @@ int idpf_send_set_rss_lut_msg(struct idpf_adapter *adapter,
int idpf_send_set_rss_key_msg(struct idpf_adapter *adapter,
struct idpf_rss_data *rss_data, u32 vport_id)
{
- struct virtchnl2_rss_key *rk __free(kfree) = NULL;
- struct idpf_vc_xn_params xn_params = {};
- ssize_t reply_sz;
- int i, buf_size;
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_SET_RSS_KEY,
+ };
+ struct virtchnl2_rss_key *rk;
+ int i, buf_size, err;
buf_size = struct_size(rk, key_flex, rss_data->rss_key_size);
rk = kzalloc(buf_size, GFP_KERNEL);
@@ -2925,20 +2519,17 @@ int idpf_send_set_rss_key_msg(struct idpf_adapter *adapter,
return -ENOMEM;
rk->vport_id = cpu_to_le32(vport_id);
- xn_params.send_buf.iov_base = rk;
- xn_params.send_buf.iov_len = buf_size;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- xn_params.vc_op = VIRTCHNL2_OP_SET_RSS_KEY;
-
rk->key_len = cpu_to_le16(rss_data->rss_key_size);
for (i = 0; i < rss_data->rss_key_size; i++)
rk->key_flex[i] = rss_data->rss_key[i];
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
+ err = idpf_send_mb_msg_kfree(adapter, &xn_params, rk, buf_size);
+ if (err)
+ return err;
- return 0;
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+
+ return err;
}
/**
@@ -3115,15 +2706,18 @@ static void idpf_parse_protocol_ids(struct virtchnl2_ptype *ptype,
*/
static int idpf_send_get_rx_ptype_msg(struct idpf_adapter *adapter)
{
- struct virtchnl2_get_ptype_info *get_ptype_info __free(kfree) = NULL;
- struct virtchnl2_get_ptype_info *ptype_info __free(kfree) = NULL;
struct libeth_rx_pt *singleq_pt_lkup __free(kfree) = NULL;
struct libeth_rx_pt *splitq_pt_lkup __free(kfree) = NULL;
- struct idpf_vc_xn_params xn_params = {};
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_GET_PTYPE_INFO,
+ };
+ struct virtchnl2_get_ptype_info *get_ptype_info;
+ struct virtchnl2_get_ptype_info *ptype_info;
+ int err = 0, max_ptype = IDPF_RX_MAX_PTYPE;
+ int buf_size = sizeof(*get_ptype_info);
int ptypes_recvd = 0, ptype_offset;
- u32 max_ptype = IDPF_RX_MAX_PTYPE;
u16 next_ptype_id = 0;
- ssize_t reply_sz;
singleq_pt_lkup = kzalloc_objs(*singleq_pt_lkup, IDPF_RX_MAX_BASE_PTYPE);
if (!singleq_pt_lkup)
@@ -3133,42 +2727,38 @@ static int idpf_send_get_rx_ptype_msg(struct idpf_adapter *adapter)
if (!splitq_pt_lkup)
return -ENOMEM;
- get_ptype_info = kzalloc_obj(*get_ptype_info);
- if (!get_ptype_info)
- return -ENOMEM;
-
- ptype_info = kzalloc(IDPF_CTLQ_MAX_BUF_LEN, GFP_KERNEL);
- if (!ptype_info)
- return -ENOMEM;
+ while (next_ptype_id < max_ptype) {
+ u16 num_ptypes;
- xn_params.vc_op = VIRTCHNL2_OP_GET_PTYPE_INFO;
- xn_params.send_buf.iov_base = get_ptype_info;
- xn_params.send_buf.iov_len = sizeof(*get_ptype_info);
- xn_params.recv_buf.iov_base = ptype_info;
- xn_params.recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
+ get_ptype_info = kzalloc(buf_size, GFP_KERNEL);
+ if (!get_ptype_info)
+ return -ENOMEM;
- while (next_ptype_id < max_ptype) {
get_ptype_info->start_ptype_id = cpu_to_le16(next_ptype_id);
if ((next_ptype_id + IDPF_RX_MAX_PTYPES_PER_BUF) > max_ptype)
- get_ptype_info->num_ptypes =
- cpu_to_le16(max_ptype - next_ptype_id);
+ num_ptypes = max_ptype - next_ptype_id;
else
- get_ptype_info->num_ptypes =
- cpu_to_le16(IDPF_RX_MAX_PTYPES_PER_BUF);
+ num_ptypes = IDPF_RX_MAX_PTYPES_PER_BUF;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
+ get_ptype_info->num_ptypes = cpu_to_le16(num_ptypes);
+ err = idpf_send_mb_msg_kfree(adapter, &xn_params,
+ get_ptype_info, buf_size);
+ if (err)
+ return err;
+ ptype_info = xn_params.recv_mem.iov_base;
+ if (xn_params.recv_mem.iov_len < sizeof(*ptype_info)) {
+ err = -EIO;
+ goto free_rx_buf;
+ }
ptypes_recvd += le16_to_cpu(ptype_info->num_ptypes);
- if (ptypes_recvd > max_ptype)
- return -EINVAL;
-
- next_ptype_id = le16_to_cpu(get_ptype_info->start_ptype_id) +
- le16_to_cpu(get_ptype_info->num_ptypes);
+ if (ptypes_recvd > max_ptype) {
+ err = -EINVAL;
+ goto free_rx_buf;
+ }
+ next_ptype_id = next_ptype_id + num_ptypes;
ptype_offset = IDPF_RX_PTYPE_HDR_SZ;
for (u16 i = 0; i < le16_to_cpu(ptype_info->num_ptypes); i++) {
@@ -3182,15 +2772,25 @@ static int idpf_send_get_rx_ptype_msg(struct idpf_adapter *adapter)
pt_10 = le16_to_cpu(ptype->ptype_id_10);
pt_8 = ptype->ptype_id_8;
+ if (xn_params.recv_mem.iov_len <
+ ptype_offset + sizeof(struct virtchnl2_ptype)) {
+ err = -EINVAL;
+ goto free_rx_buf;
+ }
+
ptype_offset += IDPF_GET_PTYPE_SIZE(ptype);
- if (ptype_offset > IDPF_CTLQ_MAX_BUF_LEN)
- return -EINVAL;
+ if (xn_params.recv_mem.iov_len < ptype_offset) {
+ err = -EINVAL;
+ goto free_rx_buf;
+ }
/* 0xFFFF indicates end of ptypes */
if (pt_10 == IDPF_INVALID_PTYPE_ID)
goto out;
- if (pt_10 >= max_ptype)
- return -EINVAL;
+ if (pt_10 >= max_ptype) {
+ err = -EINVAL;
+ goto free_rx_buf;
+ }
idpf_parse_protocol_ids(ptype, &rx_pt);
idpf_finalize_ptype_lookup(&rx_pt);
@@ -3204,13 +2804,18 @@ static int idpf_send_get_rx_ptype_msg(struct idpf_adapter *adapter)
if (!singleq_pt_lkup[pt_8].outer_ip)
singleq_pt_lkup[pt_8] = rx_pt;
}
+
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+ xn_params.recv_mem = (struct kvec) {};
}
out:
adapter->splitq_pt_lkup = no_free_ptr(splitq_pt_lkup);
adapter->singleq_pt_lkup = no_free_ptr(singleq_pt_lkup);
+free_rx_buf:
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
- return 0;
+ return err;
}
/**
@@ -3238,40 +2843,24 @@ static void idpf_rel_rx_pt_lkup(struct idpf_adapter *adapter)
int idpf_send_ena_dis_loopback_msg(struct idpf_adapter *adapter, u32 vport_id,
bool loopback_ena)
{
- struct idpf_vc_xn_params xn_params = {};
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_LOOPBACK,
+ };
struct virtchnl2_loopback loopback;
- ssize_t reply_sz;
+ int err;
loopback.vport_id = cpu_to_le32(vport_id);
loopback.enable = loopback_ena;
- xn_params.vc_op = VIRTCHNL2_OP_LOOPBACK;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- xn_params.send_buf.iov_base = &loopback;
- xn_params.send_buf.iov_len = sizeof(loopback);
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
-
- return reply_sz < 0 ? reply_sz : 0;
-}
-
-/**
- * idpf_find_ctlq - Given a type and id, find ctlq info
- * @hw: hardware struct
- * @type: type of ctrlq to find
- * @id: ctlq id to find
- *
- * Returns pointer to found ctlq info struct, NULL otherwise.
- */
-static struct idpf_ctlq_info *idpf_find_ctlq(struct idpf_hw *hw,
- enum idpf_ctlq_type type, int id)
-{
- struct idpf_ctlq_info *cq, *tmp;
+ err = idpf_send_mb_msg(adapter, &xn_params, &loopback,
+ sizeof(loopback));
+ if (err)
+ return err;
- list_for_each_entry_safe(cq, tmp, &hw->cq_list_head, cq_list)
- if (cq->q_id == id && cq->cq_type == type)
- return cq;
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
- return NULL;
+ return 0;
}
/**
@@ -3282,40 +2871,43 @@ static struct idpf_ctlq_info *idpf_find_ctlq(struct idpf_hw *hw,
*/
int idpf_init_dflt_mbx(struct idpf_adapter *adapter)
{
- struct idpf_ctlq_create_info ctlq_info[] = {
+ struct libie_ctlq_ctx *ctx = &adapter->ctlq_ctx;
+ struct libie_ctlq_create_info ctlq_info[] = {
{
- .type = IDPF_CTLQ_TYPE_MAILBOX_TX,
- .id = IDPF_DFLT_MBX_ID,
+ .type = LIBIE_CTLQ_TYPE_TX,
+ .id = LIBIE_CTLQ_MBX_ID,
.len = IDPF_DFLT_MBX_Q_LEN,
- .buf_size = IDPF_CTLQ_MAX_BUF_LEN
},
{
- .type = IDPF_CTLQ_TYPE_MAILBOX_RX,
- .id = IDPF_DFLT_MBX_ID,
+ .type = LIBIE_CTLQ_TYPE_RX,
+ .id = LIBIE_CTLQ_MBX_ID,
.len = IDPF_DFLT_MBX_Q_LEN,
- .buf_size = IDPF_CTLQ_MAX_BUF_LEN
}
};
- struct idpf_hw *hw = &adapter->hw;
+ struct libie_ctlq_xn_init_params params = {
+ .num_qs = IDPF_NUM_DFLT_MBX_Q,
+ .cctlq_info = ctlq_info,
+ .ctx = ctx,
+ };
int err;
- adapter->dev_ops.reg_ops.ctlq_reg_init(adapter, ctlq_info);
+ adapter->dev_ops.reg_ops.ctlq_reg_init(&ctx->mmio_info,
+ params.cctlq_info);
- err = idpf_ctlq_init(hw, IDPF_NUM_DFLT_MBX_Q, ctlq_info);
+ err = libie_ctlq_xn_init(¶ms);
if (err)
return err;
- hw->asq = idpf_find_ctlq(hw, IDPF_CTLQ_TYPE_MAILBOX_TX,
- IDPF_DFLT_MBX_ID);
- hw->arq = idpf_find_ctlq(hw, IDPF_CTLQ_TYPE_MAILBOX_RX,
- IDPF_DFLT_MBX_ID);
-
- if (!hw->asq || !hw->arq) {
- idpf_ctlq_deinit(hw);
-
+ adapter->asq = libie_find_ctlq(ctx, LIBIE_CTLQ_TYPE_TX,
+ LIBIE_CTLQ_MBX_ID);
+ adapter->arq = libie_find_ctlq(ctx, LIBIE_CTLQ_TYPE_RX,
+ LIBIE_CTLQ_MBX_ID);
+ if (!adapter->asq || !adapter->arq) {
+ libie_ctlq_xn_deinit(params.xnm, ctx);
return -ENOENT;
}
+ adapter->xnm = params.xnm;
adapter->state = __IDPF_VER_CHECK;
return 0;
@@ -3327,12 +2919,14 @@ int idpf_init_dflt_mbx(struct idpf_adapter *adapter)
*/
void idpf_deinit_dflt_mbx(struct idpf_adapter *adapter)
{
- if (adapter->hw.arq && adapter->hw.asq) {
- idpf_mb_clean(adapter, adapter->hw.asq);
- idpf_ctlq_deinit(&adapter->hw);
+ if (adapter->xnm) {
+ idpf_mb_clean(adapter, adapter->asq, true);
+ libie_ctlq_xn_deinit(adapter->xnm, &adapter->ctlq_ctx);
}
- adapter->hw.arq = NULL;
- adapter->hw.asq = NULL;
+
+ adapter->arq = NULL;
+ adapter->asq = NULL;
+ adapter->xnm = NULL;
}
/**
@@ -3403,15 +2997,6 @@ int idpf_vc_core_init(struct idpf_adapter *adapter)
u16 num_max_vports;
int err = 0;
- if (!adapter->vcxn_mngr) {
- adapter->vcxn_mngr = kzalloc_obj(*adapter->vcxn_mngr);
- if (!adapter->vcxn_mngr) {
- err = -ENOMEM;
- goto init_failed;
- }
- }
- idpf_vc_xn_init(adapter->vcxn_mngr);
-
while (adapter->state != __IDPF_INIT_SW) {
switch (adapter->state) {
case __IDPF_VER_CHECK:
@@ -3558,8 +3143,7 @@ int idpf_vc_core_init(struct idpf_adapter *adapter)
* the mailbox again
*/
adapter->state = __IDPF_VER_CHECK;
- if (adapter->vcxn_mngr)
- idpf_vc_xn_shutdown(adapter->vcxn_mngr);
+ idpf_deinit_dflt_mbx(adapter);
set_bit(IDPF_HR_DRV_LOAD, adapter->flags);
queue_delayed_work(adapter->vc_event_wq, &adapter->vc_event_task,
msecs_to_jiffies(task_delay));
@@ -3582,7 +3166,7 @@ void idpf_vc_core_deinit(struct idpf_adapter *adapter)
/* Avoid transaction timeouts when called during reset */
remove_in_prog = test_bit(IDPF_REMOVE_IN_PROG, adapter->flags);
if (!remove_in_prog)
- idpf_vc_xn_shutdown(adapter->vcxn_mngr);
+ idpf_deinit_dflt_mbx(adapter);
idpf_ptp_release(adapter);
idpf_deinit_task(adapter);
@@ -3591,7 +3175,7 @@ void idpf_vc_core_deinit(struct idpf_adapter *adapter)
idpf_intr_rel(adapter);
if (remove_in_prog)
- idpf_vc_xn_shutdown(adapter->vcxn_mngr);
+ idpf_deinit_dflt_mbx(adapter);
cancel_delayed_work_sync(&adapter->serv_task);
cancel_delayed_work_sync(&adapter->mbx_task);
@@ -4128,9 +3712,9 @@ static void idpf_set_mac_type(const u8 *default_mac_addr,
/**
* idpf_mac_filter_async_handler - Async callback for mac filters
- * @adapter: private data struct
- * @xn: transaction for message
- * @ctlq_msg: received message
+ * @ctx: controlq context structure
+ * @buff: response buffer pointer and size
+ * @status: async call return value
*
* In some scenarios driver can't sleep and wait for a reply (e.g.: stack is
* holding rtnl_lock) when adding a new mac filter. It puts us in a difficult
@@ -4138,13 +3722,14 @@ static void idpf_set_mac_type(const u8 *default_mac_addr,
* ultimately do is remove it from our list of mac filters and report the
* error.
*/
-static int idpf_mac_filter_async_handler(struct idpf_adapter *adapter,
- struct idpf_vc_xn *xn,
- const struct idpf_ctlq_msg *ctlq_msg)
+static void idpf_mac_filter_async_handler(void *ctx,
+ struct kvec *buff,
+ int status)
{
struct virtchnl2_mac_addr_list *ma_list;
struct idpf_vport_config *vport_config;
struct virtchnl2_mac_addr *mac_addr;
+ struct idpf_adapter *adapter = ctx;
struct idpf_mac_filter *f, *tmp;
struct list_head *ma_list_head;
struct idpf_vport *vport;
@@ -4152,18 +3737,18 @@ static int idpf_mac_filter_async_handler(struct idpf_adapter *adapter,
int i;
/* if success we're done, we're only here if something bad happened */
- if (!ctlq_msg->cookie.mbx.chnl_retval)
- return 0;
+ if (!status || status == -ETIMEDOUT)
+ return;
+ ma_list = buff->iov_base;
/* make sure at least struct is there */
- if (xn->reply_sz < sizeof(*ma_list))
+ if (buff->iov_len < sizeof(*ma_list))
goto invalid_payload;
- ma_list = ctlq_msg->ctx.indirect.payload->va;
mac_addr = ma_list->mac_addr_list;
num_entries = le16_to_cpu(ma_list->num_mac_addr);
/* we should have received a buffer at least this big */
- if (xn->reply_sz < struct_size(ma_list, mac_addr_list, num_entries))
+ if (buff->iov_len < struct_size(ma_list, mac_addr_list, num_entries))
goto invalid_payload;
vport = idpf_vid_to_vport(adapter, le32_to_cpu(ma_list->vport_id));
@@ -4183,16 +3768,13 @@ static int idpf_mac_filter_async_handler(struct idpf_adapter *adapter,
if (ether_addr_equal(mac_addr[i].addr, f->macaddr))
list_del(&f->list);
spin_unlock_bh(&vport_config->mac_filter_list_lock);
- dev_err_ratelimited(&adapter->pdev->dev, "Received error sending MAC filter request (op %d)\n",
- xn->vc_op);
-
- return 0;
+ dev_err_ratelimited(&adapter->pdev->dev, "Received error %d on sending MAC filter request\n",
+ status);
+ return;
invalid_payload:
- dev_err_ratelimited(&adapter->pdev->dev, "Received invalid MAC filter payload (op %d) (len %zd)\n",
- xn->vc_op, xn->reply_sz);
-
- return -EINVAL;
+ dev_err_ratelimited(&adapter->pdev->dev, "Received invalid MAC filter payload (len %zd)\n",
+ buff->iov_len);
}
/**
@@ -4211,19 +3793,21 @@ int idpf_add_del_mac_filters(struct idpf_adapter *adapter,
const u8 *default_mac_addr, u32 vport_id,
bool add, bool async)
{
- struct virtchnl2_mac_addr_list *ma_list __free(kfree) = NULL;
struct virtchnl2_mac_addr *mac_addr __free(kfree) = NULL;
- struct idpf_vc_xn_params xn_params = {};
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = add ? VIRTCHNL2_OP_ADD_MAC_ADDR :
+ VIRTCHNL2_OP_DEL_MAC_ADDR,
+ };
+ struct virtchnl2_mac_addr_list *ma_list;
u32 num_msgs, total_filters = 0;
struct idpf_mac_filter *f;
- ssize_t reply_sz;
- int i = 0, k;
+ int i = 0;
- xn_params.vc_op = add ? VIRTCHNL2_OP_ADD_MAC_ADDR :
- VIRTCHNL2_OP_DEL_MAC_ADDR;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- xn_params.async = async;
- xn_params.async_handler = idpf_mac_filter_async_handler;
+ if (async) {
+ xn_params.resp_cb = idpf_mac_filter_async_handler;
+ xn_params.send_ctx = adapter;
+ }
spin_lock_bh(&vport_config->mac_filter_list_lock);
@@ -4278,32 +3862,31 @@ int idpf_add_del_mac_filters(struct idpf_adapter *adapter,
*/
num_msgs = DIV_ROUND_UP(total_filters, IDPF_NUM_FILTERS_PER_MSG);
- for (i = 0, k = 0; i < num_msgs; i++) {
- u32 entries_size, buf_size, num_entries;
+ for (u32 i = 0, k = 0; i < num_msgs; i++) {
+ u32 entries_size, num_entries;
+ size_t buf_size;
+ int err;
num_entries = min_t(u32, total_filters,
IDPF_NUM_FILTERS_PER_MSG);
entries_size = sizeof(struct virtchnl2_mac_addr) * num_entries;
buf_size = struct_size(ma_list, mac_addr_list, num_entries);
- if (!ma_list || num_entries != IDPF_NUM_FILTERS_PER_MSG) {
- kfree(ma_list);
- ma_list = kzalloc(buf_size, GFP_ATOMIC);
- if (!ma_list)
- return -ENOMEM;
- } else {
- memset(ma_list, 0, buf_size);
- }
+ ma_list = kzalloc(buf_size, GFP_ATOMIC);
+ if (!ma_list)
+ return -ENOMEM;
ma_list->vport_id = cpu_to_le32(vport_id);
ma_list->num_mac_addr = cpu_to_le16(num_entries);
memcpy(ma_list->mac_addr_list, &mac_addr[k], entries_size);
- xn_params.send_buf.iov_base = ma_list;
- xn_params.send_buf.iov_len = buf_size;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
+ err = idpf_send_mb_msg_kfree(adapter, &xn_params, ma_list,
+ buf_size);
+ if (err)
+ return err;
+
+ if (!async)
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
k += num_entries;
total_filters -= num_entries;
@@ -4312,6 +3895,26 @@ int idpf_add_del_mac_filters(struct idpf_adapter *adapter,
return 0;
}
+/**
+ * idpf_promiscuous_async_handler - async callback for promiscuous mode
+ * @ctx: controlq context structure
+ * @buff: response buffer pointer and size
+ * @status: async call return value
+ *
+ * Nobody is waiting for the promiscuous virtchnl message response. Print
+ * an error message if something went wrong and return.
+ */
+static void idpf_promiscuous_async_handler(void *ctx,
+ struct kvec *buff,
+ int status)
+{
+ struct idpf_adapter *adapter = ctx;
+
+ if (status)
+ dev_err_ratelimited(&adapter->pdev->dev, "Failed to set promiscuous mode: %d\n",
+ status);
+}
+
/**
* idpf_set_promiscuous - set promiscuous and send message to mailbox
* @adapter: Driver specific private structure
@@ -4326,9 +3929,13 @@ int idpf_set_promiscuous(struct idpf_adapter *adapter,
struct idpf_vport_user_config_data *config_data,
u32 vport_id)
{
- struct idpf_vc_xn_params xn_params = {};
+ struct libie_ctlq_xn_send_params xn_params = {
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ .chnl_opcode = VIRTCHNL2_OP_CONFIG_PROMISCUOUS_MODE,
+ .resp_cb = idpf_promiscuous_async_handler,
+ .send_ctx = adapter,
+ };
struct virtchnl2_promisc_info vpi;
- ssize_t reply_sz;
u16 flags = 0;
if (test_bit(__IDPF_PROMISC_UC, config_data->user_flags))
@@ -4339,15 +3946,7 @@ int idpf_set_promiscuous(struct idpf_adapter *adapter,
vpi.vport_id = cpu_to_le32(vport_id);
vpi.flags = cpu_to_le16(flags);
- xn_params.vc_op = VIRTCHNL2_OP_CONFIG_PROMISCUOUS_MODE;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- xn_params.send_buf.iov_base = &vpi;
- xn_params.send_buf.iov_len = sizeof(vpi);
- /* setting promiscuous is only ever done asynchronously */
- xn_params.async = true;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
-
- return reply_sz < 0 ? reply_sz : 0;
+ return idpf_send_mb_msg(adapter, &xn_params, &vpi, sizeof(vpi));
}
/**
@@ -4365,26 +3964,39 @@ int idpf_idc_rdma_vc_send_sync(struct iidc_rdma_core_dev_info *cdev_info,
u8 *recv_msg, u16 *recv_len)
{
struct idpf_adapter *adapter = pci_get_drvdata(cdev_info->pdev);
- struct idpf_vc_xn_params xn_params = { };
- ssize_t reply_sz;
- u16 recv_size;
+ struct libie_ctlq_xn_send_params xn_params = {
+ .chnl_opcode = VIRTCHNL2_OP_RDMA,
+ .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+ };
+ u8 on_stack_buf[LIBIE_CP_TX_COPYBREAK];
+ void *send_buf;
+ int err;
- if (!recv_msg || !recv_len || msg_size > IDPF_CTLQ_MAX_BUF_LEN)
+ if (!recv_msg || !recv_len || msg_size > LIBIE_CTLQ_MAX_BUF_LEN)
return -EINVAL;
- recv_size = min_t(u16, *recv_len, IDPF_CTLQ_MAX_BUF_LEN);
- *recv_len = 0;
- xn_params.vc_op = VIRTCHNL2_OP_RDMA;
- xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC;
- xn_params.send_buf.iov_base = send_msg;
- xn_params.send_buf.iov_len = msg_size;
- xn_params.recv_buf.iov_base = recv_msg;
- xn_params.recv_buf.iov_len = recv_size;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
- *recv_len = reply_sz;
+ if (!libie_cp_can_send_onstack(msg_size)) {
+ send_buf = kzalloc(msg_size, GFP_KERNEL);
+ if (!send_buf)
+ return -ENOMEM;
+ } else {
+ send_buf = on_stack_buf;
+ }
- return 0;
+ memcpy(send_buf, send_msg, msg_size);
+ err = idpf_send_mb_msg(adapter, &xn_params, send_buf, msg_size);
+ if (err)
+ return err;
+
+ if (xn_params.recv_mem.iov_len > *recv_len) {
+ err = -EINVAL;
+ goto rel_buf;
+ }
+
+ *recv_len = xn_params.recv_mem.iov_len;
+ memcpy(recv_msg, xn_params.recv_mem.iov_base, *recv_len);
+rel_buf:
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+ return err;
}
EXPORT_SYMBOL_GPL(idpf_idc_rdma_vc_send_sync);
diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h
index 7210dc7b426b..0c32be40fd16 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h
+++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h
@@ -7,86 +7,6 @@
#include <linux/net/intel/virtchnl2.h>
#define IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC (60 * 1000)
-#define IDPF_VC_XN_IDX_M GENMASK(7, 0)
-#define IDPF_VC_XN_SALT_M GENMASK(15, 8)
-#define IDPF_VC_XN_RING_LEN U8_MAX
-
-/**
- * enum idpf_vc_xn_state - Virtchnl transaction status
- * @IDPF_VC_XN_IDLE: not expecting a reply, ready to be used
- * @IDPF_VC_XN_WAITING: expecting a reply, not yet received
- * @IDPF_VC_XN_COMPLETED_SUCCESS: a reply was expected and received, buffer
- * updated
- * @IDPF_VC_XN_COMPLETED_FAILED: a reply was expected and received, but there
- * was an error, buffer not updated
- * @IDPF_VC_XN_SHUTDOWN: transaction object cannot be used, VC torn down
- * @IDPF_VC_XN_ASYNC: transaction sent asynchronously and doesn't have the
- * return context; a callback may be provided to handle
- * return
- */
-enum idpf_vc_xn_state {
- IDPF_VC_XN_IDLE = 1,
- IDPF_VC_XN_WAITING,
- IDPF_VC_XN_COMPLETED_SUCCESS,
- IDPF_VC_XN_COMPLETED_FAILED,
- IDPF_VC_XN_SHUTDOWN,
- IDPF_VC_XN_ASYNC,
-};
-
-struct idpf_vc_xn;
-/* Callback for asynchronous messages */
-typedef int (*async_vc_cb) (struct idpf_adapter *, struct idpf_vc_xn *,
- const struct idpf_ctlq_msg *);
-
-/**
- * struct idpf_vc_xn - Data structure representing virtchnl transactions
- * @completed: virtchnl event loop uses that to signal when a reply is
- * available, uses kernel completion API
- * @lock: protects the transaction state fields below
- * @state: virtchnl event loop stores the data below, protected by @lock
- * @reply_sz: Original size of reply, may be > reply_buf.iov_len; it will be
- * truncated on its way to the receiver thread according to
- * reply_buf.iov_len.
- * @reply: Reference to the buffer(s) where the reply data should be written
- * to. May be 0-length (then NULL address permitted) if the reply data
- * should be ignored.
- * @async_handler: if sent asynchronously, a callback can be provided to handle
- * the reply when it's received
- * @vc_op: corresponding opcode sent with this transaction
- * @idx: index used as retrieval on reply receive, used for cookie
- * @salt: changed every message to make unique, used for cookie
- */
-struct idpf_vc_xn {
- struct completion completed;
- spinlock_t lock;
- enum idpf_vc_xn_state state;
- size_t reply_sz;
- struct kvec reply;
- async_vc_cb async_handler;
- u32 vc_op;
- u8 idx;
- u8 salt;
-};
-
-/**
- * struct idpf_vc_xn_params - Parameters for executing transaction
- * @send_buf: kvec for send buffer
- * @recv_buf: kvec for recv buffer, may be NULL, must then have zero length
- * @timeout_ms: timeout to wait for reply
- * @async: send message asynchronously, will not wait on completion
- * @async_handler: If sent asynchronously, optional callback handler. The user
- * must be careful when using async handlers as the memory for
- * the recv_buf _cannot_ be on stack if this is async.
- * @vc_op: virtchnl op to send
- */
-struct idpf_vc_xn_params {
- struct kvec send_buf;
- struct kvec recv_buf;
- int timeout_ms;
- bool async;
- async_vc_cb async_handler;
- u32 vc_op;
-};
struct idpf_adapter;
struct idpf_netdev_priv;
@@ -96,8 +16,6 @@ struct idpf_vport_max_q;
struct idpf_vport_config;
struct idpf_vport_user_config_data;
-ssize_t idpf_vc_xn_exec(struct idpf_adapter *adapter,
- const struct idpf_vc_xn_params *params);
int idpf_init_dflt_mbx(struct idpf_adapter *adapter);
void idpf_deinit_dflt_mbx(struct idpf_adapter *adapter);
int idpf_vc_core_init(struct idpf_adapter *adapter);
@@ -124,9 +42,14 @@ bool idpf_sideband_action_ena(struct idpf_vport *vport,
struct ethtool_rx_flow_spec *fsp);
unsigned int idpf_fsteer_max_rules(struct idpf_vport *vport);
-int idpf_recv_mb_msg(struct idpf_adapter *adapter, struct idpf_ctlq_info *arq);
-int idpf_send_mb_msg(struct idpf_adapter *adapter, struct idpf_ctlq_info *asq,
- u32 op, u16 msg_size, u8 *msg, u16 cookie);
+void idpf_recv_event_msg(struct libie_ctlq_ctx *ctx,
+ struct libie_ctlq_msg *ctlq_msg);
+int idpf_send_mb_msg(struct idpf_adapter *adapter,
+ struct libie_ctlq_xn_send_params *xn_params,
+ void *send_buf, size_t send_buf_size);
+int idpf_send_mb_msg_kfree(struct idpf_adapter *adapter,
+ struct libie_ctlq_xn_send_params *xn_params,
+ void *send_buf, size_t send_buf_size);
struct idpf_queue_ptr {
enum virtchnl2_queue_type type;
@@ -212,7 +135,6 @@ int idpf_send_set_rss_key_msg(struct idpf_adapter *adapter,
struct idpf_rss_data *rss_data, u32 vport_id);
int idpf_send_set_rss_lut_msg(struct idpf_adapter *adapter,
struct idpf_rss_data *rss_data, u32 vport_id);
-void idpf_vc_xn_shutdown(struct idpf_vc_xn_manager *vcxn_mngr);
int idpf_idc_rdma_vc_send_sync(struct iidc_rdma_core_dev_info *cdev_info,
u8 *send_msg, u16 msg_size,
u8 *recv_msg, u16 *recv_len);
diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl_ptp.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl_ptp.c
index 8d8fb498e092..a71c041ec9e8 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl_ptp.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl_ptp.c
@@ -15,7 +15,6 @@
*/
int idpf_ptp_get_caps(struct idpf_adapter *adapter)
{
- struct virtchnl2_ptp_get_caps *recv_ptp_caps_msg __free(kfree) = NULL;
struct virtchnl2_ptp_get_caps send_ptp_caps_msg = {
.caps = cpu_to_le32(VIRTCHNL2_CAP_PTP_GET_DEVICE_CLK_TIME |
VIRTCHNL2_CAP_PTP_GET_DEVICE_CLK_TIME_MB |
@@ -24,34 +23,34 @@ int idpf_ptp_get_caps(struct idpf_adapter *adapter)
VIRTCHNL2_CAP_PTP_ADJ_DEVICE_CLK_MB |
VIRTCHNL2_CAP_PTP_TX_TSTAMPS_MB)
};
- struct idpf_vc_xn_params xn_params = {
- .vc_op = VIRTCHNL2_OP_PTP_GET_CAPS,
- .send_buf.iov_base = &send_ptp_caps_msg,
- .send_buf.iov_len = sizeof(send_ptp_caps_msg),
+ struct libie_ctlq_xn_send_params xn_params = {
+ .chnl_opcode = VIRTCHNL2_OP_PTP_GET_CAPS,
.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
};
struct virtchnl2_ptp_cross_time_reg_offsets cross_tstamp_offsets;
struct libie_mmio_info *mmio = &adapter->ctlq_ctx.mmio_info;
struct virtchnl2_ptp_clk_adj_reg_offsets clk_adj_offsets;
struct virtchnl2_ptp_clk_reg_offsets clock_offsets;
+ struct virtchnl2_ptp_get_caps *recv_ptp_caps_msg;
struct idpf_ptp_secondary_mbx *scnd_mbx;
struct idpf_ptp *ptp = adapter->ptp;
enum idpf_ptp_access access_type;
u32 temp_offset;
- int reply_sz;
+ size_t reply_sz;
+ int err;
- recv_ptp_caps_msg = kzalloc_obj(struct virtchnl2_ptp_get_caps);
- if (!recv_ptp_caps_msg)
- return -ENOMEM;
+ err = idpf_send_mb_msg(adapter, &xn_params, &send_ptp_caps_msg,
+ sizeof(send_ptp_caps_msg));
+ if (err)
+ return err;
- xn_params.recv_buf.iov_base = recv_ptp_caps_msg;
- xn_params.recv_buf.iov_len = sizeof(*recv_ptp_caps_msg);
+ reply_sz = xn_params.recv_mem.iov_len;
+ if (reply_sz != sizeof(*recv_ptp_caps_msg)) {
+ err = -EIO;
+ goto free_resp;
+ }
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
- else if (reply_sz != sizeof(*recv_ptp_caps_msg))
- return -EIO;
+ recv_ptp_caps_msg = xn_params.recv_mem.iov_base;
ptp->caps = le32_to_cpu(recv_ptp_caps_msg->caps);
ptp->base_incval = le64_to_cpu(recv_ptp_caps_msg->base_incval);
@@ -112,7 +111,7 @@ int idpf_ptp_get_caps(struct idpf_adapter *adapter)
discipline_clock:
access_type = ptp->adj_dev_clk_time_access;
if (access_type != IDPF_PTP_DIRECT)
- return 0;
+ goto free_resp;
clk_adj_offsets = recv_ptp_caps_msg->clk_adj_offsets;
@@ -145,7 +144,9 @@ int idpf_ptp_get_caps(struct idpf_adapter *adapter)
ptp->dev_clk_regs.phy_shadj_h =
libie_pci_get_mmio_addr(mmio, temp_offset);
- return 0;
+free_resp:
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+ return err;
}
/**
@@ -160,28 +161,34 @@ int idpf_ptp_get_caps(struct idpf_adapter *adapter)
int idpf_ptp_get_dev_clk_time(struct idpf_adapter *adapter,
struct idpf_ptp_dev_timers *dev_clk_time)
{
+ struct virtchnl2_ptp_get_dev_clk_time *get_dev_clk_time_resp;
struct virtchnl2_ptp_get_dev_clk_time get_dev_clk_time_msg;
- struct idpf_vc_xn_params xn_params = {
- .vc_op = VIRTCHNL2_OP_PTP_GET_DEV_CLK_TIME,
- .send_buf.iov_base = &get_dev_clk_time_msg,
- .send_buf.iov_len = sizeof(get_dev_clk_time_msg),
- .recv_buf.iov_base = &get_dev_clk_time_msg,
- .recv_buf.iov_len = sizeof(get_dev_clk_time_msg),
+ struct libie_ctlq_xn_send_params xn_params = {
+ .chnl_opcode = VIRTCHNL2_OP_PTP_GET_DEV_CLK_TIME,
.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
};
- int reply_sz;
+ size_t reply_sz;
u64 dev_time;
+ int err;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
- if (reply_sz != sizeof(get_dev_clk_time_msg))
- return -EIO;
+ err = idpf_send_mb_msg(adapter, &xn_params, &get_dev_clk_time_msg,
+ sizeof(get_dev_clk_time_msg));
+ if (err)
+ return err;
- dev_time = le64_to_cpu(get_dev_clk_time_msg.dev_time_ns);
+ reply_sz = xn_params.recv_mem.iov_len;
+ if (reply_sz != sizeof(*get_dev_clk_time_resp)) {
+ err = -EIO;
+ goto free_resp;
+ }
+
+ get_dev_clk_time_resp = xn_params.recv_mem.iov_base;
+ dev_time = le64_to_cpu(get_dev_clk_time_resp->dev_time_ns);
dev_clk_time->dev_clk_time_ns = dev_time;
- return 0;
+free_resp:
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+ return err;
}
/**
@@ -197,27 +204,30 @@ int idpf_ptp_get_dev_clk_time(struct idpf_adapter *adapter,
int idpf_ptp_get_cross_time(struct idpf_adapter *adapter,
struct idpf_ptp_dev_timers *cross_time)
{
- struct virtchnl2_ptp_get_cross_time cross_time_msg;
- struct idpf_vc_xn_params xn_params = {
- .vc_op = VIRTCHNL2_OP_PTP_GET_CROSS_TIME,
- .send_buf.iov_base = &cross_time_msg,
- .send_buf.iov_len = sizeof(cross_time_msg),
- .recv_buf.iov_base = &cross_time_msg,
- .recv_buf.iov_len = sizeof(cross_time_msg),
+ struct virtchnl2_ptp_get_cross_time cross_time_send, *cross_time_recv;
+ struct libie_ctlq_xn_send_params xn_params = {
+ .chnl_opcode = VIRTCHNL2_OP_PTP_GET_CROSS_TIME,
.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
};
- int reply_sz;
+ int err = 0;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
- if (reply_sz != sizeof(cross_time_msg))
- return -EIO;
+ err = idpf_send_mb_msg(adapter, &xn_params, &cross_time_send,
+ sizeof(cross_time_send));
+ if (err)
+ return err;
- cross_time->dev_clk_time_ns = le64_to_cpu(cross_time_msg.dev_time_ns);
- cross_time->sys_time_ns = le64_to_cpu(cross_time_msg.sys_time_ns);
+ if (xn_params.recv_mem.iov_len != sizeof(*cross_time_recv)) {
+ err = -EIO;
+ goto free_resp;
+ }
- return 0;
+ cross_time_recv = xn_params.recv_mem.iov_base;
+ cross_time->dev_clk_time_ns = le64_to_cpu(cross_time_recv->dev_time_ns);
+ cross_time->sys_time_ns = le64_to_cpu(cross_time_recv->sys_time_ns);
+
+free_resp:
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+ return err;
}
/**
@@ -234,23 +244,18 @@ int idpf_ptp_set_dev_clk_time(struct idpf_adapter *adapter, u64 time)
struct virtchnl2_ptp_set_dev_clk_time set_dev_clk_time_msg = {
.dev_time_ns = cpu_to_le64(time),
};
- struct idpf_vc_xn_params xn_params = {
- .vc_op = VIRTCHNL2_OP_PTP_SET_DEV_CLK_TIME,
- .send_buf.iov_base = &set_dev_clk_time_msg,
- .send_buf.iov_len = sizeof(set_dev_clk_time_msg),
- .recv_buf.iov_base = &set_dev_clk_time_msg,
- .recv_buf.iov_len = sizeof(set_dev_clk_time_msg),
+ struct libie_ctlq_xn_send_params xn_params = {
+ .chnl_opcode = VIRTCHNL2_OP_PTP_SET_DEV_CLK_TIME,
.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
};
- int reply_sz;
+ int err;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
- if (reply_sz != sizeof(set_dev_clk_time_msg))
- return -EIO;
+ err = idpf_send_mb_msg(adapter, &xn_params, &set_dev_clk_time_msg,
+ sizeof(set_dev_clk_time_msg));
+ if (!err)
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
- return 0;
+ return err;
}
/**
@@ -267,23 +272,18 @@ int idpf_ptp_adj_dev_clk_time(struct idpf_adapter *adapter, s64 delta)
struct virtchnl2_ptp_adj_dev_clk_time adj_dev_clk_time_msg = {
.delta = cpu_to_le64(delta),
};
- struct idpf_vc_xn_params xn_params = {
- .vc_op = VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_TIME,
- .send_buf.iov_base = &adj_dev_clk_time_msg,
- .send_buf.iov_len = sizeof(adj_dev_clk_time_msg),
- .recv_buf.iov_base = &adj_dev_clk_time_msg,
- .recv_buf.iov_len = sizeof(adj_dev_clk_time_msg),
+ struct libie_ctlq_xn_send_params xn_params = {
+ .chnl_opcode = VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_TIME,
.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
};
- int reply_sz;
+ int err;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
- if (reply_sz != sizeof(adj_dev_clk_time_msg))
- return -EIO;
+ err = idpf_send_mb_msg(adapter, &xn_params, &adj_dev_clk_time_msg,
+ sizeof(adj_dev_clk_time_msg));
+ if (!err)
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
- return 0;
+ return err;
}
/**
@@ -301,23 +301,18 @@ int idpf_ptp_adj_dev_clk_fine(struct idpf_adapter *adapter, u64 incval)
struct virtchnl2_ptp_adj_dev_clk_fine adj_dev_clk_fine_msg = {
.incval = cpu_to_le64(incval),
};
- struct idpf_vc_xn_params xn_params = {
- .vc_op = VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_FINE,
- .send_buf.iov_base = &adj_dev_clk_fine_msg,
- .send_buf.iov_len = sizeof(adj_dev_clk_fine_msg),
- .recv_buf.iov_base = &adj_dev_clk_fine_msg,
- .recv_buf.iov_len = sizeof(adj_dev_clk_fine_msg),
+ struct libie_ctlq_xn_send_params xn_params = {
+ .chnl_opcode = VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_FINE,
.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
};
- int reply_sz;
+ int err;
- reply_sz = idpf_vc_xn_exec(adapter, &xn_params);
- if (reply_sz < 0)
- return reply_sz;
- if (reply_sz != sizeof(adj_dev_clk_fine_msg))
- return -EIO;
+ err = idpf_send_mb_msg(adapter, &xn_params, &adj_dev_clk_fine_msg,
+ sizeof(adj_dev_clk_fine_msg));
+ if (!err)
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
- return 0;
+ return err;
}
/**
@@ -336,18 +331,16 @@ int idpf_ptp_get_vport_tstamps_caps(struct idpf_vport *vport)
struct virtchnl2_ptp_tx_tstamp_latch_caps tx_tstamp_latch_caps;
struct idpf_ptp_vport_tx_tstamp_caps *tstamp_caps;
struct idpf_ptp_tx_tstamp *ptp_tx_tstamp, *tmp;
- struct idpf_vc_xn_params xn_params = {
- .vc_op = VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP_CAPS,
- .send_buf.iov_base = &send_tx_tstamp_caps,
- .send_buf.iov_len = sizeof(send_tx_tstamp_caps),
- .recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN,
+ struct libie_ctlq_xn_send_params xn_params = {
+ .chnl_opcode = VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP_CAPS,
.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
};
enum idpf_ptp_access tstamp_access, get_dev_clk_access;
struct idpf_ptp *ptp = vport->adapter->ptp;
struct list_head *head;
- int err = 0, reply_sz;
+ size_t reply_sz;
u16 num_latches;
+ int err = 0;
u32 size;
if (!ptp)
@@ -359,19 +352,19 @@ int idpf_ptp_get_vport_tstamps_caps(struct idpf_vport *vport)
get_dev_clk_access == IDPF_PTP_NONE)
return -EOPNOTSUPP;
- rcv_tx_tstamp_caps = kzalloc(IDPF_CTLQ_MAX_BUF_LEN, GFP_KERNEL);
- if (!rcv_tx_tstamp_caps)
- return -ENOMEM;
-
send_tx_tstamp_caps.vport_id = cpu_to_le32(vport->vport_id);
- xn_params.recv_buf.iov_base = rcv_tx_tstamp_caps;
- reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params);
- if (reply_sz < 0) {
- err = reply_sz;
+ err = idpf_send_mb_msg(vport->adapter, &xn_params, &send_tx_tstamp_caps,
+ sizeof(send_tx_tstamp_caps));
+ if (err)
+ return err;
+
+ rcv_tx_tstamp_caps = xn_params.recv_mem.iov_base;
+ reply_sz = xn_params.recv_mem.iov_len;
+ if (reply_sz < sizeof(*rcv_tx_tstamp_caps)) {
+ err = -EIO;
goto get_tstamp_caps_out;
}
-
num_latches = le16_to_cpu(rcv_tx_tstamp_caps->num_latches);
size = struct_size(rcv_tx_tstamp_caps, tstamp_latches, num_latches);
if (reply_sz != size) {
@@ -426,7 +419,7 @@ int idpf_ptp_get_vport_tstamps_caps(struct idpf_vport *vport)
}
vport->tx_tstamp_caps = tstamp_caps;
- kfree(rcv_tx_tstamp_caps);
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
return 0;
@@ -439,7 +432,7 @@ int idpf_ptp_get_vport_tstamps_caps(struct idpf_vport *vport)
kfree(tstamp_caps);
get_tstamp_caps_out:
- kfree(rcv_tx_tstamp_caps);
+ libie_ctlq_release_rx_buf(&xn_params.recv_mem);
return err;
}
@@ -536,9 +529,9 @@ idpf_ptp_get_tstamp_value(struct idpf_vport *vport,
/**
* idpf_ptp_get_tx_tstamp_async_handler - Async callback for getting Tx tstamps
- * @adapter: Driver specific private structure
- * @xn: transaction for message
- * @ctlq_msg: received message
+ * @ctx: adapter pointer
+ * @mem: address and size of the response
+ * @status: return value of the request
*
* Read the tstamps Tx tstamp values from a received message and put them
* directly to the skb. The number of timestamps to read is specified by
@@ -546,22 +539,26 @@ idpf_ptp_get_tstamp_value(struct idpf_vport *vport,
*
* Return: 0 on success, -errno otherwise.
*/
-static int
-idpf_ptp_get_tx_tstamp_async_handler(struct idpf_adapter *adapter,
- struct idpf_vc_xn *xn,
- const struct idpf_ctlq_msg *ctlq_msg)
+static void
+idpf_ptp_get_tx_tstamp_async_handler(void *ctx, struct kvec *mem, int status)
{
struct virtchnl2_ptp_get_vport_tx_tstamp_latches *recv_tx_tstamp_msg;
struct idpf_ptp_vport_tx_tstamp_caps *tx_tstamp_caps;
struct virtchnl2_ptp_tx_tstamp_latch tstamp_latch;
struct idpf_ptp_tx_tstamp *tx_tstamp, *tmp;
struct idpf_vport *tstamp_vport = NULL;
+ struct idpf_adapter *adapter = ctx;
struct list_head *head;
u16 num_latches;
u32 vport_id;
- int err = 0;
- recv_tx_tstamp_msg = ctlq_msg->ctx.indirect.payload->va;
+ if (status)
+ return;
+
+ recv_tx_tstamp_msg = mem->iov_base;
+ if (mem->iov_len < sizeof(*recv_tx_tstamp_msg))
+ return;
+
vport_id = le32_to_cpu(recv_tx_tstamp_msg->vport_id);
idpf_for_each_vport(adapter, vport) {
@@ -575,7 +572,7 @@ idpf_ptp_get_tx_tstamp_async_handler(struct idpf_adapter *adapter,
}
if (!tstamp_vport || !tstamp_vport->tx_tstamp_caps)
- return -EINVAL;
+ return;
tx_tstamp_caps = tstamp_vport->tx_tstamp_caps;
num_latches = le16_to_cpu(recv_tx_tstamp_msg->num_latches);
@@ -589,13 +586,13 @@ idpf_ptp_get_tx_tstamp_async_handler(struct idpf_adapter *adapter,
if (!tstamp_latch.valid)
continue;
- if (list_empty(head)) {
- err = -ENOBUFS;
+ if (list_empty(head))
goto unlock;
- }
list_for_each_entry_safe(tx_tstamp, tmp, head, list_member) {
if (tstamp_latch.index == tx_tstamp->idx) {
+ int err;
+
list_del(&tx_tstamp->list_member);
err = idpf_ptp_get_tstamp_value(tstamp_vport,
&tstamp_latch,
@@ -610,8 +607,6 @@ idpf_ptp_get_tx_tstamp_async_handler(struct idpf_adapter *adapter,
unlock:
spin_unlock_bh(&tx_tstamp_caps->latches_lock);
-
- return err;
}
/**
@@ -627,15 +622,15 @@ int idpf_ptp_get_tx_tstamp(struct idpf_vport *vport)
{
struct virtchnl2_ptp_get_vport_tx_tstamp_latches *send_tx_tstamp_msg;
struct idpf_ptp_vport_tx_tstamp_caps *tx_tstamp_caps;
- struct idpf_vc_xn_params xn_params = {
- .vc_op = VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP,
+ struct libie_ctlq_xn_send_params xn_params = {
+ .chnl_opcode = VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP,
.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
- .async = true,
- .async_handler = idpf_ptp_get_tx_tstamp_async_handler,
+ .resp_cb = idpf_ptp_get_tx_tstamp_async_handler,
+ .send_ctx = vport->adapter,
};
struct idpf_ptp_tx_tstamp *ptp_tx_tstamp;
- int reply_sz, size, msg_size;
struct list_head *head;
+ int size, msg_size;
bool state_upd;
u16 id = 0;
@@ -668,11 +663,7 @@ int idpf_ptp_get_tx_tstamp(struct idpf_vport *vport)
msg_size = struct_size(send_tx_tstamp_msg, tstamp_latches, id);
send_tx_tstamp_msg->vport_id = cpu_to_le32(vport->vport_id);
send_tx_tstamp_msg->num_latches = cpu_to_le16(id);
- xn_params.send_buf.iov_base = send_tx_tstamp_msg;
- xn_params.send_buf.iov_len = msg_size;
-
- reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params);
- kfree(send_tx_tstamp_msg);
- return min(reply_sz, 0);
+ return idpf_send_mb_msg_kfree(vport->adapter, &xn_params,
+ send_tx_tstamp_msg, msg_size);
}
--
2.47.0
^ permalink raw reply related
* [PATCH iwl-next v8 14/15] ixd: add the core initialization
From: Larysa Zaremba @ 2026-06-08 14:41 UTC (permalink / raw)
To: intel-wired-lan, Tony Nguyen
Cc: aleksander.lobakin, sridhar.samudrala, Michal Swiatkowski,
Larysa Zaremba, Fijalkowski, Maciej, Emil Tantilov, Madhu Chittim,
Josh Hay, Keller, Jacob E, jayaprakash.shanmugam, Jiri Pirko,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Simon Horman, Jonathan Corbet, Richard Cochran, Przemek Kitszel,
Andrew Lunn, netdev, linux-doc, linux-kernel, Bharath R
In-Reply-To: <20260608144127.2751230-1-larysa.zaremba@intel.com>
As the mailbox is setup, initialize the core. This makes use of the send
and receive mailbox message framework for virtchnl communication between
the driver and device Control Plane (CP).
To start with, driver confirms the virtchnl version with the CP. Once that
is done, it requests and gets the required capabilities and resources
needed such as max vectors, queues, vports etc.
Use a unified way of handling the virtchnl messages, where a single
function handles all related memory management and the caller only provides
the callbacks to fill the send buffer and to handle the response.
Place generic control queue message handling separately to facilitate the
addition of protocols other than virtchannel in the future.
Co-developed-by: Amritha Nambiar <amritha.nambiar@intel.com>
Signed-off-by: Amritha Nambiar <amritha.nambiar@intel.com>
Reviewed-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
Tested-by: Bharath R <Bharath.r@intel.com>
Signed-off-by: Larysa Zaremba <larysa.zaremba@intel.com>
---
drivers/net/ethernet/intel/ixd/Makefile | 2 +
drivers/net/ethernet/intel/ixd/ixd.h | 10 +
drivers/net/ethernet/intel/ixd/ixd_ctlq.c | 147 +++++++++++++++
drivers/net/ethernet/intel/ixd/ixd_ctlq.h | 34 ++++
drivers/net/ethernet/intel/ixd/ixd_lib.c | 29 ++-
drivers/net/ethernet/intel/ixd/ixd_main.c | 3 +
drivers/net/ethernet/intel/ixd/ixd_virtchnl.c | 178 ++++++++++++++++++
drivers/net/ethernet/intel/ixd/ixd_virtchnl.h | 12 ++
8 files changed, 414 insertions(+), 1 deletion(-)
create mode 100644 drivers/net/ethernet/intel/ixd/ixd_ctlq.c
create mode 100644 drivers/net/ethernet/intel/ixd/ixd_ctlq.h
create mode 100644 drivers/net/ethernet/intel/ixd/ixd_virtchnl.c
create mode 100644 drivers/net/ethernet/intel/ixd/ixd_virtchnl.h
diff --git a/drivers/net/ethernet/intel/ixd/Makefile b/drivers/net/ethernet/intel/ixd/Makefile
index 164b2c86952f..90abf231fb16 100644
--- a/drivers/net/ethernet/intel/ixd/Makefile
+++ b/drivers/net/ethernet/intel/ixd/Makefile
@@ -6,5 +6,7 @@
obj-$(CONFIG_IXD) += ixd.o
ixd-y := ixd_main.o
+ixd-y += ixd_ctlq.o
ixd-y += ixd_dev.o
ixd-y += ixd_lib.o
+ixd-y += ixd_virtchnl.o
diff --git a/drivers/net/ethernet/intel/ixd/ixd.h b/drivers/net/ethernet/intel/ixd/ixd.h
index c86c2b05c5b4..8e3cd5bc5a84 100644
--- a/drivers/net/ethernet/intel/ixd/ixd.h
+++ b/drivers/net/ethernet/intel/ixd/ixd.h
@@ -10,19 +10,29 @@
* struct ixd_adapter - Data structure representing a CPF
* @cp_ctx: Control plane communication context
* @init_task: Delayed initialization after reset
+ * @mbx_task: Control queue Rx handling
* @xnm: virtchnl transaction manager
* @asq: Send control queue info
* @arq: Receive control queue info
+ * @vc_ver: Negotiated virtchnl version
+ * @caps: Negotiated virtchnl capabilities
*/
struct ixd_adapter {
struct libie_ctlq_ctx cp_ctx;
struct {
struct delayed_work init_work;
u8 reset_retries;
+ u8 vc_retries;
} init_task;
+ struct delayed_work mbx_task;
struct libie_ctlq_xn_manager *xnm;
struct libie_ctlq_info *asq;
struct libie_ctlq_info *arq;
+ struct {
+ u32 major;
+ u32 minor;
+ } vc_ver;
+ struct virtchnl2_get_capabilities caps;
};
/**
diff --git a/drivers/net/ethernet/intel/ixd/ixd_ctlq.c b/drivers/net/ethernet/intel/ixd/ixd_ctlq.c
new file mode 100644
index 000000000000..c36ab8390c4c
--- /dev/null
+++ b/drivers/net/ethernet/intel/ixd/ixd_ctlq.c
@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2025 Intel Corporation */
+
+#include "ixd.h"
+#include "ixd_ctlq.h"
+#include "ixd_virtchnl.h"
+
+/**
+ * ixd_ctlq_clean_sq - Clean the send control queue after sending the message
+ * @adapter: The adapter that sent the messages
+ * @force: Clean regardless of send status
+ *
+ * Free the libie send resources after sending the message and handling
+ * the response.
+ */
+void ixd_ctlq_clean_sq(struct ixd_adapter *adapter, bool force)
+{
+ struct libie_ctlq_xn_clean_params params = {
+ .ctlq = adapter->asq,
+ .ctx = &adapter->cp_ctx,
+ .force = force,
+ .num_msgs = adapter->asq->ring_len,
+ .rel_tx_buf = kfree,
+ };
+
+ libie_ctlq_xn_send_clean(¶ms);
+}
+
+/**
+ * ixd_ctlq_init_sparams - Initialize control queue send parameters
+ * @adapter: The adapter with initialized mailbox
+ * @sparams: Parameters to initialize
+ * @msg_buf: DMA-mappable pointer to the message being sent
+ * @msg_size: Message size
+ */
+static void ixd_ctlq_init_sparams(struct ixd_adapter *adapter,
+ struct libie_ctlq_xn_send_params *sparams,
+ void *msg_buf, size_t msg_size)
+{
+ *sparams = (struct libie_ctlq_xn_send_params) {
+ .rel_tx_buf = kfree,
+ .xnm = adapter->xnm,
+ .ctlq = adapter->asq,
+ .timeout_ms = IXD_CTLQ_TIMEOUT,
+ .send_buf = (struct kvec) {
+ .iov_base = msg_buf,
+ .iov_len = msg_size,
+ },
+ };
+}
+
+/**
+ * ixd_ctlq_do_req - Perform a standard virtchnl request
+ * @adapter: The adapter with initialized mailbox
+ * @req: virtchnl request description
+ *
+ * Return: %0 if a message was sent and received a response
+ * that was successfully handled by the custom callback,
+ * negative error otherwise.
+ */
+int ixd_ctlq_do_req(struct ixd_adapter *adapter, const struct ixd_ctlq_req *req)
+{
+ u8 onstack_send_buff[LIBIE_CP_TX_COPYBREAK] __aligned_largest = {};
+ struct libie_ctlq_xn_send_params send_params = {};
+ struct kvec *recv_mem;
+ void *send_buff;
+ int err;
+
+ send_buff = libie_cp_can_send_onstack(req->send_size) ?
+ &onstack_send_buff : kzalloc(req->send_size, GFP_KERNEL);
+ if (!send_buff)
+ return -ENOMEM;
+
+ ixd_ctlq_init_sparams(adapter, &send_params, send_buff,
+ req->send_size);
+
+ send_params.chnl_opcode = req->opcode;
+
+ if (req->send_buff_init)
+ req->send_buff_init(adapter, send_buff, req->ctx);
+
+ ixd_ctlq_clean_sq(adapter, false);
+ err = libie_ctlq_xn_send(&send_params);
+ if (err)
+ return err;
+
+ recv_mem = &send_params.recv_mem;
+ if (req->recv_process)
+ err = req->recv_process(adapter, recv_mem->iov_base,
+ recv_mem->iov_len, req->ctx);
+
+ libie_ctlq_release_rx_buf(recv_mem);
+
+ return err;
+}
+
+/**
+ * ixd_ctlq_handle_msg - Default control queue message handler
+ * @ctx: Control plane communication context
+ * @msg: Message received
+ */
+static void ixd_ctlq_handle_msg(struct libie_ctlq_ctx *ctx,
+ struct libie_ctlq_msg *msg)
+{
+ struct ixd_adapter *adapter = pci_get_drvdata(ctx->mmio_info.pdev);
+
+ if (ixd_vc_can_handle_msg(msg))
+ ixd_vc_recv_event_msg(adapter, msg);
+ else
+ dev_dbg_ratelimited(ixd_to_dev(adapter),
+ "Received an unsupported opcode 0x%x from the CP\n",
+ msg->chnl_opcode);
+
+ libie_ctlq_release_rx_buf(&msg->recv_mem);
+}
+
+/**
+ * ixd_ctlq_recv_mb_msg - Receive a potential message over mailbox periodically
+ * @adapter: The adapter with initialized mailbox
+ */
+static void ixd_ctlq_recv_mb_msg(struct ixd_adapter *adapter)
+{
+ struct libie_ctlq_xn_recv_params xn_params = {
+ .xnm = adapter->xnm,
+ .ctlq = adapter->arq,
+ .ctlq_msg_handler = ixd_ctlq_handle_msg,
+ .budget = LIBIE_CTLQ_MAX_XN_ENTRIES,
+ };
+
+ libie_ctlq_xn_recv(&xn_params);
+}
+
+/**
+ * ixd_ctlq_rx_task - Periodically check for mailbox responses and events
+ * @work: work handle
+ */
+void ixd_ctlq_rx_task(struct work_struct *work)
+{
+ struct ixd_adapter *adapter;
+
+ adapter = container_of(work, struct ixd_adapter, mbx_task.work);
+
+ queue_delayed_work(system_unbound_wq, &adapter->mbx_task,
+ msecs_to_jiffies(300));
+
+ ixd_ctlq_recv_mb_msg(adapter);
+}
diff --git a/drivers/net/ethernet/intel/ixd/ixd_ctlq.h b/drivers/net/ethernet/intel/ixd/ixd_ctlq.h
new file mode 100644
index 000000000000..8839f9f8f6d5
--- /dev/null
+++ b/drivers/net/ethernet/intel/ixd/ixd_ctlq.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (C) 2025 Intel Corporation */
+
+#ifndef _IXD_CTLQ_H_
+#define _IXD_CTLQ_H_
+
+#include <linux/net/intel/virtchnl2.h>
+
+#define IXD_CTLQ_TIMEOUT 2000
+
+/**
+ * struct ixd_ctlq_req - Standard virtchnl request description
+ * @opcode: protocol opcode, only virtchnl2 is needed for now
+ * @send_size: required length of the send buffer
+ * @send_buff_init: function to initialize the allocated send buffer
+ * @recv_process: function to handle the CP response
+ * @ctx: additional context for callbacks
+ */
+struct ixd_ctlq_req {
+ enum virtchnl2_op opcode;
+ size_t send_size;
+ void (*send_buff_init)(struct ixd_adapter *adapter, void *send_buff,
+ void *ctx);
+ int (*recv_process)(struct ixd_adapter *adapter, void *recv_buff,
+ size_t recv_size, void *ctx);
+ void *ctx;
+};
+
+void ixd_ctlq_clean_sq(struct ixd_adapter *adapter, bool force);
+int ixd_ctlq_do_req(struct ixd_adapter *adapter,
+ const struct ixd_ctlq_req *req);
+void ixd_ctlq_rx_task(struct work_struct *work);
+
+#endif /* _IXD_CTLQ_H_ */
diff --git a/drivers/net/ethernet/intel/ixd/ixd_lib.c b/drivers/net/ethernet/intel/ixd/ixd_lib.c
index afc413d3650f..17f4aae594ca 100644
--- a/drivers/net/ethernet/intel/ixd/ixd_lib.c
+++ b/drivers/net/ethernet/intel/ixd/ixd_lib.c
@@ -2,6 +2,8 @@
/* Copyright (C) 2025 Intel Corporation */
#include "ixd.h"
+#include "ixd_ctlq.h"
+#include "ixd_virtchnl.h"
#define IXD_DFLT_MBX_Q_LEN 64
@@ -67,6 +69,11 @@ static void ixd_adapter_fill_dflt_ctlqs(struct ixd_adapter *adapter)
*/
void ixd_deinit_dflt_mbx(struct ixd_adapter *adapter)
{
+ cancel_delayed_work_sync(&adapter->mbx_task);
+
+ if (adapter->asq)
+ ixd_ctlq_clean_sq(adapter, true);
+
if (adapter->xnm)
libie_ctlq_xn_deinit(adapter->xnm, &adapter->cp_ctx);
@@ -108,6 +115,8 @@ int ixd_init_dflt_mbx(struct ixd_adapter *adapter)
return -ENOENT;
}
+ queue_delayed_work(system_unbound_wq, &adapter->mbx_task, 0);
+
return 0;
}
@@ -136,8 +145,26 @@ void ixd_init_task(struct work_struct *work)
adapter->init_task.reset_retries = 0;
err = ixd_init_dflt_mbx(adapter);
- if (err)
+ if (err) {
dev_err(ixd_to_dev(adapter),
"Failed to initialize the default mailbox: %pe\n",
ERR_PTR(err));
+ return;
+ }
+
+ if (!ixd_vc_dev_init(adapter)) {
+ adapter->init_task.vc_retries = 0;
+ return;
+ }
+
+ ixd_deinit_dflt_mbx(adapter);
+ if (++adapter->init_task.vc_retries > 5) {
+ dev_err(ixd_to_dev(adapter),
+ "Failed to establish mailbox communications with the hardware\n");
+ return;
+ }
+
+ ixd_trigger_reset(adapter);
+ queue_delayed_work(system_unbound_wq, &adapter->init_task.init_work,
+ msecs_to_jiffies(500));
}
diff --git a/drivers/net/ethernet/intel/ixd/ixd_main.c b/drivers/net/ethernet/intel/ixd/ixd_main.c
index b4d4000b63ed..6d5e6aca77df 100644
--- a/drivers/net/ethernet/intel/ixd/ixd_main.c
+++ b/drivers/net/ethernet/intel/ixd/ixd_main.c
@@ -2,6 +2,7 @@
/* Copyright (C) 2025 Intel Corporation */
#include "ixd.h"
+#include "ixd_ctlq.h"
#include "ixd_lan_regs.h"
MODULE_DESCRIPTION("Intel(R) Control Plane Function Device Driver");
@@ -19,6 +20,7 @@ static void ixd_remove(struct pci_dev *pdev)
/* Do not mix removal with (re)initialization */
cancel_delayed_work_sync(&adapter->init_task.init_work);
+
/* Leave the device clean on exit */
ixd_trigger_reset(adapter);
ixd_deinit_dflt_mbx(adapter);
@@ -110,6 +112,7 @@ static int ixd_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
INIT_DELAYED_WORK(&adapter->init_task.init_work,
ixd_init_task);
+ INIT_DELAYED_WORK(&adapter->mbx_task, ixd_ctlq_rx_task);
ixd_trigger_reset(adapter);
queue_delayed_work(system_unbound_wq, &adapter->init_task.init_work,
diff --git a/drivers/net/ethernet/intel/ixd/ixd_virtchnl.c b/drivers/net/ethernet/intel/ixd/ixd_virtchnl.c
new file mode 100644
index 000000000000..66049d1b1d15
--- /dev/null
+++ b/drivers/net/ethernet/intel/ixd/ixd_virtchnl.c
@@ -0,0 +1,178 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2025 Intel Corporation */
+
+#include "ixd.h"
+#include "ixd_ctlq.h"
+#include "ixd_virtchnl.h"
+
+/**
+ * ixd_vc_recv_event_msg - Handle virtchnl event message
+ * @adapter: The adapter handling the message
+ * @ctlq_msg: Message received
+ */
+void ixd_vc_recv_event_msg(struct ixd_adapter *adapter,
+ struct libie_ctlq_msg *ctlq_msg)
+{
+ int payload_size = ctlq_msg->data_len;
+ struct virtchnl2_event *v2e;
+
+ if (payload_size < sizeof(*v2e)) {
+ dev_warn_ratelimited(ixd_to_dev(adapter),
+ "Failed to receive valid payload for event msg (op 0x%X len %u)\n",
+ ctlq_msg->chnl_opcode,
+ payload_size);
+ return;
+ }
+
+ v2e = (struct virtchnl2_event *)ctlq_msg->recv_mem.iov_base;
+
+ dev_dbg(ixd_to_dev(adapter), "Got event 0x%X from the CP\n",
+ le32_to_cpu(v2e->event));
+}
+
+/**
+ * ixd_vc_can_handle_msg - Decide if an event has to be handled by virtchnl code
+ * @ctlq_msg: Message received
+ *
+ * Return: %true if virtchnl code can handle the event, %false otherwise
+ */
+bool ixd_vc_can_handle_msg(struct libie_ctlq_msg *ctlq_msg)
+{
+ return ctlq_msg->chnl_opcode == VIRTCHNL2_OP_EVENT;
+}
+
+/**
+ * ixd_handle_caps - Handle VIRTCHNL2_OP_GET_CAPS response
+ * @adapter: The adapter for which the capabilities are being updated
+ * @recv_buff: Buffer containing the response
+ * @recv_size: Response buffer size
+ * @ctx: unused
+ *
+ * Return: %0 if the response format is correct and was handled as expected,
+ * negative error otherwise.
+ */
+static int ixd_handle_caps(struct ixd_adapter *adapter, void *recv_buff,
+ size_t recv_size, void *ctx)
+{
+ if (recv_size < sizeof(adapter->caps))
+ return -EBADMSG;
+
+ adapter->caps = *(typeof(adapter->caps) *)recv_buff;
+
+ return 0;
+}
+
+/**
+ * ixd_req_vc_caps - Request and save device capability
+ * @adapter: The adapter to get the capabilities for
+ *
+ * Return: success or error if sending the get capability message fails
+ */
+static int ixd_req_vc_caps(struct ixd_adapter *adapter)
+{
+ const struct ixd_ctlq_req req = {
+ .opcode = VIRTCHNL2_OP_GET_CAPS,
+ .send_size = sizeof(struct virtchnl2_get_capabilities),
+ .ctx = NULL,
+ .send_buff_init = NULL,
+ .recv_process = ixd_handle_caps,
+ };
+
+ return ixd_ctlq_do_req(adapter, &req);
+}
+
+/**
+ * ixd_get_vc_ver - Get version info from adapter
+ *
+ * Return: filled in virtchannel2 version info, ready for sending
+ */
+static struct virtchnl2_version_info ixd_get_vc_ver(void)
+{
+ return (struct virtchnl2_version_info) {
+ .major = cpu_to_le32(VIRTCHNL2_VERSION_MAJOR_2),
+ .minor = cpu_to_le32(VIRTCHNL2_VERSION_MINOR_0),
+ };
+}
+
+static void ixd_fill_vc_ver(struct ixd_adapter *adapter, void *send_buff,
+ void *ctx)
+{
+ *(struct virtchnl2_version_info *)send_buff = ixd_get_vc_ver();
+}
+
+/**
+ * ixd_handle_vc_ver - Handle VIRTCHNL2_OP_VERSION response
+ * @adapter: The adapter for which the version is being updated
+ * @recv_buff: Buffer containing the response
+ * @recv_size: Response buffer size
+ * @ctx: Unused
+ *
+ * Return: %0 if the response format is correct and was handled as expected,
+ * negative error otherwise.
+ */
+static int ixd_handle_vc_ver(struct ixd_adapter *adapter, void *recv_buff,
+ size_t recv_size, void *ctx)
+{
+ struct virtchnl2_version_info need_ver = ixd_get_vc_ver();
+ struct virtchnl2_version_info *recv_ver;
+
+ if (recv_size < sizeof(need_ver))
+ return -EBADMSG;
+
+ recv_ver = recv_buff;
+ if (le32_to_cpu(need_ver.major) > le32_to_cpu(recv_ver->major))
+ return -EOPNOTSUPP;
+
+ adapter->vc_ver.major = le32_to_cpu(recv_ver->major);
+ adapter->vc_ver.minor = le32_to_cpu(recv_ver->minor);
+
+ return 0;
+}
+
+/**
+ * ixd_req_vc_version - Request and save Virtchannel2 version
+ * @adapter: The adapter to get the version for
+ *
+ * Return: success or error if sending fails or the response was not as expected
+ */
+static int ixd_req_vc_version(struct ixd_adapter *adapter)
+{
+ const struct ixd_ctlq_req req = {
+ .opcode = VIRTCHNL2_OP_VERSION,
+ .send_size = sizeof(struct virtchnl2_version_info),
+ .ctx = NULL,
+ .send_buff_init = ixd_fill_vc_ver,
+ .recv_process = ixd_handle_vc_ver,
+ };
+
+ return ixd_ctlq_do_req(adapter, &req);
+}
+
+/**
+ * ixd_vc_dev_init - virtchnl device core initialization
+ * @adapter: device information
+ *
+ * Return: %0 on success or error if any step of the initialization fails
+ */
+int ixd_vc_dev_init(struct ixd_adapter *adapter)
+{
+ int err;
+
+ err = ixd_req_vc_version(adapter);
+ if (err) {
+ dev_warn(ixd_to_dev(adapter),
+ "Getting virtchnl version failed, error=%pe\n",
+ ERR_PTR(err));
+ return err;
+ }
+
+ err = ixd_req_vc_caps(adapter);
+ if (err) {
+ dev_warn(ixd_to_dev(adapter),
+ "Getting virtchnl capabilities failed, error=%pe\n",
+ ERR_PTR(err));
+ return err;
+ }
+
+ return err;
+}
diff --git a/drivers/net/ethernet/intel/ixd/ixd_virtchnl.h b/drivers/net/ethernet/intel/ixd/ixd_virtchnl.h
new file mode 100644
index 000000000000..1a53da8b545c
--- /dev/null
+++ b/drivers/net/ethernet/intel/ixd/ixd_virtchnl.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (C) 2025 Intel Corporation */
+
+#ifndef _IXD_VIRTCHNL_H_
+#define _IXD_VIRTCHNL_H_
+
+int ixd_vc_dev_init(struct ixd_adapter *adapter);
+bool ixd_vc_can_handle_msg(struct libie_ctlq_msg *ctlq_msg);
+void ixd_vc_recv_event_msg(struct ixd_adapter *adapter,
+ struct libie_ctlq_msg *ctlq_msg);
+
+#endif /* _IXD_VIRTCHNL_H_ */
--
2.47.0
^ permalink raw reply related
* [PATCH iwl-next v8 15/15] ixd: add devlink support
From: Larysa Zaremba @ 2026-06-08 14:41 UTC (permalink / raw)
To: intel-wired-lan, Tony Nguyen
Cc: aleksander.lobakin, sridhar.samudrala, Michal Swiatkowski,
Larysa Zaremba, Fijalkowski, Maciej, Emil Tantilov, Madhu Chittim,
Josh Hay, Keller, Jacob E, jayaprakash.shanmugam, Jiri Pirko,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Simon Horman, Jonathan Corbet, Richard Cochran, Przemek Kitszel,
Andrew Lunn, netdev, linux-doc, linux-kernel, Bharath R
In-Reply-To: <20260608144127.2751230-1-larysa.zaremba@intel.com>
From: Amritha Nambiar <amritha.nambiar@intel.com>
Enable initial support for the devlink interface with the ixd driver. The
ixd hardware is a single function PCIe device. So, the PCIe adapter gets
its own devlink instance to manage device-wide resources or configuration.
$ devlink dev show
pci/0000:83:00.6
$ devlink dev info pci/0000:83:00.6
pci/0000:83:00.6:
driver ixd
serial_number 00-a0-c9-ff-ff-23-45-67
versions:
fixed:
device.type MEV
running:
virtchnl 2.0
Signed-off-by: Amritha Nambiar <amritha.nambiar@intel.com>
Reviewed-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
Reviewed-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
Reviewed-by: Przemek Kitszel <przemyslaw.kitszel@intel.com>
Tested-by: Bharath R <Bharath.r@intel.com>
Signed-off-by: Larysa Zaremba <larysa.zaremba@intel.com>
---
Documentation/networking/devlink/index.rst | 1 +
Documentation/networking/devlink/ixd.rst | 30 ++++++
drivers/net/ethernet/intel/ixd/Kconfig | 1 +
drivers/net/ethernet/intel/ixd/Makefile | 1 +
drivers/net/ethernet/intel/ixd/ixd.h | 1 +
drivers/net/ethernet/intel/ixd/ixd_devlink.c | 97 ++++++++++++++++++++
drivers/net/ethernet/intel/ixd/ixd_devlink.h | 50 ++++++++++
drivers/net/ethernet/intel/ixd/ixd_lib.c | 3 +
drivers/net/ethernet/intel/ixd/ixd_main.c | 14 ++-
9 files changed, 195 insertions(+), 3 deletions(-)
create mode 100644 Documentation/networking/devlink/ixd.rst
create mode 100644 drivers/net/ethernet/intel/ixd/ixd_devlink.c
create mode 100644 drivers/net/ethernet/intel/ixd/ixd_devlink.h
diff --git a/Documentation/networking/devlink/index.rst b/Documentation/networking/devlink/index.rst
index f7ba7dcf477d..f0c077843fa7 100644
--- a/Documentation/networking/devlink/index.rst
+++ b/Documentation/networking/devlink/index.rst
@@ -88,6 +88,7 @@ parameters, info versions, and other features it supports.
ionic
iosm
ixgbe
+ ixd
kvaser_pciefd
kvaser_usb
mlx4
diff --git a/Documentation/networking/devlink/ixd.rst b/Documentation/networking/devlink/ixd.rst
new file mode 100644
index 000000000000..17b63c8425aa
--- /dev/null
+++ b/Documentation/networking/devlink/ixd.rst
@@ -0,0 +1,30 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+===================
+ixd devlink support
+===================
+
+This document describes the devlink features implemented by the ``ixd``
+device driver.
+
+Info versions
+=============
+
+The ``ixd`` driver reports the following versions
+
+.. list-table:: devlink info versions implemented
+ :widths: 5 5 5 90
+
+ * - Name
+ - Type
+ - Example
+ - Description
+ * - ``device.type``
+ - fixed
+ - MEV
+ - The hardware type for this device
+ * - ``virtchnl``
+ - running
+ - 2.0
+ - 2-digit version number (major.minor) of the communication channel
+ (virtchnl) used by the device.
diff --git a/drivers/net/ethernet/intel/ixd/Kconfig b/drivers/net/ethernet/intel/ixd/Kconfig
index 24510c50070e..34181c59dcdc 100644
--- a/drivers/net/ethernet/intel/ixd/Kconfig
+++ b/drivers/net/ethernet/intel/ixd/Kconfig
@@ -7,6 +7,7 @@ config IXD
select LIBETH
select LIBIE_CP
select LIBIE_PCI
+ select NET_DEVLINK
help
This driver supports Intel(R) Control Plane PCI Function
of Intel E2100 and later IPUs and FNICs.
diff --git a/drivers/net/ethernet/intel/ixd/Makefile b/drivers/net/ethernet/intel/ixd/Makefile
index 90abf231fb16..03760a2580b9 100644
--- a/drivers/net/ethernet/intel/ixd/Makefile
+++ b/drivers/net/ethernet/intel/ixd/Makefile
@@ -8,5 +8,6 @@ obj-$(CONFIG_IXD) += ixd.o
ixd-y := ixd_main.o
ixd-y += ixd_ctlq.o
ixd-y += ixd_dev.o
+ixd-y += ixd_devlink.o
ixd-y += ixd_lib.o
ixd-y += ixd_virtchnl.o
diff --git a/drivers/net/ethernet/intel/ixd/ixd.h b/drivers/net/ethernet/intel/ixd/ixd.h
index 8e3cd5bc5a84..343c5053045f 100644
--- a/drivers/net/ethernet/intel/ixd/ixd.h
+++ b/drivers/net/ethernet/intel/ixd/ixd.h
@@ -23,6 +23,7 @@ struct ixd_adapter {
struct delayed_work init_work;
u8 reset_retries;
u8 vc_retries;
+ bool success;
} init_task;
struct delayed_work mbx_task;
struct libie_ctlq_xn_manager *xnm;
diff --git a/drivers/net/ethernet/intel/ixd/ixd_devlink.c b/drivers/net/ethernet/intel/ixd/ixd_devlink.c
new file mode 100644
index 000000000000..23ab11226978
--- /dev/null
+++ b/drivers/net/ethernet/intel/ixd/ixd_devlink.c
@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2025, Intel Corporation. */
+
+#include "ixd.h"
+#include "ixd_devlink.h"
+
+#define IXD_DEVLINK_INFO_LEN 128
+
+/**
+ * ixd_fill_dsn - Get the serial number for the ixd device
+ * @adapter: adapter to query
+ * @buf: storage buffer for the info request
+ */
+static void ixd_fill_dsn(struct ixd_adapter *adapter, char *buf)
+{
+ u8 dsn[8];
+
+ /* Copy the DSN into an array in Big Endian format */
+ put_unaligned_be64(pci_get_dsn(adapter->cp_ctx.mmio_info.pdev), dsn);
+
+ snprintf(buf, IXD_DEVLINK_INFO_LEN, "%8phD", dsn);
+}
+
+/**
+ * ixd_fill_device_name - Get the name of the underlying hardware
+ * @adapter: adapter to query
+ * @buf: storage buffer for the info request
+ * @buf_size: size of the storage buffer
+ */
+static void ixd_fill_device_name(struct ixd_adapter *adapter, char *buf,
+ size_t buf_size)
+{
+ if (adapter->caps.device_type == cpu_to_le32(VIRTCHNL2_MEV_DEVICE))
+ snprintf(buf, buf_size, "%s", "MEV");
+ else
+ snprintf(buf, buf_size, "%s", "UNKNOWN");
+}
+
+/**
+ * ixd_devlink_info_get - .info_get devlink handler
+ * @devlink: devlink instance structure
+ * @req: the devlink info request
+ * @extack: extended netdev ack structure
+ *
+ * Callback for the devlink .info_get operation. Reports information about the
+ * device.
+ *
+ * Return: zero on success or an error code on failure.
+ */
+static int ixd_devlink_info_get(struct devlink *devlink,
+ struct devlink_info_req *req,
+ struct netlink_ext_ack *extack)
+{
+ struct ixd_adapter *adapter = devlink_priv(devlink);
+ char buf[IXD_DEVLINK_INFO_LEN];
+ int err;
+
+ ixd_fill_dsn(adapter, buf);
+ err = devlink_info_serial_number_put(req, buf);
+ if (err)
+ return err;
+
+ ixd_fill_device_name(adapter, buf, IXD_DEVLINK_INFO_LEN);
+ err = devlink_info_version_fixed_put(req, "device.type", buf);
+ if (err)
+ return err;
+
+ snprintf(buf, sizeof(buf), "%u.%u",
+ adapter->vc_ver.major, adapter->vc_ver.minor);
+
+ return devlink_info_version_running_put(req, "virtchnl", buf);
+}
+
+static const struct devlink_ops ixd_devlink_ops = {
+ .info_get = ixd_devlink_info_get,
+};
+
+/**
+ * ixd_adapter_alloc - Allocate devlink and return adapter pointer
+ * @dev: the device to allocate for
+ *
+ * Allocate a devlink instance for this device and return the private area as
+ * the adapter structure.
+ *
+ * Return: adapter structure on success, NULL on failure
+ */
+struct ixd_adapter *ixd_adapter_alloc(struct device *dev)
+{
+ struct devlink *devlink;
+
+ devlink = devlink_alloc(&ixd_devlink_ops, sizeof(struct ixd_adapter),
+ dev);
+ if (!devlink)
+ return NULL;
+
+ return devlink_priv(devlink);
+}
diff --git a/drivers/net/ethernet/intel/ixd/ixd_devlink.h b/drivers/net/ethernet/intel/ixd/ixd_devlink.h
new file mode 100644
index 000000000000..b23a1b37aebc
--- /dev/null
+++ b/drivers/net/ethernet/intel/ixd/ixd_devlink.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2025, Intel Corporation. */
+
+#ifndef _IXD_DEVLINK_H_
+#define _IXD_DEVLINK_H_
+
+#include <net/devlink.h>
+
+#include "ixd.h"
+
+struct ixd_adapter *ixd_adapter_alloc(struct device *dev);
+
+/**
+ * ixd_devlink_free - teardown the devlink
+ * @adapter: the adapter structure to free
+ *
+ */
+static inline void ixd_devlink_free(struct ixd_adapter *adapter)
+{
+ struct devlink *devlink = priv_to_devlink(adapter);
+
+ devlink_free(devlink);
+}
+
+/**
+ * ixd_devlink_unregister - Unregister devlink for this adapter.
+ * @adapter: the adapter structure to cleanup
+ *
+ * Init task must be completed or cancelled beforehand.
+ */
+static inline void ixd_devlink_unregister(struct ixd_adapter *adapter)
+{
+ if (!adapter->init_task.success)
+ return;
+
+ devlink_unregister(priv_to_devlink(adapter));
+}
+
+/**
+ * ixd_devlink_register - Register devlink interface for this adapter
+ * @adapter: pointer to ixd adapter structure to be associated with devlink
+ *
+ * Register the devlink instance associated with this adapter
+ */
+static inline void ixd_devlink_register(struct ixd_adapter *adapter)
+{
+ devlink_register(priv_to_devlink(adapter));
+}
+
+#endif /* _IXD_DEVLINK_H_ */
diff --git a/drivers/net/ethernet/intel/ixd/ixd_lib.c b/drivers/net/ethernet/intel/ixd/ixd_lib.c
index 17f4aae594ca..23927b0471d7 100644
--- a/drivers/net/ethernet/intel/ixd/ixd_lib.c
+++ b/drivers/net/ethernet/intel/ixd/ixd_lib.c
@@ -3,6 +3,7 @@
#include "ixd.h"
#include "ixd_ctlq.h"
+#include "ixd_devlink.h"
#include "ixd_virtchnl.h"
#define IXD_DFLT_MBX_Q_LEN 64
@@ -154,6 +155,8 @@ void ixd_init_task(struct work_struct *work)
if (!ixd_vc_dev_init(adapter)) {
adapter->init_task.vc_retries = 0;
+ adapter->init_task.success = true;
+ ixd_devlink_register(adapter);
return;
}
diff --git a/drivers/net/ethernet/intel/ixd/ixd_main.c b/drivers/net/ethernet/intel/ixd/ixd_main.c
index 6d5e6aca77df..50d9a13c851f 100644
--- a/drivers/net/ethernet/intel/ixd/ixd_main.c
+++ b/drivers/net/ethernet/intel/ixd/ixd_main.c
@@ -4,6 +4,7 @@
#include "ixd.h"
#include "ixd_ctlq.h"
#include "ixd_lan_regs.h"
+#include "ixd_devlink.h"
MODULE_DESCRIPTION("Intel(R) Control Plane Function Device Driver");
MODULE_IMPORT_NS("LIBIE_CP");
@@ -21,11 +22,14 @@ static void ixd_remove(struct pci_dev *pdev)
/* Do not mix removal with (re)initialization */
cancel_delayed_work_sync(&adapter->init_task.init_work);
+ ixd_devlink_unregister(adapter);
+
/* Leave the device clean on exit */
ixd_trigger_reset(adapter);
ixd_deinit_dflt_mbx(adapter);
libie_pci_unmap_all_mmio_regions(&adapter->cp_ctx.mmio_info);
+ ixd_devlink_free(adapter);
}
/**
@@ -93,7 +97,7 @@ static int ixd_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
if (WARN_ON(ent->device != IXD_DEV_ID_CPF))
return -EINVAL;
- adapter = devm_kzalloc(&pdev->dev, sizeof(*adapter), GFP_KERNEL);
+ adapter = ixd_adapter_alloc(&pdev->dev);
if (!adapter)
return -ENOMEM;
@@ -102,13 +106,13 @@ static int ixd_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
err = libie_pci_init_dev(pdev);
if (err)
- return err;
+ goto free_adapter;
pci_set_drvdata(pdev, adapter);
err = ixd_iomap_regions(adapter);
if (err)
- return err;
+ goto free_adapter;
INIT_DELAYED_WORK(&adapter->init_task.init_work,
ixd_init_task);
@@ -119,6 +123,10 @@ static int ixd_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
msecs_to_jiffies(500));
return 0;
+
+free_adapter:
+ ixd_devlink_free(adapter);
+ return err;
}
static const struct pci_device_id ixd_pci_tbl[] = {
--
2.47.0
^ permalink raw reply related
* Re: [PATCH 4/4] block: add configurable error injection
From: Jens Axboe @ 2026-06-08 14:53 UTC (permalink / raw)
To: Christoph Hellwig
Cc: Jonathan Corbet, Damien Le Moal, Hannes Reinecke, Keith Busch,
linux-block, linux-doc, Hannes Reinecke
In-Reply-To: <20260608051416.1205282-5-hch@lst.de>
On 6/7/26 11:14 PM, Christoph Hellwig wrote:
> diff --git a/block/blk.h b/block/blk.h
> index e8b7d5517086..10df23b2cb90 100644
> --- a/block/blk.h
> +++ b/block/blk.h
> @@ -660,6 +660,18 @@ static inline bool should_fail_request(struct block_device *part,
> }
> #endif /* CONFIG_FAIL_MAKE_REQUEST */
>
> +void blk_error_injection_init(struct gendisk *disk);
> +void blk_error_injection_exit(struct gendisk *disk);
> +bool __blk_error_inject(struct bio *bio);
> +static inline bool blk_error_inject(struct bio *bio)
> +{
> + if (!IS_ENABLED(CONFIG_BLK_ERROR_INJECTION))
> + return false;
> + if (!test_bit(GD_ERROR_INJECT, &bio->bi_bdev->bd_disk->state))
> + return false;
> + return __blk_error_inject(bio);
> +}
I really hate this part, that's a pretty deep set of pointer chasings to
figure out if injection is enabled or not, when in practice error
injection is only ever enabled for specific test cases and distros
invariably will set CONFIG_BLK_ERROR_INJECTION because they turn on
every damn thing under the sun.
IOW, that won't fly for the hot path. Maybe a static key would be useful
here?
--
Jens Axboe
^ permalink raw reply
* [PATCH v5 07/34] KVM: x86: Activate master clock immediately on vCPU creation
From: David Woodhouse @ 2026-06-08 14:47 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
Previously, the master clock was only activated when the first vCPU
processed KVM_REQ_MASTERCLOCK_UPDATE during KVM_RUN. This meant that
KVM_GET_CLOCK could not return the host_tsc field until after the
first KVM_RUN, making it impossible for userspace to follow the
documented TSC migration procedure without a dummy vCPU run.
Fix this by calling kvm_update_masterclock() directly from
kvm_arch_vcpu_postcreate(), after kvm_synchronize_tsc() has already
set all_vcpus_matched_freq. This ensures the master clock is active
immediately, and KVM_GET_CLOCK returns a valid {host_tsc, realtime}
pair as soon as a vCPU exists.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
arch/x86/kvm/x86.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index b7e5f6e3dc6c..c1897d939da9 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -13098,6 +13098,8 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
return;
vcpu_load(vcpu);
kvm_synchronize_tsc(vcpu, NULL);
+ if (!vcpu->kvm->arch.use_master_clock)
+ kvm_update_masterclock(vcpu->kvm);
vcpu_put(vcpu);
/* poll control enabled by default */
--
2.54.0
^ permalink raw reply related
* [PATCH v5 11/34] KVM: x86: Restructure get_kvmclock()
From: David Woodhouse @ 2026-06-08 14:47 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
Move get/put_cpu inside the use_master_clock branch since they are only
needed there (for RDTSC and get_cpu_tsc_khz() to be on the same CPU).
Simplify the use_master_clock condition: the open-coded CONSTANT_TSC ||
cpu_tsc_khz check is unnecessary since use_master_clock can only be true
when the TSC is usable.
Wrap the entire use_master_clock block in #ifdef CONFIG_X86_64, since
use_master_clock is never true on 32-bit (host_tsc_clocksource is only
set under CONFIG_X86_64).
When the clock read fails (e.g. clocksource transitioning away from
TSC), fall back to the non-master-clock path (get_kvmclock_base_ns)
rather than proceeding with uninitialised data or spinning in the
seqcount loop.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
arch/x86/kvm/x86.c | 34 +++++++++++++++++++++++-----------
1 file changed, 23 insertions(+), 11 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index fce898811fe7..6983a7494fcd 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -3209,21 +3209,30 @@ static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
do {
seq = read_seqcount_begin(&ka->pvclock_sc);
- /* both __this_cpu_read() and rdtsc() should be on the same cpu */
- get_cpu();
-
data->flags = 0;
- if (ka->use_master_clock &&
- (static_cpu_has(X86_FEATURE_CONSTANT_TSC) || __this_cpu_read(cpu_tsc_khz))) {
#ifdef CONFIG_X86_64
+ if (ka->use_master_clock) {
struct timespec64 ts;
+ /*
+ * The RDTSC and get_cpu_tsc_khz() must happen on
+ * the same CPU.
+ */
+ get_cpu();
+
if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) {
data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec;
data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC;
- } else
-#endif
- data->host_tsc = rdtsc();
+ } else {
+ /*
+ * Clock read failed (e.g. clocksource is
+ * transitioning away from TSC). Fall back to
+ * the non-master-clock path rather than
+ * spinning.
+ */
+ put_cpu();
+ goto fallback;
+ }
data->flags |= KVM_CLOCK_TSC_STABLE;
hv_clock.tsc_timestamp = ka->master_cycle_now;
@@ -3232,11 +3241,14 @@ static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
&hv_clock.tsc_shift,
&hv_clock.tsc_to_system_mul);
data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc);
- } else {
+
+ put_cpu();
+ } else
+#endif
+ {
+fallback:
data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset;
}
-
- put_cpu();
} while (read_seqcount_retry(&ka->pvclock_sc, seq));
}
--
2.54.0
^ permalink raw reply related
* [PATCH v5 13/34] KVM: x86: Use get_kvmclock() in kvm_get_wall_clock_epoch()
From: David Woodhouse @ 2026-06-08 14:47 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
Now that get_kvmclock() correctly handles TSC scaling and captures both
wallclock and kvmclock from the same TSC reading,
kvm_get_wall_clock_epoch() can simply call it instead of duplicating
the pvclock computation.
This eliminates the last instance of the "definition C" kvmclock
calculation that computed nanoseconds directly from the host TSC
without accounting for guest TSC scaling.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
arch/x86/kvm/x86.c | 59 +++++++---------------------------------------
1 file changed, 9 insertions(+), 50 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 7ae6a7705353..fc9366b83912 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -3494,63 +3494,22 @@ int kvm_guest_time_update(struct kvm_vcpu *v)
* wallclock and kvmclock times, and subtracting one from the other.
*
* Fall back to using their values at slightly different moments by
- * calling ktime_get_real_ns() and get_kvmclock_ns() separately.
+ * calling ktime_get_real_ns() and get_kvmclock() separately.
*/
uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm)
{
-#ifdef CONFIG_X86_64
- struct pvclock_vcpu_time_info hv_clock;
- struct kvm_arch *ka = &kvm->arch;
- unsigned long seq, local_tsc_khz;
- struct timespec64 ts;
- uint64_t host_tsc;
-
- do {
- seq = read_seqcount_begin(&ka->pvclock_sc);
-
- local_tsc_khz = 0;
- if (!ka->use_master_clock)
- break;
-
- /*
- * The TSC read and the call to get_cpu_tsc_khz() must happen
- * on the same CPU.
- */
- get_cpu();
-
- local_tsc_khz = get_cpu_tsc_khz();
-
- if (local_tsc_khz &&
- !kvm_get_walltime_and_clockread(&ts, &host_tsc))
- local_tsc_khz = 0; /* Fall back to old method */
-
- put_cpu();
-
- /*
- * These values must be snapshotted within the seqcount loop.
- * After that, it's just mathematics which can happen on any
- * CPU at any time.
- */
- hv_clock.tsc_timestamp = ka->master_cycle_now;
- hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
+ struct kvm_clock_data data;
- } while (read_seqcount_retry(&ka->pvclock_sc, seq));
+ get_kvmclock(kvm, &data);
/*
- * If the conditions were right, and obtaining the wallclock+TSC was
- * successful, calculate the KVM clock at the corresponding time and
- * subtract one from the other to get the guest's epoch in nanoseconds
- * since 1970-01-01.
+ * If get_kvmclock() captured both wallclock and kvmclock from the
+ * same TSC reading, use them for a precise epoch calculation.
*/
- if (local_tsc_khz) {
- kvm_get_time_scale(NSEC_PER_SEC, local_tsc_khz * NSEC_PER_USEC,
- &hv_clock.tsc_shift,
- &hv_clock.tsc_to_system_mul);
- return ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec -
- __pvclock_read_cycles(&hv_clock, host_tsc);
- }
-#endif
- return ktime_get_real_ns() - get_kvmclock_ns(kvm);
+ if (data.flags & KVM_CLOCK_REALTIME)
+ return data.realtime - data.clock;
+
+ return ktime_get_real_ns() - data.clock;
}
/*
--
2.54.0
^ permalink raw reply related
* [PATCH v5 12/34] KVM: x86: Fix KVM clock precision in get_kvmclock() with TSC scaling
From: David Woodhouse @ 2026-06-08 14:47 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
When in master clock mode, the KVM clock is defined in terms of the
guest TSC. But get_kvmclock() was computing it from the host TSC
without applying TSC scaling, leading to a systemic drift from the
values the guest computes from its own TSC.
Store the VM's TSC scaling ratio in kvm_arch and precompute the
guest-TSC-based mul/shift in pvclock_update_vm_gtod_copy(). Use these
in get_kvmclock() to scale the host TSC delta to guest TSC before
converting to nanoseconds.
This avoids "definition C" of the KVM clock described in the
earlier commit "KVM: x86/xen: Do not corrupt KVM clock in
kvm_xen_shared_info_init()".
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
arch/x86/include/asm/kvm_host.h | 4 +++
arch/x86/kvm/x86.c | 52 +++++++++++++++++++++++++++++----
2 files changed, 51 insertions(+), 5 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 37264212c7df..5348fd5ea3f3 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1490,6 +1490,7 @@ struct kvm_arch {
u64 last_tsc_write;
u32 last_tsc_khz;
u64 last_tsc_offset;
+ u64 last_tsc_scaling_ratio;
u64 cur_tsc_nsec;
u64 cur_tsc_write;
u64 cur_tsc_offset;
@@ -1504,6 +1505,9 @@ struct kvm_arch {
bool use_master_clock;
u64 master_kernel_ns;
u64 master_cycle_now;
+ u64 master_tsc_scaling_ratio;
+ s8 master_tsc_shift;
+ u32 master_tsc_mul;
#ifdef CONFIG_KVM_HYPERV
struct kvm_hv hyperv;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 6983a7494fcd..7ae6a7705353 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -2781,6 +2781,7 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
kvm->arch.last_tsc_write = tsc;
kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
kvm->arch.last_tsc_offset = offset;
+ kvm->arch.last_tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio;
vcpu->arch.last_guest_tsc = tsc;
@@ -3109,6 +3110,8 @@ static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
*
*/
+static unsigned long get_cpu_tsc_khz(void);
+
static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
@@ -3132,9 +3135,30 @@ static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
&& !ka->backwards_tsc_observed
&& !ka->boot_vcpu_runs_old_kvmclock;
- if (ka->use_master_clock)
+ if (ka->use_master_clock) {
+ u64 tsc_hz;
+
atomic_set(&kvm_guest_has_master_clock, 1);
+ /*
+ * Copy the scaling ratio and precompute the mul/shift for
+ * converting guest TSC to nanoseconds. These are used by
+ * get_kvmclock() to compute kvmclock from the host TSC
+ * without needing a vCPU reference.
+ */
+ ka->master_tsc_scaling_ratio = ka->last_tsc_scaling_ratio;
+ tsc_hz = (u64)get_cpu_tsc_khz() * 1000;
+ if (tsc_hz && kvm_caps.has_tsc_control)
+ tsc_hz = kvm_scale_tsc(tsc_hz,
+ ka->master_tsc_scaling_ratio);
+ if (tsc_hz)
+ kvm_get_time_scale(NSEC_PER_SEC, tsc_hz,
+ &ka->master_tsc_shift,
+ &ka->master_tsc_mul);
+ else
+ ka->use_master_clock = false;
+ }
+
vclock_mode = pvclock_gtod_data.clock.vclock_mode;
trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
vcpus_matched);
@@ -3237,10 +3261,28 @@ static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
data->flags |= KVM_CLOCK_TSC_STABLE;
hv_clock.tsc_timestamp = ka->master_cycle_now;
hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
- kvm_get_time_scale(NSEC_PER_SEC, get_cpu_tsc_khz() * 1000LL,
- &hv_clock.tsc_shift,
- &hv_clock.tsc_to_system_mul);
- data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc);
+
+ /*
+ * Use the precomputed guest-TSC-based mul/shift
+ * so that the kvmclock value matches what the
+ * guest computes from its own TSC.
+ */
+ hv_clock.tsc_shift = ka->master_tsc_shift;
+ hv_clock.tsc_to_system_mul = ka->master_tsc_mul;
+
+ if (kvm_caps.has_tsc_control) {
+ u64 tsc_delta = data->host_tsc - ka->master_cycle_now;
+
+ tsc_delta = kvm_scale_tsc(tsc_delta,
+ ka->master_tsc_scaling_ratio);
+ data->clock = hv_clock.system_time +
+ pvclock_scale_delta(tsc_delta,
+ hv_clock.tsc_to_system_mul,
+ hv_clock.tsc_shift);
+ } else {
+ data->clock = __pvclock_read_cycles(&hv_clock,
+ data->host_tsc);
+ }
put_cpu();
} else
--
2.54.0
^ permalink raw reply related
* [PATCH v5 25/34] KVM: x86/xen: Prevent runstate times from becoming negative
From: David Woodhouse @ 2026-06-08 14:48 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
When kvm_xen_update_runstate() is invoked to set a vCPU's runstate, the
time spent in the previous runstate is accounted. This is based on the
delta between the current KVM clock time, and the previous value stored
in vcpu->arch.xen.runstate_entry_time.
If the KVM clock goes backwards, that delta will be negative. Or, since
it's an unsigned 64-bit integer, very *large*. Linux guests deal with
that particularly badly, reporting 100% steal time for ever more (well,
for *centuries* at least, until the delta has been consumed).
So when a negative delta is detected, just refrain from updating the
runstate times until the KVM clock catches up with runstate_entry_time
again.
Also clamp steal_ns to delta_ns to prevent steal time from exceeding
the total elapsed time, and handle negative steal_ns (which can happen
if run_delay goes backwards across a scheduler update).
The userspace APIs for setting the runstate times do not allow them to
be set past the current KVM clock, but userspace can still adjust the
KVM clock *after* setting the runstate times, which would cause this
situation to occur.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Reviewed-by: Paul Durrant <paul@xen.org>
---
arch/x86/kvm/xen.c | 28 ++++++++++++++++++++++------
1 file changed, 22 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c
index 82e34edbfdbd..b1d67ece5db3 100644
--- a/arch/x86/kvm/xen.c
+++ b/arch/x86/kvm/xen.c
@@ -586,29 +586,45 @@ void kvm_xen_update_runstate(struct kvm_vcpu *v, int state)
{
struct kvm_vcpu_xen *vx = &v->arch.xen;
u64 now = get_kvmclock_ns(v->kvm);
- u64 delta_ns = now - vx->runstate_entry_time;
u64 run_delay = current->sched_info.run_delay;
+ s64 delta_ns = now - vx->runstate_entry_time;
+ s64 steal_ns = run_delay - vx->last_steal;
+ /*
+ * If the vCPU was never run before, its prior state should
+ * be considered RUNSTATE_offline.
+ */
if (unlikely(!vx->runstate_entry_time))
vx->current_runstate = RUNSTATE_offline;
+ /*
+ * If KVM clock went backwards, just update the current runstate
+ * but don't account any time. Leave entry_time unchanged so the
+ * next positive delta covers the full period once the clock
+ * catches up. Update last_steal every time so stolen time only
+ * reflects the interval since the most recent call.
+ */
+ if (delta_ns < 0)
+ goto update_guest;
+
/*
* Time waiting for the scheduler isn't "stolen" if the
* vCPU wasn't running anyway.
*/
- if (vx->current_runstate == RUNSTATE_running) {
- u64 steal_ns = run_delay - vx->last_steal;
+ if (vx->current_runstate == RUNSTATE_running && steal_ns > 0) {
+ if (steal_ns > delta_ns)
+ steal_ns = delta_ns;
delta_ns -= steal_ns;
-
vx->runstate_times[RUNSTATE_runnable] += steal_ns;
}
- vx->last_steal = run_delay;
vx->runstate_times[vx->current_runstate] += delta_ns;
- vx->current_runstate = state;
vx->runstate_entry_time = now;
+ update_guest:
+ vx->current_runstate = state;
+ vx->last_steal = run_delay;
if (vx->runstate_cache.active)
kvm_xen_update_runstate_guest(v, state == RUNSTATE_runnable);
}
--
2.54.0
^ permalink raw reply related
* [PATCH v5 21/34] KVM: x86: Allow KVM master clock mode when TSCs are offset from each other
From: David Woodhouse @ 2026-06-08 14:48 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
Previously, a guest writing different TSC values on different vCPUs could
force KVM out of master clock mode. With this change, only a frequency
mismatch disables master clock. The only ways for non-master-clock mode
to happen now are archaic hardware without a TSC-based clocksource, or a
VMM that sets different TSC frequencies across vCPUs.
Running at a different frequency would lead to a systemic skew between
the clock(s) as observed by different vCPUs due to arithmetic precision
in the scaling. So that should indeed force the clock to be based on the
host's CLOCK_MONOTONIC_RAW instead of being in masterclock mode where it
is defined by the guest TSC.
But when the vCPUs merely have a different TSC *offset*, that's not a
problem. The offset is applied to that vCPU's kvmclock->tsc_timestamp
field, and it all comes out in the wash.
Track frequency matching separately from offset matching using a
dedicated freq generation counter (cur_tsc_freq_generation) that only
bumps on actual frequency changes. Each vCPU is counted exactly once per
freq generation via a per-vCPU this_tsc_freq_generation field, preventing
repeated syncs of the same vCPU from falsely re-enabling master clock.
Note that the generation-based counting has a known limitation: if all
vCPUs are in sync and one changes away and then back again, the other
vCPUs are still at the old generation and won't be counted until they
sync again (which may never happen). This was always the case for the
offset tracking and isn't expected VMM behaviour — although it is the
scenario that the VM-wide KVM_SET_TSC_KHZ ioctl was introduced to handle
cleanly.
While at it, restructure the existing TSC offset generation tracking to
use the same pattern: reset counter to zero on new generation, then
unconditionally count vCPUs that haven't been seen in this generation.
Both counters now use a consistent >= online_vcpus threshold (1-based
counting where the reference vCPU is included in the count).
Use frequency match for master clock eligibility, and full TSC match
(including offset) only for PVCLOCK_TSC_STABLE_BIT, which tells the
guest it is safe to skip cross-vCPU monotonicity enforcement.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
arch/x86/include/asm/kvm_host.h | 4 ++
arch/x86/kvm/x86.c | 68 ++++++++++++++++++++++++++-------
2 files changed, 58 insertions(+), 14 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index eb81f90284ba..699a1a197194 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -970,6 +970,7 @@ struct kvm_vcpu_arch {
u64 this_tsc_nsec;
u64 this_tsc_write;
u64 this_tsc_generation;
+ u64 this_tsc_freq_generation;
bool tsc_catchup;
bool tsc_always_catchup;
s8 virtual_tsc_shift;
@@ -1493,6 +1494,9 @@ struct kvm_arch {
u64 cur_tsc_offset;
u64 cur_tsc_generation;
bool all_vcpus_matched_tsc;
+ bool all_vcpus_matched_freq;
+ int nr_vcpus_matched_freq;
+ u64 cur_tsc_freq_generation;
int nr_vcpus_matched_tsc;
u32 default_tsc_khz;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index ac66f8e7116f..86c30be4c5d2 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -2647,14 +2647,37 @@ static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu, bool new_generation)
struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
/*
- * To use the masterclock, the host clocksource must be based on TSC
- * and all vCPUs must have matching TSCs. Note, the count for matching
- * vCPUs doesn't include the reference vCPU, hence "+1".
+ * Track whether all vCPUs have matching TSC offsets (for
+ * PVCLOCK_TSC_STABLE_BIT) and matching frequencies (for
+ * master clock eligibility).
+ */
+
+ /*
+ * A new vCPU might already have incremented ->online_vcpus
+ * and cause a temporary false negative here. But will then
+ * call kvm_synchronize_tsc() from kvm_arch_vcpu_postcreate()
+ * and finish the job.
*/
- ka->all_vcpus_matched_tsc = (ka->nr_vcpus_matched_tsc + 1 ==
- atomic_read(&vcpu->kvm->online_vcpus));
+ int online = atomic_read(&vcpu->kvm->online_vcpus);
- bool use_master_clock = ka->all_vcpus_matched_tsc &&
+ ka->all_vcpus_matched_tsc = (ka->nr_vcpus_matched_tsc >= online);
+ /*
+ * all_vcpus_matched_freq starts true and is cleared when
+ * __kvm_synchronize_tsc() detects a frequency mismatch.
+ * Re-enable when all vCPUs have synced with matching frequency.
+ * If all offsets also match, that implies frequencies match too.
+ */
+ if (ka->all_vcpus_matched_tsc ||
+ ka->nr_vcpus_matched_freq >= online)
+ ka->all_vcpus_matched_freq = true;
+
+ /*
+ * To use the masterclock, the host clocksource must be based on TSC
+ * and all vCPUs must have matching TSC *frequency*. Different offsets
+ * are fine — each vCPU's pvclock has its own tsc_timestamp that
+ * accounts for its offset.
+ */
+ bool use_master_clock = ka->all_vcpus_matched_freq &&
gtod_is_based_on_tsc(gtod->clock.vclock_mode);
/*
@@ -2818,7 +2841,22 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
* Track the TSC frequency, scaling ratio, and offset for the current
* generation. These are used to detect matching TSC writes and to
* compute the guest TSC from the host clock.
+ *
+ * If the frequency changed, master clock mode can no longer be used
+ * since the kvmclock scaling factors differ between vCPUs.
*/
+ if (vcpu->arch.virtual_tsc_khz != kvm->arch.cur_tsc_khz) {
+ kvm->arch.cur_tsc_freq_generation++;
+ kvm->arch.all_vcpus_matched_freq = false;
+ kvm->arch.nr_vcpus_matched_freq = 0;
+ }
+
+ /* Count each vCPU once per freq generation */
+ if (vcpu->arch.this_tsc_freq_generation != kvm->arch.cur_tsc_freq_generation) {
+ vcpu->arch.this_tsc_freq_generation = kvm->arch.cur_tsc_freq_generation;
+ kvm->arch.nr_vcpus_matched_freq++;
+ }
+
kvm->arch.cur_tsc_khz = vcpu->arch.virtual_tsc_khz;
kvm->arch.cur_tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio;
@@ -2835,17 +2873,18 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
* exact software computation in compute_guest_tsc()
*/
kvm->arch.cur_tsc_generation++;
+ kvm->arch.all_vcpus_matched_tsc = false;
+ kvm->arch.nr_vcpus_matched_tsc = 0;
kvm->arch.cur_tsc_nsec = ns;
kvm->arch.cur_tsc_write = tsc;
kvm->arch.cur_tsc_offset = offset;
- kvm->arch.nr_vcpus_matched_tsc = 0;
- kvm->arch.all_vcpus_matched_tsc = false;
- } else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {
+ }
+
+ if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {
+ vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
kvm->arch.nr_vcpus_matched_tsc++;
}
- /* Keep track of which generation this VCPU has synchronized to */
- vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;
@@ -3180,7 +3219,7 @@ static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
bool host_tsc_clocksource, vcpus_matched;
lockdep_assert_held(&kvm->arch.tsc_write_lock);
- vcpus_matched = ka->all_vcpus_matched_tsc;
+ vcpus_matched = ka->all_vcpus_matched_freq;
/*
* If the host uses TSC clock, then passthrough TSC as stable
@@ -3527,7 +3566,7 @@ int kvm_guest_time_update(struct kvm_vcpu *v)
/* If the host uses TSC clocksource, then it is stable */
hv_clock.flags = 0;
- if (use_master_clock)
+ if (use_master_clock && ka->all_vcpus_matched_tsc)
hv_clock.flags |= PVCLOCK_TSC_STABLE_BIT;
if (vcpu->pv_time.active) {
@@ -6354,7 +6393,7 @@ static int kvm_vcpu_ioctl_get_clock_guest(struct kvm_vcpu *v, void __user *argp)
hv_clock.tsc_shift = vcpu->pvclock_tsc_shift;
hv_clock.tsc_to_system_mul = vcpu->pvclock_tsc_mul;
- hv_clock.flags = PVCLOCK_TSC_STABLE_BIT;
+ hv_clock.flags = ka->all_vcpus_matched_tsc ? PVCLOCK_TSC_STABLE_BIT : 0;
if (copy_to_user(argp, &hv_clock, sizeof(hv_clock)))
return -EFAULT;
@@ -13649,6 +13688,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
mutex_init(&kvm->arch.apic_map_lock);
seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock);
kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();
+ kvm->arch.all_vcpus_matched_freq = true;
raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
pvclock_update_vm_gtod_copy(kvm);
--
2.54.0
^ permalink raw reply related
* [PATCH v5 28/34] KVM: selftests: Add Xen/generic CPUID timing leaf test
From: David Woodhouse @ 2026-06-08 14:48 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
Verify that userspace can correctly populate Xen and generic CPUID
timing leaves using the KVM_VCPU_TSC_SCALE and
KVM_VCPU_TSC_SCALE attributes.
This validates that the removal of KVM's runtime Xen CPUID modification
doesn't break guests: userspace queries the effective TSC and bus
frequencies, computes the pvclock mul/shift, populates the CPUID leaves,
and the guest verifies the values match.
The test exercises:
- KVM_VCPU_TSC_SCALE at native and scaled frequencies
- KVM_VCPU_TSC_SCALE ratio verification against effective frequency
- Generic timing leaf 0x40000010 (EAX=tsc_khz, EBX=bus_khz)
- Xen leaf 3 sub-leaf 0 (ECX=guest TSC kHz)
- Xen leaf 3 sub-leaf 1 (ECX=mul, EDX=shift)
Gracefully skips TSC scaling tests on hardware without support.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
tools/testing/selftests/kvm/Makefile.kvm | 1 +
.../selftests/kvm/x86/xen_cpuid_timing_test.c | 230 ++++++++++++++++++
2 files changed, 231 insertions(+)
create mode 100644 tools/testing/selftests/kvm/x86/xen_cpuid_timing_test.c
diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
index 7ecaaf82056e..58aac2980cdf 100644
--- a/tools/testing/selftests/kvm/Makefile.kvm
+++ b/tools/testing/selftests/kvm/Makefile.kvm
@@ -141,6 +141,7 @@ TEST_GEN_PROGS_x86 += x86/xss_msr_test
TEST_GEN_PROGS_x86 += x86/debug_regs
TEST_GEN_PROGS_x86 += x86/tsc_msrs_test
TEST_GEN_PROGS_x86 += x86/vmx_pmu_caps_test
+TEST_GEN_PROGS_x86 += x86/xen_cpuid_timing_test
TEST_GEN_PROGS_x86 += x86/xen_shinfo_test
TEST_GEN_PROGS_x86 += x86/xen_vmcall_test
TEST_GEN_PROGS_x86 += x86/sev_init2_tests
diff --git a/tools/testing/selftests/kvm/x86/xen_cpuid_timing_test.c b/tools/testing/selftests/kvm/x86/xen_cpuid_timing_test.c
new file mode 100644
index 000000000000..a0c262b8db89
--- /dev/null
+++ b/tools/testing/selftests/kvm/x86/xen_cpuid_timing_test.c
@@ -0,0 +1,230 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Test that userspace can correctly populate Xen and generic CPUID
+ * timing leaves using KVM_GET_TSC_KHZ and KVM_VCPU_TSC_SCALE.
+ *
+ * This validates that the removal of KVM's runtime Xen CPUID modification
+ * doesn't break guests, because userspace has all the information needed.
+ */
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "test_util.h"
+#include "kvm_util.h"
+#include "processor.h"
+
+#include <asm/pvclock-abi.h>
+
+#define XEN_CPUID_BASE 0x40000100
+#define XEN_CPUID_LEAF(n) (XEN_CPUID_BASE + (n))
+#define GENERIC_TIMING_LEAF 0x40000010
+
+/* Values set by host, verified by guest */
+static uint32_t expected_tsc_khz;
+static uint32_t expected_bus_khz;
+static uint32_t expected_tsc_mul;
+static int8_t expected_tsc_shift;
+static uint64_t host_khz;
+
+static void guest_code(void)
+{
+ uint32_t eax, ebx, ecx, edx;
+
+ /* Check generic timing leaf 0x40000010 */
+ __cpuid(GENERIC_TIMING_LEAF, 0, &eax, &ebx, &ecx, &edx);
+ GUEST_ASSERT_EQ(eax, expected_tsc_khz);
+ GUEST_ASSERT_EQ(ebx, expected_bus_khz);
+
+ /* Check Xen leaf 3, sub-leaf 0: ECX = guest TSC frequency */
+ __cpuid(XEN_CPUID_LEAF(3), 0, &eax, &ebx, &ecx, &edx);
+ GUEST_ASSERT_EQ(ecx, expected_tsc_khz);
+
+ /* Check Xen leaf 3, sub-leaf 1: ECX = mul, EDX = shift */
+ __cpuid(XEN_CPUID_LEAF(3), 1, &eax, &ebx, &ecx, &edx);
+ GUEST_ASSERT_EQ(ecx, expected_tsc_mul);
+ GUEST_ASSERT_EQ((int8_t)edx, expected_tsc_shift);
+
+ GUEST_SYNC(0);
+}
+
+static void add_cpuid_entry(struct kvm_vcpu *vcpu, uint32_t function,
+ uint32_t index, uint32_t eax, uint32_t ebx,
+ uint32_t ecx, uint32_t edx)
+{
+ struct kvm_cpuid2 *cpuid = vcpu->cpuid;
+ struct kvm_cpuid_entry2 *entry;
+ int n = cpuid->nent;
+
+ vcpu->cpuid = realloc(vcpu->cpuid,
+ sizeof(*cpuid) + (n + 1) * sizeof(*entry));
+ cpuid = vcpu->cpuid;
+ cpuid->nent = n + 1;
+
+ entry = &cpuid->entries[n];
+ memset(entry, 0, sizeof(*entry));
+ entry->function = function;
+ entry->index = index;
+ entry->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
+ entry->eax = eax;
+ entry->ebx = ebx;
+ entry->ecx = ecx;
+ entry->edx = edx;
+}
+
+/*
+ * Compute pvclock mul/shift from frequency, matching kvm_get_time_scale().
+ */
+static void compute_tsc_mul_shift(uint64_t tsc_hz, uint32_t *mul, int8_t *shift)
+{
+ uint64_t scaled = 1000000000ULL;
+ uint64_t base = tsc_hz;
+ int32_t s = 0;
+ uint32_t base32;
+
+ while (base > scaled * 2 || base >> 32) {
+ base >>= 1;
+ s--;
+ }
+ base32 = (uint32_t)base;
+ while (base32 <= scaled || scaled >> 32) {
+ if (scaled >> 32 || base32 & (1U << 31))
+ scaled >>= 1;
+ else
+ base32 <<= 1;
+ s++;
+ }
+ *mul = (uint32_t)((scaled << 32) / base32);
+ *shift = (int8_t)s;
+}
+
+static void run_test(uint64_t tsc_khz)
+{
+ struct kvm_vcpu *vcpu;
+ struct kvm_vm *vm;
+ struct ucall uc;
+ uint32_t effective_tsc_khz, effective_bus_khz;
+ int bus_cycle_ns;
+
+ vm = vm_create_with_one_vcpu(&vcpu, guest_code);
+
+ if (tsc_khz) {
+ pr_info("Testing at TSC frequency %lu kHz\n", tsc_khz);
+ vcpu_ioctl(vcpu, KVM_SET_TSC_KHZ, (void *)(unsigned long)tsc_khz);
+ } else {
+ pr_info("Testing at native TSC frequency\n");
+ }
+
+ effective_tsc_khz = __vcpu_ioctl(vcpu, KVM_GET_TSC_KHZ, NULL);
+ bus_cycle_ns = vm_check_cap(vm, KVM_CAP_X86_APIC_BUS_CYCLES_NS);
+ effective_bus_khz = bus_cycle_ns > 0 ? 1000000 / bus_cycle_ns : 1000000;
+
+ /* If scaling wasn't applied, skip this frequency */
+ if (tsc_khz && effective_tsc_khz == host_khz) {
+ pr_info(" TSC scaling not available, skipping\n");
+ kvm_vm_release(vm);
+ return;
+ }
+
+ pr_info(" Effective TSC: %u kHz, Bus: %u kHz\n", effective_tsc_khz, effective_bus_khz);
+
+ /* Also exercise KVM_VCPU_TSC_SCALE if available */
+ {
+ struct { uint64_t ratio; uint64_t frac_bits; } scale;
+ struct kvm_device_attr scale_attr = {
+ .group = KVM_VCPU_TSC_CTRL,
+ .attr = 1, /* KVM_VCPU_TSC_SCALE */
+ .addr = (uint64_t)(uintptr_t)&scale,
+ };
+
+ if (!__vcpu_ioctl(vcpu, KVM_HAS_DEVICE_ATTR, &scale_attr)) {
+ vcpu_ioctl(vcpu, KVM_GET_DEVICE_ATTR, &scale_attr);
+ pr_info(" TSC scale: ratio=%lu frac_bits=%lu\n",
+ scale.ratio, scale.frac_bits);
+
+ /*
+ * Verify: applying the ratio to the host TSC frequency
+ * should give approximately the effective frequency.
+ */
+ if (tsc_khz) {
+ uint64_t computed = ((__uint128_t)host_khz * scale.ratio) >> scale.frac_bits;
+ int64_t diff = (int64_t)computed - (int64_t)effective_tsc_khz;
+
+ TEST_ASSERT(diff >= -1 && diff <= 1,
+ "TSC_SCALE ratio mismatch: computed %lu vs effective %u (diff %ld)",
+ computed, effective_tsc_khz, diff);
+ }
+ }
+ }
+
+ compute_tsc_mul_shift((uint64_t)effective_tsc_khz * 1000,
+ &expected_tsc_mul, &expected_tsc_shift);
+
+ expected_tsc_khz = effective_tsc_khz;
+ expected_bus_khz = effective_bus_khz;
+
+ sync_global_to_guest(vm, expected_tsc_khz);
+ sync_global_to_guest(vm, expected_bus_khz);
+ sync_global_to_guest(vm, expected_tsc_mul);
+ sync_global_to_guest(vm, expected_tsc_shift);
+
+ /* Populate CPUID leaves as a VMM would */
+ add_cpuid_entry(vcpu, GENERIC_TIMING_LEAF, 0,
+ effective_tsc_khz, effective_bus_khz, 0, 0);
+ add_cpuid_entry(vcpu, XEN_CPUID_LEAF(3), 0,
+ 0, 0, effective_tsc_khz, 0);
+ add_cpuid_entry(vcpu, XEN_CPUID_LEAF(3), 1,
+ 0, 0, expected_tsc_mul,
+ (uint32_t)(uint8_t)expected_tsc_shift);
+
+ vcpu_set_cpuid(vcpu);
+
+ pr_info(" pvclock mul=%u shift=%d\n", expected_tsc_mul, expected_tsc_shift);
+
+ vcpu_run(vcpu);
+ TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
+
+ switch (get_ucall(vcpu, &uc)) {
+ case UCALL_ABORT:
+ REPORT_GUEST_ASSERT(uc);
+ break;
+ case UCALL_SYNC:
+ break;
+ default:
+ TEST_FAIL("Unexpected ucall");
+ }
+
+ kvm_vm_release(vm);
+}
+
+int main(void)
+{
+ uint64_t freq;
+ struct kvm_vcpu *vcpu;
+ struct kvm_vm *vm;
+ struct kvm_device_attr attr = {
+ .group = KVM_VCPU_TSC_CTRL,
+ .attr = KVM_VCPU_TSC_SCALE,
+ };
+
+ TEST_REQUIRE(sys_clocksource_is_based_on_tsc());
+
+ /* Check KVM_VCPU_TSC_SCALE is supported (implies TSC scaling) */
+ vm = vm_create_with_one_vcpu(&vcpu, guest_code);
+ TEST_REQUIRE(!__vcpu_ioctl(vcpu, KVM_HAS_DEVICE_ATTR, &attr));
+ host_khz = __vcpu_ioctl(vcpu, KVM_GET_TSC_KHZ, NULL);
+ kvm_vm_release(vm);
+
+ /* Native frequency */
+ run_test(0);
+
+ /* Scaled frequencies — skip if TSC scaling not available */
+ for (freq = 1000000; freq <= 4000000; freq += 1000000) {
+ if (freq == host_khz)
+ continue;
+ run_test(freq);
+ }
+
+ pr_info("PASS: All CPUID timing leaf tests passed\n");
+ return 0;
+}
--
2.54.0
^ permalink raw reply related
* [PATCH v5 31/34] KVM: x86: Use ktime_get_snapshot_id() for master clock
From: David Woodhouse @ 2026-06-08 14:48 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
Replace the KVM-private vgettsc()/do_kvmclock_base()/do_monotonic()/
do_realtime() timekeeping reimplementation with calls to the generic
ktime_get_snapshot_id() interface.
The snapshot provides both the system time and the raw_cycles (TSC)
atomically paired. When raw_cycles is zero, the clocksource could not
provide a raw hardware counter value, which is equivalent to the
previous vgettsc() returning VDSO_CLOCKMODE_NONE.
For kvm_get_time_and_clockread(), the kvmclock base time is
CLOCK_MONOTONIC_RAW + offs_boot. The snapshot provides the raw time
atomically paired with the TSC; offs_boot is added separately as it
only changes at suspend/resume boundaries.
This is a step towards eliminating the pvclock_gtod_data private copy
of timekeeping state and the associated notifier callback.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Assisted-by: Kiro:claude-opus-4.6-1m
---
arch/x86/kvm/x86.c | 46 +++++++++++++++++++++++++++++++++++-----------
1 file changed, 35 insertions(+), 11 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 96250264d403..2713aebb96ae 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -35,6 +35,7 @@
#include "smm.h"
#include <linux/clocksource.h>
+#include <linux/timekeeping.h>
#include <linux/interrupt.h>
#include <linux/kvm.h>
#include <linux/fs.h>
@@ -3162,14 +3163,32 @@ static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
* reports the TSC value from which it do so. Returns true if host is
* using TSC based clocksource.
*/
+static bool kvm_snapshot_has_tsc(struct system_time_snapshot *snap,
+ u64 *tsc_timestamp)
+{
+ if (snap->cs_id == CSID_X86_TSC) {
+ *tsc_timestamp = snap->cycles;
+ return true;
+ }
+
+ if (snap->hw_csid == CSID_X86_TSC && snap->hw_cycles) {
+ *tsc_timestamp = snap->hw_cycles;
+ return true;
+ }
+
+ return false;
+}
+
static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
{
- /* checked again under seqlock below */
- if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
+ struct system_time_snapshot snap;
+
+ ktime_get_snapshot_id(CLOCK_MONOTONIC_RAW, &snap);
+ if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp))
return false;
- return gtod_is_based_on_tsc(do_kvmclock_base(kernel_ns,
- tsc_timestamp));
+ *kernel_ns = ktime_to_ns(ktime_mono_to_any(snap.systime, TK_OFFS_BOOT));
+ return true;
}
/*
@@ -3178,12 +3197,14 @@ static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
*/
bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
{
- /* checked again under seqlock below */
- if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
+ struct system_time_snapshot snap;
+
+ ktime_get_snapshot_id(CLOCK_MONOTONIC, &snap);
+ if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp))
return false;
- return gtod_is_based_on_tsc(do_monotonic(kernel_ns,
- tsc_timestamp));
+ *kernel_ns = ktime_to_ns(snap.systime);
+ return true;
}
/*
@@ -3196,11 +3217,14 @@ bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
u64 *tsc_timestamp)
{
- /* checked again under seqlock below */
- if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
+ struct system_time_snapshot snap;
+
+ ktime_get_snapshot_id(CLOCK_REALTIME, &snap);
+ if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp))
return false;
- return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
+ *ts = ktime_to_timespec64(snap.systime);
+ return true;
}
#endif
--
2.54.0
^ permalink raw reply related
* [PATCH v5 34/34] KVM: x86: Remove pvclock_gtod_data and private timekeeping code
From: David Woodhouse @ 2026-06-08 14:48 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
Remove the now-unused KVM-private timekeeping infrastructure:
- struct pvclock_clock and struct pvclock_gtod_data
- update_pvclock_gtod() and its seqcount-protected state copy
- read_tsc() (KVM's private TSC reader with cycle_last clamping)
- vgettsc() (KVM's private clocksource interpolation)
- do_kvmclock_base(), do_monotonic(), do_realtime()
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Assisted-by: Kiro:claude-opus-4.6-1m
---
Documentation/virt/kvm/devices/vcpu.rst | 4 +-
arch/x86/kvm/vmx/vmx.c | 2 +
arch/x86/kvm/x86.c | 177 +-----------------
.../testing/selftests/kvm/x86/pvclock_test.c | 7 +-
4 files changed, 9 insertions(+), 181 deletions(-)
diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst
index 167aa4140d30..3d1a89c2b4f7 100644
--- a/Documentation/virt/kvm/devices/vcpu.rst
+++ b/Documentation/virt/kvm/devices/vcpu.rst
@@ -243,9 +243,9 @@ Returns:
Specifies the guest's TSC offset relative to the host's TSC. The guest's
TSC is then derived by the following equation:
- guest_tsc = ((host_tsc * tsc_scale_ratio) >> tsc_scale_bits) + KVM_VCPU_TSC_OFFSET
+ guest_tsc = ((host_tsc * tsc_ratio) >> tsc_frac_bits) + KVM_VCPU_TSC_OFFSET
-The values of tsc_scale_ratio and tsc_scale_bits can be obtained using
+The values of tsc_ratio and tsc_frac_bits can be obtained using
the KVM_VCPU_TSC_SCALE attribute.
This attribute is useful to adjust the guest's TSC on live migration,
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index ed207cc7692d..1aaf3924a799 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -8674,6 +8674,8 @@ __init int vmx_hardware_setup(void)
if (cpu_has_vmx_tsc_scaling() && boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
kvm_caps.has_tsc_control = true;
+ else
+ vmcs_config.cpu_based_2nd_exec_ctrl &= ~SECONDARY_EXEC_TSC_SCALING;
kvm_caps.max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX;
kvm_caps.tsc_scaling_ratio_frac_bits = 48;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 93a428c37847..966057913366 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -2347,58 +2347,6 @@ static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
return kvm_set_msr_ignored_check(vcpu, index, *data, true);
}
-struct pvclock_clock {
- int vclock_mode;
- u64 cycle_last;
- u64 mask;
- u32 mult;
- u32 shift;
- u64 base_cycles;
- u64 offset;
-};
-
-struct pvclock_gtod_data {
- seqcount_t seq;
-
- struct pvclock_clock clock; /* extract of a clocksource struct */
- struct pvclock_clock raw_clock; /* extract of a clocksource struct */
-
- ktime_t offs_boot;
- u64 wall_time_sec;
-};
-
-static struct pvclock_gtod_data pvclock_gtod_data;
-
-static void update_pvclock_gtod(struct timekeeper *tk)
-{
- struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
-
- write_seqcount_begin(&vdata->seq);
-
- /* copy pvclock gtod data */
- vdata->clock.vclock_mode = tk->tkr_mono.clock->vdso_clock_mode;
- vdata->clock.cycle_last = tk->tkr_mono.cycle_last;
- vdata->clock.mask = tk->tkr_mono.mask;
- vdata->clock.mult = tk->tkr_mono.mult;
- vdata->clock.shift = tk->tkr_mono.shift;
- vdata->clock.base_cycles = tk->tkr_mono.xtime_nsec;
- vdata->clock.offset = tk->tkr_mono.base;
-
- vdata->raw_clock.vclock_mode = tk->tkr_raw.clock->vdso_clock_mode;
- vdata->raw_clock.cycle_last = tk->tkr_raw.cycle_last;
- vdata->raw_clock.mask = tk->tkr_raw.mask;
- vdata->raw_clock.mult = tk->tkr_raw.mult;
- vdata->raw_clock.shift = tk->tkr_raw.shift;
- vdata->raw_clock.base_cycles = tk->tkr_raw.xtime_nsec;
- vdata->raw_clock.offset = tk->tkr_raw.base;
-
- vdata->wall_time_sec = tk->xtime_sec;
-
- vdata->offs_boot = tk->offs_boot;
-
- write_seqcount_end(&vdata->seq);
-}
-
static s64 get_kvmclock_base_ns(void)
{
/* Count up from boot time, but with the frequency of the raw clock. */
@@ -3037,128 +2985,6 @@ static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
#ifdef CONFIG_X86_64
-static u64 read_tsc(void)
-{
- u64 ret = (u64)rdtsc_ordered();
- u64 last = pvclock_gtod_data.clock.cycle_last;
-
- if (likely(ret >= last))
- return ret;
-
- /*
- * GCC likes to generate cmov here, but this branch is extremely
- * predictable (it's just a function of time and the likely is
- * very likely) and there's a data dependence, so force GCC
- * to generate a branch instead. I don't barrier() because
- * we don't actually need a barrier, and if this function
- * ever gets inlined it will generate worse code.
- */
- asm volatile ("");
- return last;
-}
-
-static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
- int *mode)
-{
- u64 tsc_pg_val;
- long v;
-
- switch (clock->vclock_mode) {
- case VDSO_CLOCKMODE_HVCLOCK:
- if (hv_read_tsc_page_tsc(hv_get_tsc_page(),
- tsc_timestamp, &tsc_pg_val)) {
- /* TSC page valid */
- *mode = VDSO_CLOCKMODE_HVCLOCK;
- v = (tsc_pg_val - clock->cycle_last) &
- clock->mask;
- } else {
- /* TSC page invalid */
- *mode = VDSO_CLOCKMODE_NONE;
- }
- break;
- case VDSO_CLOCKMODE_TSC:
- *mode = VDSO_CLOCKMODE_TSC;
- *tsc_timestamp = read_tsc();
- v = (*tsc_timestamp - clock->cycle_last) &
- clock->mask;
- break;
- default:
- *mode = VDSO_CLOCKMODE_NONE;
- }
-
- if (*mode == VDSO_CLOCKMODE_NONE)
- *tsc_timestamp = v = 0;
-
- return v * clock->mult;
-}
-
-/*
- * As with get_kvmclock_base_ns(), this counts from boot time, at the
- * frequency of CLOCK_MONOTONIC_RAW (hence adding gtos->offs_boot).
- */
-static int do_kvmclock_base(s64 *t, u64 *tsc_timestamp)
-{
- struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
- unsigned long seq;
- int mode;
- u64 ns;
-
- do {
- seq = read_seqcount_begin(>od->seq);
- ns = gtod->raw_clock.base_cycles;
- ns += vgettsc(>od->raw_clock, tsc_timestamp, &mode);
- ns >>= gtod->raw_clock.shift;
- ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot));
- } while (unlikely(read_seqcount_retry(>od->seq, seq)));
- *t = ns;
-
- return mode;
-}
-
-/*
- * This calculates CLOCK_MONOTONIC at the time of the TSC snapshot, with
- * no boot time offset.
- */
-static int do_monotonic(s64 *t, u64 *tsc_timestamp)
-{
- struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
- unsigned long seq;
- int mode;
- u64 ns;
-
- do {
- seq = read_seqcount_begin(>od->seq);
- ns = gtod->clock.base_cycles;
- ns += vgettsc(>od->clock, tsc_timestamp, &mode);
- ns >>= gtod->clock.shift;
- ns += ktime_to_ns(gtod->clock.offset);
- } while (unlikely(read_seqcount_retry(>od->seq, seq)));
- *t = ns;
-
- return mode;
-}
-
-static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
-{
- struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
- unsigned long seq;
- int mode;
- u64 ns;
-
- do {
- seq = read_seqcount_begin(>od->seq);
- ts->tv_sec = gtod->wall_time_sec;
- ns = gtod->clock.base_cycles;
- ns += vgettsc(>od->clock, tsc_timestamp, &mode);
- ns >>= gtod->clock.shift;
- } while (unlikely(read_seqcount_retry(>od->seq, seq)));
-
- ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
- ts->tv_nsec = ns;
-
- return mode;
-}
-
/*
* Calculates the kvmclock_base_ns (CLOCK_MONOTONIC_RAW + boot time) and
* reports the TSC value from which it do so. Returns true if host is
@@ -6231,7 +6057,7 @@ static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu,
break;
}
case KVM_VCPU_TSC_SCALE:
- r = -EINVAL; /* Read only */
+ r = kvm_caps.has_tsc_control ? -EINVAL : -ENXIO;
break;
default:
r = -ENXIO;
@@ -10405,7 +10231,6 @@ static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
{
struct timekeeper *tk = priv;
- update_pvclock_gtod(tk);
#ifdef CONFIG_X86_64
kvm_host_has_tsc_clocksource =
diff --git a/tools/testing/selftests/kvm/x86/pvclock_test.c b/tools/testing/selftests/kvm/x86/pvclock_test.c
index aecd62fc8a93..4c1869fa482e 100644
--- a/tools/testing/selftests/kvm/x86/pvclock_test.c
+++ b/tools/testing/selftests/kvm/x86/pvclock_test.c
@@ -14,7 +14,6 @@
#include "test_util.h"
#include "kvm_util.h"
#include "processor.h"
-#include "apic.h"
#include <asm/pvclock-abi.h>
@@ -262,10 +261,12 @@ int main(int argc, char *argv[])
return 0;
}
+static volatile uint32_t vcpu_counter;
+
static void guest_code_stable_bit(void)
{
- uint32_t apic_id = GET_APIC_ID_FIELD(xapic_read_reg(APIC_ID));
- uint64_t gpa = KVMCLOCK_GPA + apic_id * sizeof(struct pvclock_vcpu_time_info);
+ uint32_t idx = __atomic_fetch_add(&vcpu_counter, 1, __ATOMIC_SEQ_CST);
+ uint64_t gpa = KVMCLOCK_GPA + idx * sizeof(struct pvclock_vcpu_time_info);
wrmsr(MSR_KVM_SYSTEM_TIME_NEW, gpa | KVM_MSR_ENABLED);
GUEST_SYNC(0);
--
2.54.0
^ permalink raw reply related
* [PATCH v5 17/34] KVM: x86: Remove implicit rdtsc() from kvm_compute_l1_tsc_offset()
From: David Woodhouse @ 2026-06-08 14:47 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
Let the callers pass the host TSC value in as an explicit parameter.
This leaves some fairly obviously stupid code, which is using this
function to compare the guest TSC at some *other* time, with the
newly-minted TSC value from rdtsc(). Unless it's being used to measure
*elapsed* time, that isn't very sensible.
In this case, "obviously stupid" is an improvement over being
non-obviously so.
No functional change intended.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Reviewed-by: Paul Durrant <paul@xen.org>
---
arch/x86/kvm/x86.c | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index a6c31a0d9955..bce4c7a6a6fe 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -2693,11 +2693,12 @@ u64 kvm_scale_tsc(u64 tsc, u64 ratio)
return _tsc;
}
-static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
+static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc,
+ u64 target_tsc)
{
u64 tsc;
- tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);
+ tsc = kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);
return target_tsc - tsc;
}
@@ -2859,7 +2860,7 @@ static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
bool synchronizing = false;
raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
- offset = kvm_compute_l1_tsc_offset(vcpu, data);
+ offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data);
ns = get_kvmclock_base_ns();
elapsed = ns - kvm->arch.last_tsc_nsec;
@@ -2908,7 +2909,7 @@ static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
} else {
u64 delta = nsec_to_cycles(vcpu, elapsed);
data += delta;
- offset = kvm_compute_l1_tsc_offset(vcpu, data);
+ offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data);
}
matched = true;
}
@@ -4155,7 +4156,8 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
if (msr_info->host_initiated) {
kvm_synchronize_tsc(vcpu, &data);
} else if (!vcpu->arch.guest_tsc_protected) {
- u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;
+ u64 adj = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data) -
+ vcpu->arch.l1_tsc_offset;
adjust_tsc_offset_guest(vcpu, adj);
vcpu->arch.ia32_tsc_adjust_msr += adj;
}
@@ -5279,7 +5281,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
mark_tsc_unstable("KVM discovered backwards TSC");
if (kvm_check_tsc_unstable()) {
- u64 offset = kvm_compute_l1_tsc_offset(vcpu,
+ u64 offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(),
vcpu->arch.last_guest_tsc);
kvm_vcpu_write_tsc_offset(vcpu, offset);
if (!vcpu->arch.guest_tsc_protected)
--
2.54.0
^ permalink raw reply related
* [PATCH v5 14/34] KVM: x86: Fix compute_guest_tsc() to handle negative time deltas
From: David Woodhouse @ 2026-06-08 14:47 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
The compute_guest_tsc() function computes the guest TSC at a given
kernel_ns timestamp. When the master clock reference point
(master_kernel_ns) is earlier than vcpu->arch.this_tsc_nsec, the delta
is negative. Since pvclock_scale_delta() takes a u64, the negative
value wraps to a huge positive number, producing a wildly wrong result.
Handle negative deltas explicitly by negating the delta, scaling it,
and subtracting from this_tsc_write.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
arch/x86/kvm/x86.c | 20 +++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index fc9366b83912..8aae22401046 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -2588,11 +2588,21 @@ static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
{
- u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
- vcpu->arch.virtual_tsc_mult,
- vcpu->arch.virtual_tsc_shift);
- tsc += vcpu->arch.this_tsc_write;
- return tsc;
+ s64 delta_ns = kernel_ns - vcpu->arch.this_tsc_nsec;
+ u64 tsc;
+
+ /* Handle negative deltas gracefully (master clock ref may be earlier) */
+ if (delta_ns < 0) {
+ tsc = pvclock_scale_delta(-delta_ns,
+ vcpu->arch.virtual_tsc_mult,
+ vcpu->arch.virtual_tsc_shift);
+ return vcpu->arch.this_tsc_write - tsc;
+ }
+
+ tsc = pvclock_scale_delta(delta_ns,
+ vcpu->arch.virtual_tsc_mult,
+ vcpu->arch.virtual_tsc_shift);
+ return vcpu->arch.this_tsc_write + tsc;
}
#ifdef CONFIG_X86_64
--
2.54.0
^ permalink raw reply related
* [PATCH v5 19/34] KVM: x86: Kill last_tsc_{nsec,write,offset} fields
From: David Woodhouse @ 2026-06-08 14:48 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
These pointlessly duplicate the cur_tsc_{nsec,write,offset} values.
The only place they were used was where the TSC is stable and a new
vCPU is being synchronized to the previous setting, in which case the
cur_tsc_* value is definitely identical.
Rename last_tsc_khz and last_tsc_scaling_ratio to cur_tsc_khz and
cur_tsc_scaling_ratio respectively, since they are properties of the
current TSC generation.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Reviewed-by: Paul Durrant <paul@xen.org>
---
arch/x86/include/asm/kvm_host.h | 7 ++----
arch/x86/kvm/x86.c | 42 ++++++++++++++++-----------------
2 files changed, 22 insertions(+), 27 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 5348fd5ea3f3..59298a8f78eb 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1486,11 +1486,8 @@ struct kvm_arch {
* preemption-disabled region, so it must be a raw spinlock.
*/
raw_spinlock_t tsc_write_lock;
- u64 last_tsc_nsec;
- u64 last_tsc_write;
- u32 last_tsc_khz;
- u64 last_tsc_offset;
- u64 last_tsc_scaling_ratio;
+ u32 cur_tsc_khz;
+ u64 cur_tsc_scaling_ratio;
u64 cur_tsc_nsec;
u64 cur_tsc_write;
u64 cur_tsc_offset;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index c8c0633263fb..bbd642e0dc54 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -2813,14 +2813,12 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
vcpu->kvm->arch.user_set_tsc = true;
/*
- * We also track th most recent recorded KHZ, write and time to
- * allow the matching interval to be extended at each write.
+ * Track the TSC frequency, scaling ratio, and offset for the current
+ * generation. These are used to detect matching TSC writes and to
+ * compute the guest TSC from the host clock.
*/
- kvm->arch.last_tsc_nsec = ns;
- kvm->arch.last_tsc_write = tsc;
- kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
- kvm->arch.last_tsc_offset = offset;
- kvm->arch.last_tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio;
+ kvm->arch.cur_tsc_khz = vcpu->arch.virtual_tsc_khz;
+ kvm->arch.cur_tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio;
vcpu->arch.last_guest_tsc = tsc;
@@ -2833,8 +2831,6 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
* nanosecond time, offset, and write, so if TSCs are in
* sync, we can match exact offset, and if not, we can match
* exact software computation in compute_guest_tsc()
- *
- * These values are tracked in kvm->arch.cur_xxx variables.
*/
kvm->arch.cur_tsc_generation++;
kvm->arch.cur_tsc_nsec = ns;
@@ -2874,7 +2870,7 @@ static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
}
offset = kvm_compute_l1_tsc_offset(vcpu, host_tsc, data);
- elapsed = ns - kvm->arch.last_tsc_nsec;
+ elapsed = ns - kvm->arch.cur_tsc_nsec;
if (vcpu->arch.virtual_tsc_khz) {
if (data == 0) {
@@ -2884,7 +2880,7 @@ static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
*/
synchronizing = true;
} else if (kvm->arch.user_set_tsc) {
- u64 tsc_exp = kvm->arch.last_tsc_write +
+ u64 tsc_exp = kvm->arch.cur_tsc_write +
nsec_to_cycles(vcpu, elapsed);
u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
/*
@@ -2915,14 +2911,14 @@ static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
* it's better to try to match offsets from the beginning.
*/
if (synchronizing &&
- vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
+ vcpu->arch.virtual_tsc_khz == kvm->arch.cur_tsc_khz) {
/*
* If synchronizing, the "last written" TSC value/time
* recorded by __kvm_synchronize_tsc() should not change
* (i.e. should be precisely the same as the existing
* generation).
*/
- data = kvm->arch.last_tsc_write;
+ data = kvm->arch.cur_tsc_write;
if (!kvm_check_tsc_unstable()) {
offset = kvm->arch.cur_tsc_offset;
@@ -3207,7 +3203,7 @@ static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
* get_kvmclock() to compute kvmclock from the host TSC
* without needing a vCPU reference.
*/
- ka->master_tsc_scaling_ratio = ka->last_tsc_scaling_ratio;
+ ka->master_tsc_scaling_ratio = ka->cur_tsc_scaling_ratio;
tsc_hz = (u64)get_cpu_tsc_khz() * 1000;
if (tsc_hz && kvm_caps.has_tsc_control)
tsc_hz = kvm_scale_tsc(tsc_hz,
@@ -6088,8 +6084,8 @@ static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu,
raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
matched = (vcpu->arch.virtual_tsc_khz &&
- kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz &&
- kvm->arch.last_tsc_offset == offset);
+ kvm->arch.cur_tsc_khz == vcpu->arch.virtual_tsc_khz &&
+ kvm->arch.cur_tsc_offset == offset);
tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset;
ns = get_kvmclock_base_ns();
@@ -13543,13 +13539,15 @@ int kvm_arch_enable_virtualization_cpu(void)
}
/*
- * We have to disable TSC offset matching.. if you were
- * booting a VM while issuing an S4 host suspend....
- * you may have some problem. Solving this issue is
- * left as an exercise to the reader.
+ * Adjust the TSC matching reference by the same
+ * delta applied to each vCPU's offset, so that
+ * future KVM_SET_TSC / vCPU creation still matches
+ * correctly against the adjusted TSC timeline.
+ * Scale from host to guest TSC rate.
*/
- kvm->arch.last_tsc_nsec = 0;
- kvm->arch.last_tsc_write = 0;
+ kvm->arch.cur_tsc_write -=
+ kvm_scale_tsc(delta_cyc,
+ kvm->arch.cur_tsc_scaling_ratio);
}
}
--
2.54.0
^ permalink raw reply related
* [PATCH v5 09/34] KVM: x86: Avoid NTP frequency skew for KVM clock on 32-bit host
From: David Woodhouse @ 2026-06-08 14:47 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
Commit 53fafdbb8b21 ("KVM: x86: switch KVMCLOCK base to monotonic raw
clock") did so only for 64-bit hosts, by capturing the boot offset from
within the existing clocksource notifier update_pvclock_gtod().
That notifier was added in commit 16e8d74d2da9 ("KVM: x86: notifier for
clocksource changes") but only on x86_64, because its original purpose
was just to disable the "master clock" mode which is only supported on
x86_64.
Now that the notifier is used for more than disabling master clock mode,
enable it for the 32-bit build too so that get_kvmclock_base_ns() can be
unaffected by NTP sync on 32-bit too.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Reviewed-by: Paul Durrant <paul@xen.org>
---
arch/x86/kvm/x86.c | 19 ++++++-------------
1 file changed, 6 insertions(+), 13 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 6337f9b9d7ac..50bd2871b051 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -2342,7 +2342,6 @@ static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
return kvm_set_msr_ignored_check(vcpu, index, *data, true);
}
-#ifdef CONFIG_X86_64
struct pvclock_clock {
int vclock_mode;
u64 cycle_last;
@@ -2400,13 +2399,6 @@ static s64 get_kvmclock_base_ns(void)
/* Count up from boot time, but with the frequency of the raw clock. */
return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot));
}
-#else
-static s64 get_kvmclock_base_ns(void)
-{
- /* Master clock not used, so we can just use CLOCK_BOOTTIME. */
- return ktime_get_boottime_ns();
-}
-#endif
static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs)
{
@@ -10173,6 +10165,7 @@ static void pvclock_irq_work_fn(struct irq_work *w)
}
static DEFINE_IRQ_WORK(pvclock_irq_work, pvclock_irq_work_fn);
+#endif
/*
* Notification about pvclock gtod data update.
@@ -10180,26 +10173,26 @@ static DEFINE_IRQ_WORK(pvclock_irq_work, pvclock_irq_work_fn);
static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
void *priv)
{
- struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
struct timekeeper *tk = priv;
update_pvclock_gtod(tk);
+#ifdef CONFIG_X86_64
/*
* Disable master clock if host does not trust, or does not use,
* TSC based clocksource. Delegate queue_work() to irq_work as
* this is invoked with tk_core.seq write held.
*/
- if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) &&
+ if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode) &&
atomic_read(&kvm_guest_has_master_clock) != 0)
irq_work_queue(&pvclock_irq_work);
+#endif
return 0;
}
static struct notifier_block pvclock_gtod_notifier = {
.notifier_call = pvclock_gtod_notify,
};
-#endif
void kvm_setup_xss_caps(void)
{
@@ -10388,9 +10381,9 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops)
if (pi_inject_timer == -1)
pi_inject_timer = housekeeping_enabled(HK_TYPE_TIMER);
-#ifdef CONFIG_X86_64
pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
+#ifdef CONFIG_X86_64
if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
#endif
@@ -10447,8 +10440,8 @@ void kvm_x86_vendor_exit(void)
CPUFREQ_TRANSITION_NOTIFIER);
cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);
}
-#ifdef CONFIG_X86_64
pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
+#ifdef CONFIG_X86_64
irq_work_sync(&pvclock_irq_work);
cancel_work_sync(&pvclock_gtod_work);
#endif
--
2.54.0
^ permalink raw reply related
* [PATCH v5 26/34] KVM: x86: Avoid redundant masterclock updates from multiple vCPUs
From: David Woodhouse @ 2026-06-08 14:48 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
When a masterclock update is triggered (e.g. by the clocksource change
notifier), KVM_REQ_MASTERCLOCK_UPDATE is set on all vCPUs. Without this
fix, each vCPU independently processes the request and redundantly
re-executes the entire pvclock_update_vm_gtod_copy() sequence, serialized
only by tsc_write_lock. Each redundant re-snapshot of the master clock
reference point introduces potential clock drift.
Fix this by having __kvm_start_pvclock_update() check, after acquiring
the lock, whether the requesting vCPU's KVM_REQ_MASTERCLOCK_UPDATE is
still set. If another vCPU already did the update and cleared it, bail
out. Otherwise, clear the request on all other vCPUs before proceeding.
The caller in vcpu_enter_guest() now uses kvm_test_request() (non-clearing)
since the clearing is done inside __kvm_start_pvclock_update() under the
lock.
Suggested-by: Dongli Zhang <dongli.zhang@oracle.com>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
arch/x86/kvm/x86.c | 60 +++++++++++++++++++++++++++++++++++-----------
1 file changed, 46 insertions(+), 14 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 4fc21d701588..54d4b1b3cfe4 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -3288,10 +3288,39 @@ static void kvm_make_mclock_inprogress_request(struct kvm *kvm)
kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
}
-static void __kvm_start_pvclock_update(struct kvm *kvm)
+static void kvm_clear_mclock_inprogress_request(struct kvm *kvm)
{
+ struct kvm_vcpu *vcpu;
+ unsigned long i;
+
+ kvm_for_each_vcpu(i, vcpu, kvm)
+ kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
+}
+
+static bool __kvm_start_pvclock_update(struct kvm *kvm, struct kvm_vcpu *requesting_vcpu)
+{
+ struct kvm_vcpu *vcpu;
+ unsigned long i;
+
raw_spin_lock_irq(&kvm->arch.tsc_write_lock);
+
+ /*
+ * If another vCPU already did the update while we were waiting
+ * for the lock, our request will have been cleared. Bail out.
+ */
+ if (requesting_vcpu &&
+ !kvm_test_request(KVM_REQ_MASTERCLOCK_UPDATE, requesting_vcpu)) {
+ kvm_clear_mclock_inprogress_request(kvm);
+ raw_spin_unlock_irq(&kvm->arch.tsc_write_lock);
+ return false;
+ }
+
+ /* The update is VM-wide; prevent other vCPUs from redoing it. */
+ kvm_for_each_vcpu(i, vcpu, kvm)
+ kvm_clear_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
+
write_seqcount_begin(&kvm->arch.pvclock_sc);
+ return true;
}
static void kvm_start_pvclock_update(struct kvm *kvm)
@@ -3299,7 +3328,7 @@ static void kvm_start_pvclock_update(struct kvm *kvm)
kvm_make_mclock_inprogress_request(kvm);
/* no guest entries from this point */
- __kvm_start_pvclock_update(kvm);
+ __kvm_start_pvclock_update(kvm, NULL);
}
static void kvm_end_pvclock_update(struct kvm *kvm)
@@ -3308,22 +3337,25 @@ static void kvm_end_pvclock_update(struct kvm *kvm)
struct kvm_vcpu *vcpu;
unsigned long i;
- write_seqcount_end(&ka->pvclock_sc);
- raw_spin_unlock_irq(&ka->tsc_write_lock);
kvm_for_each_vcpu(i, vcpu, kvm)
kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
/* guest entries allowed */
- kvm_for_each_vcpu(i, vcpu, kvm)
- kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
+ kvm_clear_mclock_inprogress_request(kvm);
+
+ write_seqcount_end(&ka->pvclock_sc);
+ raw_spin_unlock_irq(&ka->tsc_write_lock);
}
-static void kvm_update_masterclock(struct kvm *kvm)
+static void kvm_update_masterclock(struct kvm *kvm, struct kvm_vcpu *vcpu)
{
kvm_hv_request_tsc_page_update(kvm);
- kvm_start_pvclock_update(kvm);
- pvclock_update_vm_gtod_copy(kvm);
- kvm_end_pvclock_update(kvm);
+ kvm_make_mclock_inprogress_request(kvm);
+
+ if (__kvm_start_pvclock_update(kvm, vcpu)) {
+ pvclock_update_vm_gtod_copy(kvm);
+ kvm_end_pvclock_update(kvm);
+ }
}
/*
@@ -10157,7 +10189,7 @@ static void kvm_hyperv_tsc_notifier(void)
kvm_caps.max_guest_tsc_khz = tsc_khz;
list_for_each_entry(kvm, &vm_list, vm_list) {
- __kvm_start_pvclock_update(kvm);
+ __kvm_start_pvclock_update(kvm, NULL);
pvclock_update_vm_gtod_copy(kvm);
kvm_end_pvclock_update(kvm);
}
@@ -11535,8 +11567,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
kvm_mmu_free_obsolete_roots(vcpu);
if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
__kvm_migrate_timers(vcpu);
- if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
- kvm_update_masterclock(vcpu->kvm);
+ if (kvm_test_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
+ kvm_update_masterclock(vcpu->kvm, vcpu);
if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
kvm_gen_kvmclock_update(vcpu);
if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
@@ -13273,7 +13305,7 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
vcpu_load(vcpu);
kvm_synchronize_tsc(vcpu, NULL);
if (!vcpu->kvm->arch.use_master_clock)
- kvm_update_masterclock(vcpu->kvm);
+ kvm_update_masterclock(vcpu->kvm, NULL);
vcpu_put(vcpu);
/* poll control enabled by default */
--
2.54.0
^ permalink raw reply related
* [PATCH v5 33/34] KVM: x86: Replace pvclock_gtod_data vclock_mode with boolean
From: David Woodhouse @ 2026-06-08 14:48 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
The remaining users of pvclock_gtod_data only need to know whether
the host clocksource is TSC-based. Replace all vclock_mode checks
with a simple kvm_host_has_tsc_clocksource boolean, updated by the
pvclock_gtod_notify callback.
This is inherently racy (as it always was — kvm_track_tsc_matching
never held the gtod seqcount), relying on eventual consistency: the
notifier fires on every timekeeping update and will correct any
transient inconsistency within one tick.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Assisted-by: Kiro:claude-opus-4.6-1m
---
arch/x86/kvm/x86.c | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index c18947c5b63f..93a428c37847 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -2649,6 +2649,8 @@ static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
}
#ifdef CONFIG_X86_64
+static bool kvm_host_has_tsc_clocksource;
+
static inline bool gtod_is_based_on_tsc(int mode)
{
return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK;
@@ -2678,7 +2680,6 @@ static bool kvm_use_master_clock(struct kvm *kvm)
static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu, bool new_generation)
{
struct kvm_arch *ka = &vcpu->kvm->arch;
- struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
/*
* Track whether all vCPUs have matching TSC offsets (for
@@ -2712,7 +2713,7 @@ static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu, bool new_generation)
* accounts for its offset.
*/
bool use_master_clock = kvm_use_master_clock(vcpu->kvm) &&
- gtod_is_based_on_tsc(gtod->clock.vclock_mode);
+ kvm_host_has_tsc_clocksource;
/*
* Request a masterclock update if the masterclock needs to be toggled
@@ -2726,7 +2727,7 @@ static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu, bool new_generation)
trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
atomic_read(&vcpu->kvm->online_vcpus),
- ka->use_master_clock, gtod->clock.vclock_mode);
+ ka->use_master_clock, kvm_host_has_tsc_clocksource);
}
#else
static inline void kvm_track_tsc_matching(struct kvm_vcpu *vcpu,
@@ -2850,7 +2851,7 @@ static inline bool kvm_check_tsc_unstable(void)
* TSC is marked unstable when we're running on Hyper-V,
* 'TSC page' clocksource is good.
*/
- if (pvclock_gtod_data.clock.vclock_mode == VDSO_CLOCKMODE_HVCLOCK)
+ if (kvm_host_has_tsc_clocksource)
return false;
#endif
return check_tsc_unstable();
@@ -3315,7 +3316,7 @@ static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
ka->use_master_clock = false;
}
- vclock_mode = pvclock_gtod_data.clock.vclock_mode;
+ vclock_mode = kvm_host_has_tsc_clocksource;
trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
ka->all_vcpus_matched_freq);
#endif
@@ -10407,12 +10408,15 @@ static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
update_pvclock_gtod(tk);
#ifdef CONFIG_X86_64
+ kvm_host_has_tsc_clocksource =
+ gtod_is_based_on_tsc(tk->tkr_mono.clock->vdso_clock_mode);
+
/*
* Disable master clock if host does not trust, or does not use,
* TSC based clocksource. Delegate queue_work() to irq_work as
* this is invoked with tk_core.seq write held.
*/
- if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode) &&
+ if (!kvm_host_has_tsc_clocksource &&
atomic_read(&kvm_guest_has_master_clock) != 0)
irq_work_queue(&pvclock_irq_work);
#endif
--
2.54.0
^ permalink raw reply related
* [PATCH v5 27/34] KVM: x86: Remove runtime Xen TSC frequency CPUID update
From: David Woodhouse @ 2026-06-08 14:48 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
Remove the code in kvm_cpuid() that dynamically updates the Xen TSC
info CPUID leaf at runtime. This code was updating the wrong sub-leaf
anyway (0x40000x03/2 EAX is the *host* TSC frequency per the Xen ABI,
not the guest frequency which belongs in 0x40000x03/0 ECX).
Userspace now has all the information it needs to populate the Xen TSC
info leaves (and the generic 0x40000010 timing leaf) at vCPU setup time:
- KVM_GET_CLOCK_GUEST returns the pvclock_vcpu_time_info structure
containing tsc_to_system_mul and tsc_shift (Xen leaf index 1)
- KVM_VCPU_TSC_SCALE returns the effective TSC and bus
frequencies in kHz (Xen leaf index 2, and 0x40000010)
- KVM_VCPU_TSC_SCALE returns the raw hardware scaling ratio for
precise arithmetic (VMClock)
This eliminates the last instance of KVM modifying guest CPUID entries
at runtime for timing information.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
arch/x86/kvm/cpuid.c | 16 ----------------
arch/x86/kvm/xen.h | 13 -------------
2 files changed, 29 deletions(-)
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 621d950ec692..826637a0b72d 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -2117,22 +2117,6 @@ bool kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx,
} else if (function == 0x80000007) {
if (kvm_hv_invtsc_suppressed(vcpu))
*edx &= ~feature_bit(CONSTANT_TSC);
- } else if (IS_ENABLED(CONFIG_KVM_XEN) &&
- kvm_xen_is_tsc_leaf(vcpu, function)) {
- /*
- * Update guest TSC frequency information if necessary.
- * Ignore failures, there is no sane value that can be
- * provided if KVM can't get the TSC frequency.
- */
- if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu))
- kvm_guest_time_update(vcpu);
-
- if (index == 1) {
- *ecx = vcpu->arch.pvclock_tsc_mul;
- *edx = vcpu->arch.pvclock_tsc_shift;
- } else if (index == 2) {
- *eax = div_u64(vcpu->arch.hw_tsc_hz, 1000);
- }
}
} else {
*eax = *ebx = *ecx = *edx = 0;
diff --git a/arch/x86/kvm/xen.h b/arch/x86/kvm/xen.h
index 59e6128a7bd3..f372855857a8 100644
--- a/arch/x86/kvm/xen.h
+++ b/arch/x86/kvm/xen.h
@@ -50,14 +50,6 @@ static inline void kvm_xen_sw_enable_lapic(struct kvm_vcpu *vcpu)
kvm_xen_inject_vcpu_vector(vcpu);
}
-static inline bool kvm_xen_is_tsc_leaf(struct kvm_vcpu *vcpu, u32 function)
-{
- return static_branch_unlikely(&kvm_xen_enabled.key) &&
- vcpu->arch.xen.cpuid.base &&
- function <= vcpu->arch.xen.cpuid.limit &&
- function == (vcpu->arch.xen.cpuid.base | XEN_CPUID_LEAF(3));
-}
-
static inline bool kvm_xen_msr_enabled(struct kvm *kvm)
{
return static_branch_unlikely(&kvm_xen_enabled.key) &&
@@ -177,11 +169,6 @@ static inline bool kvm_xen_timer_enabled(struct kvm_vcpu *vcpu)
{
return false;
}
-
-static inline bool kvm_xen_is_tsc_leaf(struct kvm_vcpu *vcpu, u32 function)
-{
- return false;
-}
#endif
int kvm_xen_hypercall(struct kvm_vcpu *vcpu);
--
2.54.0
^ permalink raw reply related
* [PATCH v5 01/34] KVM: x86/xen: Do not corrupt KVM clock in kvm_xen_shared_info_init()
From: David Woodhouse @ 2026-06-08 14:47 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
The KVM clock is an interesting thing. It is defined as "nanoseconds
since the guest was created", but in practice it runs at two *different*
rates — or three different rates, if you count implementation bugs.
Definition A is that it runs synchronously with the CLOCK_MONOTONIC_RAW
of the host, with a delta of kvm->arch.kvmclock_offset.
But that version doesn't actually get used in the common case, where the
host has a reliable TSC and the guest TSCs are all running at the same
rate and in sync with each other, and kvm->arch.use_master_clock is set.
In that common case, definition B is used: There is a reference point in
time at kvm->arch.master_kernel_ns (again a CLOCK_MONOTONIC_RAW time),
and a corresponding host TSC value kvm->arch.master_cycle_now. This
fixed point in time is converted to guest units (the time offset by
kvmclock_offset and the TSC Value scaled and offset to be a guest TSC
value) and advertised to the guest in the pvclock structure. While in
this 'use_master_clock' mode, the fixed point in time never needs to be
changed, and the clock runs precisely in time with the guest TSC, at the
rate advertised in the pvclock structure.
The third definition C is implemented in kvm_get_wall_clock_epoch() and
__get_kvmclock(), using the master_cycle_now and master_kernel_ns fields
but converting the *host* TSC cycles directly to a value in nanoseconds
instead of scaling via the guest TSC.
One might naïvely think that all three definitions are identical, since
CLOCK_MONOTONIC_RAW is not skewed by NTP frequency corrections; all
three are just the result of counting the host TSC at a known frequency,
or the scaled guest TSC at a known precise fraction of the host's
frequency. The problem is with arithmetic precision, and the way that
frequency scaling is done in a division-free way by multiplying by a
scale factor, then shifting right. In practice, all three ways of
calculating the KVM clock will suffer a systemic drift from each other.
Eventually, definition C should just be eliminated. Commit 451a707813ae
("KVM: x86/xen: improve accuracy of Xen timers") worked around it for
the specific case of Xen timers, which are defined in terms of the KVM
clock and suffered from a continually increasing error in timer expiry
times. That commit notes that get_kvmclock_ns() is non-trivial to fix
and says "I'll come back to that", which remains true.
Definitions A and B do need to coexist, the former to handle the case
where the host or guest TSC is suboptimally configured. But KVM should
be more careful about switching between them, and the discontinuity in
guest time which could result.
In particular, KVM_REQ_MASTERCLOCK_UPDATE will take a new snapshot of
time as the reference in master_kernel_ns and master_cycle_now, yanking
the guest's clock back to match definition A at that moment.
When invoked from in 'use_master_clock' mode, kvm_update_masterclock()
should probably *adjust* kvm->arch.kvmclock_offset to account for the
drift, instead of yanking the clock back to definition A. But in the
meantime there are a bunch of places where it just doesn't need to be
invoked at all.
To start with: there is no need to do such an update when a Xen guest
populates the shared_info page. This seems to have been a hangover from
the very first implementation of shared_info which automatically
populated the vcpu_info structures at their default locations, but even
then it should just have raised KVM_REQ_CLOCK_UPDATE on each vCPU
instead of using KVM_REQ_MASTERCLOCK_UPDATE. And now that userspace is
expected to explicitly set the vcpu_info even in its default locations,
there's not even any need for that either.
Fixes: 629b5348841a ("KVM: x86/xen: update wallclock region")
Reviewed-by: Paul Durrant <paul@xen.org>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
arch/x86/kvm/xen.c | 2 --
1 file changed, 2 deletions(-)
diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c
index 91fd3673c09a..82e34edbfdbd 100644
--- a/arch/x86/kvm/xen.c
+++ b/arch/x86/kvm/xen.c
@@ -98,8 +98,6 @@ static int kvm_xen_shared_info_init(struct kvm *kvm)
wc->version = wc_version + 1;
read_unlock_irq(&gpc->lock);
- kvm_make_all_cpus_request(kvm, KVM_REQ_MASTERCLOCK_UPDATE);
-
out:
srcu_read_unlock(&kvm->srcu, idx);
return ret;
--
2.54.0
^ permalink raw reply related
* [PATCH v5 15/34] KVM: x86: Restructure kvm_guest_time_update() for TSC upscaling
From: David Woodhouse @ 2026-06-08 14:47 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
Restructure kvm_guest_time_update() so that kernel_ns/host_tsc are
always "now" when doing TSC catchup, then swap in the master clock
reference values afterward for the hv_clock.
This makes the TSC upscaling code considerably simpler: the catchup
adjustment is computed as the delta between what the guest TSC *should*
be at "now" and what it actually is, rather than mixing "now" and
"master clock reference" timestamps.
The seqcount loop now also contains the kvm_get_time_and_clockread()
call (matching get_kvmclock's pattern), with the same WARN for
unexpected failure.
Based on a suggestion by Sean Christopherson.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
arch/x86/kvm/x86.c | 74 +++++++++++++++++++++++++++++++++-------------
1 file changed, 53 insertions(+), 21 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 8aae22401046..92e32d720523 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -3363,46 +3363,63 @@ static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock,
int kvm_guest_time_update(struct kvm_vcpu *v)
{
struct pvclock_vcpu_time_info hv_clock = {};
- unsigned long flags;
u64 tgt_tsc_hz;
unsigned seq;
struct kvm_vcpu_arch *vcpu = &v->arch;
struct kvm_arch *ka = &v->kvm->arch;
s64 kernel_ns;
u64 tsc_timestamp, host_tsc;
+ u64 master_host_tsc = 0;
+ s64 master_kernel_ns = 0;
bool use_master_clock;
- kernel_ns = 0;
- host_tsc = 0;
-
/*
* If the host uses TSC clock, then passthrough TSC as stable
* to the guest.
*/
do {
seq = read_seqcount_begin(&ka->pvclock_sc);
+
use_master_clock = ka->use_master_clock;
- if (use_master_clock) {
- host_tsc = ka->master_cycle_now;
- kernel_ns = ka->master_kernel_ns;
- }
+
+ /*
+ * The TSC read and the call to get_cpu_tsc_khz() must happen
+ * on the same CPU.
+ */
+ get_cpu();
+
+ tgt_tsc_hz = (u64)get_cpu_tsc_khz() * 1000;
+
+#ifdef CONFIG_X86_64
+ if (use_master_clock &&
+ !kvm_get_time_and_clockread(&kernel_ns, &host_tsc) &&
+ !read_seqcount_retry(&ka->pvclock_sc, seq))
+ use_master_clock = false;
+#endif
+
+ put_cpu();
+
+ if (!use_master_clock)
+ break;
+
+ master_host_tsc = ka->master_cycle_now;
+ master_kernel_ns = ka->master_kernel_ns;
} while (read_seqcount_retry(&ka->pvclock_sc, seq));
- /* Keep irq disabled to prevent changes to the clock */
- local_irq_save(flags);
- tgt_tsc_hz = (u64)get_cpu_tsc_khz() * 1000;
if (unlikely(tgt_tsc_hz == 0)) {
- local_irq_restore(flags);
kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
return 1;
}
+
if (!use_master_clock) {
+ unsigned long flags;
+
+ local_irq_save(flags);
host_tsc = rdtsc();
kernel_ns = get_kvmclock_base_ns();
+ local_irq_restore(flags);
}
- tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
-
/*
* We may have to catch up the TSC to match elapsed wall clock
* time for two reasons, even if kvmclock is used.
@@ -3411,17 +3428,32 @@ int kvm_guest_time_update(struct kvm_vcpu *v)
* entry to avoid unknown leaps of TSC even when running
* again on the same CPU. This may cause apparent elapsed
* time to disappear, and the guest to stand still or run
- * very slowly.
+ * very slowly.
*/
if (vcpu->tsc_catchup) {
- u64 tsc = compute_guest_tsc(v, kernel_ns);
- if (tsc > tsc_timestamp) {
- adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
- tsc_timestamp = tsc;
- }
+ s64 adjustment;
+
+ /*
+ * Calculate the delta between what the guest TSC *should* be
+ * and what it actually is according to kvm_read_l1_tsc().
+ */
+ adjustment = compute_guest_tsc(v, kernel_ns) -
+ kvm_read_l1_tsc(v, host_tsc);
+ if (adjustment > 0)
+ adjust_tsc_offset_guest(v, adjustment);
}
- local_irq_restore(flags);
+ /*
+ * Now that TSC upscaling is out of the way, the remaining calculations
+ * are all relative to the reference time that's placed in hv_clock.
+ * If the master clock is NOT in use, the reference time is "now". If
+ * master clock is in use, the reference time comes from there.
+ */
+ if (use_master_clock) {
+ host_tsc = master_host_tsc;
+ kernel_ns = master_kernel_ns;
+ }
+ tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
/* With all the info we got, fill in the values */
--
2.54.0
^ permalink raw reply related
* [PATCH v5 05/34] KVM: selftests: Add KVM/PV clock selftest to prove timer correction
From: David Woodhouse @ 2026-06-08 14:47 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: Jack Allister <jalliste@amazon.com>
A VM's KVM/PV clock has an inherent relationship to its TSC. When either
the host system live-updates or the VM is live-migrated this pairing of
the two clock sources should stay the same. In reality this is not the
case without some correction taking place.
The KVM_GET_CLOCK_GUEST/KVM_SET_CLOCK_GUEST ioctls can be used to
perform a correction on the PVTI (PV time information) structure held by
KVM to effectively fix up the kvmclock_offset prior to the guest VM
resuming in either a live-update/migration scenario.
This test proves that without the necessary fixup there is a perceived
change in the guest TSC and KVM/PV clock relationship before and after a
simulated LU/LM takes place, and that the correction eliminates it.
The test:
1. Snapshots the PVTI at boot (PVTI0).
2. Induces a change in PVTI data (KVM_REQ_MASTERCLOCK_UPDATE).
3. Snapshots the PVTI after the change (PVTI1).
4. Requests correction via KVM_SET_CLOCK_GUEST using PVTI0.
5. Snapshots the PVTI after correction (PVTI2).
Then samples the TSC at a single point in time and calculates the KVM
clock using each PVTI snapshot. The corrected clock should match the
boot clock to within ±1ns.
The test enumerates multiple TSC frequencies from 1GHz to 5GHz at 500MHz
steps, crossing the 32-bit boundary, to exercise the scaling path at
various ratios. The sleep duration between snapshots is configurable via
the -s/--sleep command line option.
Co-developed-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Jack Allister <jalliste@amazon.com>
Reviewed-by: Paul Durrant <paul@xen.org>
Cc: Dongli Zhang <dongli.zhang@oracle.com>
---
tools/testing/selftests/kvm/Makefile.kvm | 1 +
.../testing/selftests/kvm/x86/pvclock_test.c | 440 ++++++++++++++++++
2 files changed, 441 insertions(+)
create mode 100644 tools/testing/selftests/kvm/x86/pvclock_test.c
diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
index 9118a5a51b89..fb935ae3bf38 100644
--- a/tools/testing/selftests/kvm/Makefile.kvm
+++ b/tools/testing/selftests/kvm/Makefile.kvm
@@ -105,6 +105,7 @@ TEST_GEN_PROGS_x86 += x86/pmu_counters_test
TEST_GEN_PROGS_x86 += x86/pmu_event_filter_test
TEST_GEN_PROGS_x86 += x86/private_mem_conversions_test
TEST_GEN_PROGS_x86 += x86/private_mem_kvm_exits_test
+TEST_GEN_PROGS_x86 += x86/pvclock_test
TEST_GEN_PROGS_x86 += x86/set_boot_cpu_id
TEST_GEN_PROGS_x86 += x86/set_sregs_test
TEST_GEN_PROGS_x86 += x86/smaller_maxphyaddr_emulation_test
diff --git a/tools/testing/selftests/kvm/x86/pvclock_test.c b/tools/testing/selftests/kvm/x86/pvclock_test.c
new file mode 100644
index 000000000000..aecd62fc8a93
--- /dev/null
+++ b/tools/testing/selftests/kvm/x86/pvclock_test.c
@@ -0,0 +1,440 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright © Amazon.com, Inc. or its affiliates.
+ *
+ * Tests for pvclock API
+ * KVM_SET_CLOCK_GUEST/KVM_GET_CLOCK_GUEST
+ */
+#include <getopt.h>
+#include <stdint.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "test_util.h"
+#include "kvm_util.h"
+#include "processor.h"
+#include "apic.h"
+
+#include <asm/pvclock-abi.h>
+
+/*
+ * Reproduce the pvclock calculation the guest uses to convert TSC to
+ * nanoseconds. This must match the kernel's __pvclock_read_cycles().
+ */
+static inline uint64_t pvclock_scale_delta(uint64_t delta, uint32_t mul,
+ int8_t shift)
+{
+ if (shift < 0)
+ delta >>= -shift;
+ else
+ delta <<= shift;
+ return ((__uint128_t)delta * mul) >> 32;
+}
+
+static inline uint64_t pvclock_read_cycles(struct pvclock_vcpu_time_info *src,
+ uint64_t tsc)
+{
+ uint64_t delta = tsc - src->tsc_timestamp;
+
+ return src->system_time + pvclock_scale_delta(delta,
+ src->tsc_to_system_mul,
+ src->tsc_shift);
+}
+
+static inline void pvti_snapshot(struct pvclock_vcpu_time_info *dst,
+ volatile struct pvclock_vcpu_time_info *src)
+{
+ uint32_t version;
+
+ do {
+ version = src->version;
+ __asm__ __volatile__("" ::: "memory");
+ *dst = *src;
+ __asm__ __volatile__("" ::: "memory");
+ } while ((src->version & 1) || src->version != version);
+}
+
+enum {
+ STAGE_FIRST_BOOT,
+ STAGE_UNCORRECTED,
+ STAGE_CORRECTED
+};
+
+#define KVMCLOCK_GPA 0xc0000000ull
+#define KVMCLOCK_SIZE sizeof(struct pvclock_vcpu_time_info)
+
+static void trigger_pvti_update(void)
+{
+ /*
+ * Toggle between KVM's old and new system time methods to coerce KVM
+ * into updating the fields in the PV time info struct.
+ */
+ wrmsr(MSR_KVM_SYSTEM_TIME, KVMCLOCK_GPA | KVM_MSR_ENABLED);
+ wrmsr(MSR_KVM_SYSTEM_TIME_NEW, KVMCLOCK_GPA | KVM_MSR_ENABLED);
+}
+
+static void guest_code(void)
+{
+ struct pvclock_vcpu_time_info *pvti =
+ (void *)(unsigned long)KVMCLOCK_GPA;
+ struct pvclock_vcpu_time_info pvti_boot;
+ struct pvclock_vcpu_time_info pvti_uncorrected;
+ struct pvclock_vcpu_time_info pvti_corrected;
+ uint64_t tsc_guest;
+ uint64_t clk_boot, clk_uncorrected, clk_corrected;
+ int64_t delta_corrected;
+
+ /* Set up kvmclock and snapshot the initial pvclock parameters. */
+ wrmsr(MSR_KVM_SYSTEM_TIME_NEW, KVMCLOCK_GPA | KVM_MSR_ENABLED);
+ pvti_snapshot(&pvti_boot, pvti);
+ GUEST_SYNC(STAGE_FIRST_BOOT);
+
+ /*
+ * Trigger an update of the PVTI. Calculating the KVM clock using this
+ * updated structure will show a delta from the original.
+ */
+ trigger_pvti_update();
+ pvti_snapshot(&pvti_uncorrected, pvti);
+ GUEST_SYNC(STAGE_UNCORRECTED);
+
+ /*
+ * Snapshot the corrected time (the host does KVM_SET_CLOCK_GUEST when
+ * handling STAGE_UNCORRECTED).
+ */
+ pvti_snapshot(&pvti_corrected, pvti);
+
+ /*
+ * Sample the TSC at a single point in time, then calculate the
+ * effective KVM clock using the PVTI from each stage. Verify that the
+ * corrected clock matches the boot clock to within ±2ns.
+ */
+ tsc_guest = rdtsc();
+
+ clk_boot = pvclock_read_cycles(&pvti_boot, tsc_guest);
+ clk_uncorrected = pvclock_read_cycles(&pvti_uncorrected, tsc_guest);
+ clk_corrected = pvclock_read_cycles(&pvti_corrected, tsc_guest);
+
+ delta_corrected = clk_boot - clk_corrected;
+
+ __GUEST_ASSERT(delta_corrected >= -2 && delta_corrected <= 2,
+ "corrected delta %ld out of range (boot=%lu uncorrected=%lu corrected=%lu)",
+ delta_corrected, clk_boot, clk_uncorrected, clk_corrected);
+
+ GUEST_SYNC(STAGE_CORRECTED);
+}
+
+static void run_test(struct kvm_vm *vm, struct kvm_vcpu *vcpu,
+ unsigned int sleep_sec)
+{
+ struct pvclock_vcpu_time_info pvti_before;
+ struct ucall uc;
+
+ for (;;) {
+ vcpu_run(vcpu);
+ TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
+
+ switch (get_ucall(vcpu, &uc)) {
+ case UCALL_ABORT:
+ REPORT_GUEST_ASSERT(uc);
+ break;
+ case UCALL_SYNC:
+ break;
+ default:
+ TEST_FAIL("Unexpected ucall");
+ }
+
+ switch (uc.args[1]) {
+ case STAGE_FIRST_BOOT:
+ /* Save the pvclock parameters before the update. */
+ vcpu_ioctl(vcpu, KVM_GET_CLOCK_GUEST, &pvti_before);
+
+ /* Sleep to let the clocks diverge. */
+ sleep(sleep_sec);
+ break;
+
+ case STAGE_UNCORRECTED:
+ /* Restore the original pvclock parameters. */
+ vcpu_ioctl(vcpu, KVM_SET_CLOCK_GUEST, &pvti_before);
+ break;
+
+ case STAGE_CORRECTED:
+ /* Guest verified the delta in-guest. */
+ return;
+
+ default:
+ TEST_FAIL("Unknown stage %lu", uc.args[1]);
+ }
+ }
+}
+
+static void configure_pvclock(struct kvm_vm *vm)
+{
+ unsigned int nr_pages;
+
+ nr_pages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, getpagesize());
+ vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
+ KVMCLOCK_GPA, 1, nr_pages, 0);
+ virt_map(vm, KVMCLOCK_GPA, KVMCLOCK_GPA, nr_pages);
+}
+
+static void run_at_frequency(uint64_t tsc_khz, unsigned int sleep_sec)
+{
+ struct kvm_vcpu *vcpu;
+ struct kvm_vm *vm;
+
+ pr_info("Testing at TSC frequency %lu kHz\n", tsc_khz);
+ vm = vm_create_with_one_vcpu(&vcpu, guest_code);
+ configure_pvclock(vm);
+ vcpu_ioctl(vcpu, KVM_SET_TSC_KHZ, (void *)tsc_khz);
+ run_test(vm, vcpu, sleep_sec);
+ kvm_vm_free(vm);
+}
+
+static void test_tsc_stable_bit(void);
+static void test_clock_guest_with_offsets(void);
+
+static void usage(const char *name)
+{
+ printf("Usage: %s [options]\n"
+ " -s, --sleep SEC sleep duration between snapshots (default: 2)\n"
+ " -h, --help show this help\n", name);
+}
+
+int main(int argc, char *argv[])
+{
+ static const struct option long_opts[] = {
+ { "sleep", required_argument, NULL, 's' },
+ { "help", no_argument, NULL, 'h' },
+ { NULL, 0, NULL, 0 },
+ };
+ unsigned int sleep_sec = 2;
+ struct kvm_vcpu *vcpu;
+ struct kvm_vm *vm;
+ uint64_t host_khz;
+ uint64_t freq;
+ int opt;
+
+ while ((opt = getopt_long(argc, argv, "s:h", long_opts, NULL)) != -1) {
+ switch (opt) {
+ case 's':
+ sleep_sec = atoi(optarg);
+ break;
+ case 'h':
+ default:
+ usage(argv[0]);
+ return opt == 'h' ? 0 : 1;
+ }
+ }
+
+ TEST_REQUIRE(sys_clocksource_is_based_on_tsc());
+ TEST_REQUIRE(kvm_has_cap(KVM_CAP_TSC_CONTROL));
+
+ vm = vm_create_with_one_vcpu(&vcpu, guest_code);
+ configure_pvclock(vm);
+
+ /* Check KVM_GET_CLOCK_GUEST is supported */
+ {
+ struct pvclock_vcpu_time_info tmp;
+ int ret = __vcpu_ioctl(vcpu, KVM_GET_CLOCK_GUEST, &tmp);
+ TEST_REQUIRE(ret != -1 || errno != ENOTTY);
+ }
+
+ /* First run at native frequency (no scaling). */
+ run_test(vm, vcpu, sleep_sec);
+
+ /*
+ * Then enumerate a range of TSC frequencies crossing the 32-bit
+ * boundary, to exercise the scaling path at various ratios.
+ */
+ host_khz = __vcpu_ioctl(vcpu, KVM_GET_TSC_KHZ, NULL);
+ kvm_vm_free(vm);
+
+ for (freq = 1000000; freq <= 5000000; freq += 500000) {
+ if (freq == host_khz)
+ continue;
+ run_at_frequency(freq, sleep_sec);
+ }
+
+ test_tsc_stable_bit();
+ test_clock_guest_with_offsets();
+
+ return 0;
+}
+
+static void guest_code_stable_bit(void)
+{
+ uint32_t apic_id = GET_APIC_ID_FIELD(xapic_read_reg(APIC_ID));
+ uint64_t gpa = KVMCLOCK_GPA + apic_id * sizeof(struct pvclock_vcpu_time_info);
+
+ wrmsr(MSR_KVM_SYSTEM_TIME_NEW, gpa | KVM_MSR_ENABLED);
+ GUEST_SYNC(0);
+ GUEST_SYNC(0);
+ GUEST_SYNC(0);
+}
+
+static void set_tsc_offset(struct kvm_vcpu *vcpu, uint64_t offset)
+{
+ struct kvm_device_attr attr = {
+ .group = KVM_VCPU_TSC_CTRL,
+ .attr = KVM_VCPU_TSC_OFFSET,
+ .addr = (__u64)(uintptr_t)&offset,
+ };
+ vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &attr);
+}
+
+static void run_vcpu_once(struct kvm_vcpu *vcpu)
+{
+ struct ucall uc;
+
+ vcpu_run(vcpu);
+ TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
+ switch (get_ucall(vcpu, &uc)) {
+ case UCALL_ABORT:
+ REPORT_GUEST_ASSERT(uc);
+ break;
+ case UCALL_SYNC:
+ break;
+ default:
+ TEST_FAIL("Unexpected ucall");
+ }
+}
+
+static void test_tsc_stable_bit(void)
+{
+ struct pvclock_vcpu_time_info pvti;
+ struct kvm_vcpu *vcpus[2];
+ struct kvm_vm *vm;
+ int ret;
+
+ pr_info("Testing PVCLOCK_TSC_STABLE_BIT with matched/unmatched TSCs\n");
+
+ vm = vm_create_with_vcpus(2, guest_code_stable_bit, vcpus);
+ configure_pvclock(vm);
+
+ /*
+ * Case 1: All TSCs matched (same frequency and offset).
+ * Master clock should be active, PVCLOCK_TSC_STABLE_BIT set.
+ */
+ run_vcpu_once(vcpus[0]);
+
+ ret = __vcpu_ioctl(vcpus[0], KVM_GET_CLOCK_GUEST, &pvti);
+ TEST_ASSERT(!ret, "GET_CLOCK_GUEST should succeed with matched TSCs");
+ TEST_ASSERT(pvti.flags & PVCLOCK_TSC_STABLE_BIT,
+ "PVCLOCK_TSC_STABLE_BIT should be set with matched TSCs");
+
+ /*
+ * Case 2: Different TSC offset, same frequency.
+ * Master clock should still be active (frequency matches), but
+ * PVCLOCK_TSC_STABLE_BIT should be cleared (offsets differ).
+ */
+ set_tsc_offset(vcpus[1], 12345678);
+ run_vcpu_once(vcpus[1]);
+ run_vcpu_once(vcpus[0]);
+
+ ret = __vcpu_ioctl(vcpus[0], KVM_GET_CLOCK_GUEST, &pvti);
+ if (ret) {
+ /* Master clock disabled by offset mismatch — old kernel */
+ pr_info(" Skipping offset tests (master clock requires matched offsets)\n");
+ goto out_stable;
+ }
+ TEST_ASSERT(!(pvti.flags & PVCLOCK_TSC_STABLE_BIT),
+ "PVCLOCK_TSC_STABLE_BIT should be clear with offset-mismatched TSCs");
+
+ /*
+ * Case 3: Different TSC frequency.
+ * Master clock should be disabled entirely.
+ */
+ vcpu_ioctl(vcpus[1], KVM_SET_TSC_KHZ,
+ (void *)(unsigned long)(__vcpu_ioctl(vcpus[1], KVM_GET_TSC_KHZ, NULL) / 2));
+ /* Write TSC to trigger kvm_synchronize_tsc / kvm_track_tsc_matching */
+ vcpu_set_msr(vcpus[1], MSR_IA32_TSC, 0);
+ run_vcpu_once(vcpus[1]);
+
+ ret = __vcpu_ioctl(vcpus[0], KVM_GET_CLOCK_GUEST, &pvti);
+ TEST_ASSERT(ret && errno == EINVAL,
+ "GET_CLOCK_GUEST should fail with frequency-mismatched TSCs, got %d (errno %d)",
+ ret, errno);
+
+out_stable:
+ kvm_vm_free(vm);
+}
+
+static void test_clock_guest_with_offsets(void)
+{
+ struct pvclock_vcpu_time_info pvti0, pvti1, pvti1_after;
+ struct kvm_vcpu *vcpus[2];
+ struct kvm_vm *vm;
+ int64_t delta;
+ int ret;
+
+ pr_info("Testing KVM_[GS]ET_CLOCK_GUEST with different TSC offsets\n");
+
+ vm = vm_create_with_vcpus(2, guest_code_stable_bit, vcpus);
+ configure_pvclock(vm);
+
+ /* Set different TSC offsets on the two vCPUs */
+ set_tsc_offset(vcpus[0], 0);
+ set_tsc_offset(vcpus[1], 1000000000ull);
+
+ /* Run both to establish kvmclock */
+ run_vcpu_once(vcpus[0]);
+ run_vcpu_once(vcpus[1]);
+
+ /* GET_CLOCK_GUEST on both — should succeed (master clock active) */
+ ret = __vcpu_ioctl(vcpus[0], KVM_GET_CLOCK_GUEST, &pvti0);
+ if (ret) {
+ pr_info(" Skipping (master clock requires matched offsets on this kernel)\n");
+ kvm_vm_free(vm);
+ return;
+ }
+ ret = __vcpu_ioctl(vcpus[1], KVM_GET_CLOCK_GUEST, &pvti1);
+ TEST_ASSERT(!ret, "GET_CLOCK_GUEST on vcpu1 failed");
+
+ /* The tsc_timestamps should differ (different offsets) */
+ TEST_ASSERT(pvti0.tsc_timestamp != pvti1.tsc_timestamp,
+ "tsc_timestamps should differ with different offsets");
+
+ /* Sleep to let time elapse, then restore vcpu0's clock */
+ sleep(1);
+ vcpu_ioctl(vcpus[0], KVM_SET_CLOCK_GUEST, &pvti0);
+
+ /* Run vcpu0 to process the clock update */
+ run_vcpu_once(vcpus[0]);
+
+ /* GET_CLOCK_GUEST on vcpu1 — should reflect the correction */
+ ret = __vcpu_ioctl(vcpus[1], KVM_GET_CLOCK_GUEST, &pvti1_after);
+ TEST_ASSERT(!ret, "GET_CLOCK_GUEST on vcpu1 after SET failed");
+
+ /*
+ * After SET on vcpu0, verify the correction worked by getting
+ * the clock on vcpu0 again. The mul/shift should be the same,
+ * and computing kvmclock at the same TSC should give the same
+ * result as the original (within ±2ns).
+ */
+ {
+ struct pvclock_vcpu_time_info pvti0_after;
+ uint64_t tsc_now, clk_from_old, clk_from_new;
+
+ ret = __vcpu_ioctl(vcpus[0], KVM_GET_CLOCK_GUEST, &pvti0_after);
+ TEST_ASSERT(!ret, "GET_CLOCK_GUEST on vcpu0 after SET failed");
+
+ tsc_now = pvti0_after.tsc_timestamp;
+ clk_from_old = pvclock_read_cycles(&pvti0, tsc_now);
+ clk_from_new = pvclock_read_cycles(&pvti0_after, tsc_now);
+
+ delta = (int64_t)clk_from_new - (int64_t)clk_from_old;
+ TEST_ASSERT(delta >= -2 && delta <= 2,
+ "clock correction delta should be <=2ns, got %ld ns",
+ delta);
+ }
+
+ /*
+ * Also verify that vcpu1's clock is still accessible (master
+ * clock still active with different offsets).
+ */
+ ret = __vcpu_ioctl(vcpus[1], KVM_GET_CLOCK_GUEST, &pvti1_after);
+ TEST_ASSERT(!ret, "GET_CLOCK_GUEST on vcpu1 after SET failed");
+
+ kvm_vm_free(vm);
+}
--
2.54.0
^ permalink raw reply related
* [PATCH v5 02/34] KVM: x86: Improve accuracy of KVM clock when TSC scaling is in force
From: David Woodhouse @ 2026-06-08 14:47 UTC (permalink / raw)
To: Paolo Bonzini, Jonathan Corbet, Shuah Khan, Sean Christopherson,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen, x86,
H. Peter Anvin, Vitaly Kuznetsov, Juergen Gross, Boris Ostrovsky,
David Woodhouse, Paul Durrant, Jonathan Cameron, Sascha Bischoff,
Marc Zyngier, Joey Gouly, Jack Allister, Dongli Zhang, joe.jin,
kvm, linux-doc, linux-kernel, xen-devel, linux-kselftest
In-Reply-To: <20260608145455.89187-1-dwmw2@infradead.org>
From: David Woodhouse <dwmw@amazon.co.uk>
The kvm_guest_time_update() function scales the host TSC frequency to
the guest's using kvm_scale_tsc() and the v->arch.l1_tsc_scaling_ratio
scaling ratio previously calculated for that vCPU. Then calculates the
scaling factors for the KVM clock itself based on that guest TSC
frequency.
However, it uses kHz as the unit when scaling, and then multiplies by
1000 only at the end.
With a host TSC frequency of 3000MHz and a guest set to 2500MHz, the
result of kvm_scale_tsc() will actually come out at 2,499,999kHz. So
the KVM clock advertised to the guest is based on a frequency of
2,499,999,000 Hz.
By using Hz as the unit from the beginning, the KVM clock would be based
on a more accurate frequency of 2,499,999,999 Hz in this example.
Use u64 for the hw_tsc_hz field since an unsigned int would overflow for
TSC frequencies above 4GHz. Use div_u64() for the Xen CPUID leaf to
play nice with 32-bit kernels.
Fixes: 78db6a503796 ("KVM: x86: rewrite handling of scaled TSC for kvmclock")
Reviewed-by: Paul Durrant <paul@xen.org>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
arch/x86/include/asm/kvm_host.h | 2 +-
arch/x86/kvm/cpuid.c | 2 +-
arch/x86/kvm/x86.c | 17 +++++++++--------
3 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index c470e40a00aa..37264212c7df 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -950,7 +950,7 @@ struct kvm_vcpu_arch {
gpa_t time;
s8 pvclock_tsc_shift;
u32 pvclock_tsc_mul;
- unsigned int hw_tsc_khz;
+ u64 hw_tsc_hz;
struct gfn_to_pfn_cache pv_time;
/* set guest stopped flag in pvclock flags field */
bool pvclock_set_guest_stopped_request;
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index e69156b54cff..621d950ec692 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -2131,7 +2131,7 @@ bool kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx,
*ecx = vcpu->arch.pvclock_tsc_mul;
*edx = vcpu->arch.pvclock_tsc_shift;
} else if (index == 2) {
- *eax = vcpu->arch.hw_tsc_khz;
+ *eax = div_u64(vcpu->arch.hw_tsc_hz, 1000);
}
}
} else {
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 0a1b63c63d1a..d9ef165df6a1 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -3314,7 +3314,8 @@ static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock,
int kvm_guest_time_update(struct kvm_vcpu *v)
{
struct pvclock_vcpu_time_info hv_clock = {};
- unsigned long flags, tgt_tsc_khz;
+ unsigned long flags;
+ u64 tgt_tsc_hz;
unsigned seq;
struct kvm_vcpu_arch *vcpu = &v->arch;
struct kvm_arch *ka = &v->kvm->arch;
@@ -3340,8 +3341,8 @@ int kvm_guest_time_update(struct kvm_vcpu *v)
/* Keep irq disabled to prevent changes to the clock */
local_irq_save(flags);
- tgt_tsc_khz = get_cpu_tsc_khz();
- if (unlikely(tgt_tsc_khz == 0)) {
+ tgt_tsc_hz = (u64)get_cpu_tsc_khz() * 1000;
+ if (unlikely(tgt_tsc_hz == 0)) {
local_irq_restore(flags);
kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
return 1;
@@ -3376,16 +3377,16 @@ int kvm_guest_time_update(struct kvm_vcpu *v)
/* With all the info we got, fill in the values */
if (kvm_caps.has_tsc_control) {
- tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz,
+ tgt_tsc_hz = kvm_scale_tsc(tgt_tsc_hz,
v->arch.l1_tsc_scaling_ratio);
- tgt_tsc_khz = tgt_tsc_khz ? : 1;
+ tgt_tsc_hz = tgt_tsc_hz ? : 1;
}
- if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
- kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
+ if (unlikely(vcpu->hw_tsc_hz != tgt_tsc_hz)) {
+ kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_hz,
&vcpu->pvclock_tsc_shift,
&vcpu->pvclock_tsc_mul);
- vcpu->hw_tsc_khz = tgt_tsc_khz;
+ vcpu->hw_tsc_hz = tgt_tsc_hz;
}
hv_clock.tsc_shift = vcpu->pvclock_tsc_shift;
--
2.54.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox