[RFC PATCH v2 8/9] igb: Add dirty page tracking statistics

["Cédric Le Goater" <clg@redhat.com>]

Add a GET_STATS command (cmd 7) to the migration DVSEC for monitoring
dirty page tracking and DMA activity per VF. The device DMA-writes an
igb_mig_stats_resp struct to the shared buffer.

The dma_writes counter is also reported in the dirty query buffer so
the driver gets it alongside the bitmap without an extra config read.

Counters are reset on first DIRTY_ENABLE or device reset, so the
driver can read final values after DIRTY_DISABLE.

AI-used-for: code (prototype)
Signed-off-by: Cédric Le Goater <clg@redhat.com>
---
 hw/net/igb_core.h      |  1 +
 hw/net/igb_migration.h | 22 ++++++++++++++
 hw/net/igb_migration.c | 65 ++++++++++++++++++++++++++++++++++++++++--
 hw/net/trace-events    |  2 +-
 4 files changed, 86 insertions(+), 4 deletions(-)

diff --git a/hw/net/igb_core.h b/hw/net/igb_core.h
index 7d6f44b7ffe9..380cc88b7959 100644
--- a/hw/net/igb_core.h
+++ b/hw/net/igb_core.h
@@ -103,6 +103,7 @@ struct IGBCore {
     int64_t timadj;
 
     IGBVfDirtyState vf_dirty[IGB_MAX_VF_FUNCTIONS];
+    IgbVfMigStats vf_mig_stats[IGB_MAX_VF_FUNCTIONS];
 };
 
 void
diff --git a/hw/net/igb_migration.h b/hw/net/igb_migration.h
index 2bb9a0ed36ce..b334bb8f78ba 100644
--- a/hw/net/igb_migration.h
+++ b/hw/net/igb_migration.h
@@ -64,6 +64,7 @@
 #define IGB_MIG_CMD_DIRTY_ENABLE        4
 #define IGB_MIG_CMD_DIRTY_DISABLE       5
 #define IGB_MIG_CMD_DIRTY_QUERY         6
+#define IGB_MIG_CMD_GET_STATS           7
 
 /* STATUS register: state in [7:0], error code [15:8] */
 #define IGB_MIG_STATUS_STATE_MASK       0xFF
@@ -111,6 +112,15 @@ typedef struct IGBVfDirtyState {
     uint32_t num_ranges;
 } IGBVfDirtyState;
 
+typedef struct IgbVfMigStats {
+    uint64_t dma_writes;
+    uint64_t dma_bytes;
+    uint32_t dirty_pages_set;
+    uint32_t dirty_pages_cleared;
+    uint32_t dirty_page_count;
+    uint32_t dirty_query_count;
+} IgbVfMigStats;
+
 typedef struct IgbVfMigState {
     uint32_t mig_state;
     uint32_t mig_data[IGB_VF_STATE_MAX_SIZE / sizeof(uint32_t)];
@@ -149,6 +159,18 @@ struct igb_mig_dirty_query {
     uint8_t bitmap[];
 };
 
+/*
+ * GET_STATS:    device writes igb_mig_stats_resp to buffer.
+ */
+struct igb_mig_stats_resp {
+    uint64_t dma_writes;
+    uint64_t dma_bytes;
+    uint32_t dirty_pages_set;
+    uint32_t dirty_pages_cleared;
+    uint32_t dirty_page_count;
+    uint32_t dirty_query_count;
+};
+
 typedef struct IGBCore IGBCore;
 typedef struct IgbVfState IgbVfState;
 
diff --git a/hw/net/igb_migration.c b/hw/net/igb_migration.c
index 46299ff77ab9..2504d5bd308b 100644
--- a/hw/net/igb_migration.c
+++ b/hw/net/igb_migration.c
@@ -438,6 +438,7 @@ void igb_core_dirty_track_dma(IGBCore *core, int vfn,
                               dma_addr_t addr, dma_addr_t len)
 {
     IGBVfDirtyState *ds = &core->vf_dirty[vfn];
+    IgbVfMigStats *stats = &core->vf_mig_stats[vfn];
     bool matched = false;
     uint32_t i;
 
@@ -447,6 +448,9 @@ void igb_core_dirty_track_dma(IGBCore *core, int vfn,
 
     trace_igb_core_dirty_track_dma(vfn, addr, len);
 
+    stats->dma_writes++;
+    stats->dma_bytes += len;
+
     for (i = 0; i < ds->num_ranges; i++) {
         IGBVfDirtyRange *r = &ds->ranges[i];
         uint64_t r_end = r->iova + r->size;
@@ -466,7 +470,10 @@ void igb_core_dirty_track_dma(IGBCore *core, int vfn,
 
         for (page = start_page; page <= end_page; page++) {
             if (page < r->nbits) {
-                set_bit(page, r->bitmap);
+                if (!test_and_set_bit(page, r->bitmap)) {
+                    stats->dirty_pages_set++;
+                    stats->dirty_page_count++;
+                }
             }
         }
     }
@@ -493,6 +500,15 @@ static uint32_t igb_core_vf_dirty_enable(IgbVfState *s, uint64_t pgsize,
     IGBVfDirtyState *ds = igb_core_vf_dirty_state(s);
     IGBVfDirtyRange *r;
 
+    /*
+     * Reset stats on first enable so the driver can read them after
+     * disable
+     */
+    if (ds->num_ranges == 0) {
+        memset(&igbvf_get_core(s)->vf_mig_stats[s->vfn], 0,
+               sizeof(IgbVfMigStats));
+    }
+
     if (ds->num_ranges >= IGB_MIG_CAPS_MAX_RANGES) {
         return IGB_MIG_ERR_TOO_MANY_RANGES;
     }
@@ -635,12 +651,14 @@ static IGBVfDirtyRange *igb_core_vf_dirty_range_valid(IGBVfDirtyState *ds,
 static uint8_t igbvf_mig_cmd_dirty_query(IgbVfState *s)
 {
     IgbVfMigState *ms = &s->mig;
+    IgbVfMigStats *stats = &igbvf_get_core(s)->vf_mig_stats[s->vfn];
     IGBVfDirtyState *ds = &igbvf_get_core(s)->vf_dirty[s->vfn];
     uint64_t buf_addr = ms->mig_data_buf_addr;
     uint64_t range_iova = 0, range_size = 0;
     IGBVfDirtyRange *range;
     uint32_t bmp_bytes, dirty_pages;
     uint32_t val32;
+    uint64_t val64;
     size_t out_size;
     g_autofree void *bitmap = NULL;
 
@@ -686,6 +704,10 @@ static uint8_t igbvf_mig_cmd_dirty_query(IgbVfState *s)
 
     dirty_pages = bitmap_count_one(bitmap, range_size / range->page_size);
 
+    stats->dirty_pages_cleared += dirty_pages;
+    stats->dirty_page_count -= MIN(stats->dirty_page_count, dirty_pages);
+    stats->dirty_query_count++;
+
     val32 = cpu_to_le32(out_size);
     address_space_write(&address_space_memory,
                         buf_addr + offsetof(struct igb_mig_dirty_query,
@@ -696,8 +718,13 @@ static uint8_t igbvf_mig_cmd_dirty_query(IgbVfState *s)
                         buf_addr + offsetof(struct igb_mig_dirty_query,
                                             dirty_page_count),
                         MEMTXATTRS_UNSPECIFIED, &val32, sizeof(val32));
-
-    trace_igbvf_mig_dirty_query(s->vfn, (uint64_t)out_size, dirty_pages);
+    val64 = cpu_to_le64(stats->dma_writes);
+    address_space_write(&address_space_memory,
+                        buf_addr + offsetof(struct igb_mig_dirty_query,
+                                            dma_writes),
+                        MEMTXATTRS_UNSPECIFIED, &val64, sizeof(val64));
+    trace_igbvf_mig_dirty_query(s->vfn, (uint64_t)out_size, dirty_pages,
+                                stats->dma_writes);
     return 0;
 }
 
@@ -898,6 +925,32 @@ static uint8_t igbvf_mig_set_state(IgbVfState *s, uint32_t new_state)
     return 0;
 }
 
+static uint8_t igbvf_mig_cmd_get_stats(IgbVfState *s)
+{
+    IgbVfMigState *ms = &s->mig;
+    IgbVfMigStats *stats = &igbvf_get_core(s)->vf_mig_stats[s->vfn];
+    struct igb_mig_stats_resp resp;
+    MemTxResult r;
+
+    if (!ms->mig_data_buf_addr) {
+        return IGB_MIG_ERR_NO_BUFFER;
+    }
+
+    resp.dma_writes = cpu_to_le64(stats->dma_writes);
+    resp.dma_bytes = cpu_to_le64(stats->dma_bytes);
+    resp.dirty_pages_set = cpu_to_le32(stats->dirty_pages_set);
+    resp.dirty_pages_cleared = cpu_to_le32(stats->dirty_pages_cleared);
+    resp.dirty_page_count = cpu_to_le32(stats->dirty_page_count);
+    resp.dirty_query_count = cpu_to_le32(stats->dirty_query_count);
+
+    r = address_space_write(&address_space_memory, ms->mig_data_buf_addr,
+                            MEMTXATTRS_UNSPECIFIED, &resp, sizeof(resp));
+    if (r != MEMTX_OK) {
+        return IGB_MIG_ERR_DMA_FAILED;
+    }
+    return 0;
+}
+
 static void igbvf_mig_update_status(IgbVfState *s, uint8_t err)
 {
     IgbVfMigState *ms = &s->mig;
@@ -955,6 +1008,10 @@ static void igbvf_mig_cmd_ctrl(IgbVfState *s, uint32_t val)
         err = igbvf_mig_cmd_dirty_query(s);
         break;
 
+    case IGB_MIG_CMD_GET_STATS:
+        err = igbvf_mig_cmd_get_stats(s);
+        break;
+
     default:
         err = IGB_MIG_ERR_UNK_CMD;
         break;
@@ -1059,6 +1116,8 @@ void igbvf_mig_state_reset(IgbVfState *s)
     memset(ms->mig_data, 0, sizeof(ms->mig_data));
     ms->mig_saved_vfre = true;
     ms->mig_saved_vfte = true;
+    memset(&igbvf_get_core(s)->vf_mig_stats[s->vfn], 0,
+           sizeof(IgbVfMigStats));
 
     pci_set_long(PCI_DEVICE(s)->config +
                  IGB_MIG_DVSEC_OFFSET + IGB_MIG_BUF_ADDR_LO, 0);
diff --git a/hw/net/trace-events b/hw/net/trace-events
index eb5c61a8f6fb..6c2c5493094a 100644
--- a/hw/net/trace-events
+++ b/hw/net/trace-events
@@ -302,7 +302,7 @@ igbvf_mig_load_state(uint16_t vfn, uint32_t size, bool vfre, bool vfte) "VF%u: l
 igbvf_mig_reset(uint16_t vfn) "VF%u: migration state reset"
 igbvf_mig_dirty_enable(uint16_t vfn, uint64_t pgsize, uint64_t nbits) "VF%u: dirty tracking enabled pgsize=%"PRIu64" nbits=%"PRIu64
 igbvf_mig_dirty_disable(uint16_t vfn) "VF%u: dirty tracking disabled"
-igbvf_mig_dirty_query(uint16_t vfn, uint64_t size, uint32_t dirty_pages) "VF%u: dirty query returned %"PRIu64" bytes, %u dirty pages"
+igbvf_mig_dirty_query(uint16_t vfn, uint64_t size, uint32_t dirty_pages, uint64_t dma_writes) "VF%u: dirty query returned %"PRIu64" bytes, %u dirty pages (dma_writes=%"PRIu64")"
 igb_core_dirty_track_dma(int vfn, uint64_t addr, uint64_t len) "VF%d: dirty DMA addr=0x%"PRIx64" len=%"PRIu64
 igb_core_dirty_track_dma_drop(int vfn, uint64_t addr, uint64_t len) "VF%d: dirty DMA dropped addr=0x%"PRIx64" len=%"PRIu64" no matching range"
 igbvf_mig_quiesce(uint16_t vfn, uint32_t vfre, uint32_t vfte) "VF%u: quiesce VFRE=0x%x VFTE=0x%x"
-- 
2.55.0