[Qemu-devel] [RFC v4 1/9] exec.c: Add new exclusive bitmap to ram_list

[Alvise Rigo <a.rigo@virtualopensystems.com>]

The purpose of this new bitmap is to flag the memory pages that are in
the middle of LL/SC operations (after a LL, before a SC) on a per-vCPU
basis.
For all these pages, the corresponding TLB entries will be generated
in such a way to force the slow-path if at least one vCPU has the bit
not set.
When the system starts, the whole memory is dirty (all the bitmap is
set). A page, after being marked as exclusively-clean, will be
restored as dirty after the SC.

The accessors to this bitmap are currently not atomic, but they have to
be so in a real multi-threading TCG.

Suggested-by: Jani Kokkonen <jani.kokkonen@huawei.com>
Suggested-by: Claudio Fontana <claudio.fontana@huawei.com>
Signed-off-by: Alvise Rigo <a.rigo@virtualopensystems.com>
---
 exec.c                  |  7 +++--
 include/exec/memory.h   |  3 ++-
 include/exec/ram_addr.h | 68 +++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 75 insertions(+), 3 deletions(-)

diff --git a/exec.c b/exec.c
index 7d60e15..f113076 100644
--- a/exec.c
+++ b/exec.c
@@ -1493,11 +1493,14 @@ static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
         int i;
 
         /* ram_list.dirty_memory[] is protected by the iothread lock.  */
-        for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
+        for (i = 0; i < DIRTY_MEMORY_EXCLUSIVE; i++) {
             ram_list.dirty_memory[i] =
                 bitmap_zero_extend(ram_list.dirty_memory[i],
                                    old_ram_size, new_ram_size);
-       }
+        }
+        ram_list.dirty_memory[DIRTY_MEMORY_EXCLUSIVE] = bitmap_zero_extend(
+                ram_list.dirty_memory[DIRTY_MEMORY_EXCLUSIVE],
+                old_ram_size * smp_cpus, new_ram_size * smp_cpus);
     }
     cpu_physical_memory_set_dirty_range(new_block->offset,
                                         new_block->used_length,
diff --git a/include/exec/memory.h b/include/exec/memory.h
index 1394715..a525259 100644
--- a/include/exec/memory.h
+++ b/include/exec/memory.h
@@ -19,7 +19,8 @@
 #define DIRTY_MEMORY_VGA       0
 #define DIRTY_MEMORY_CODE      1
 #define DIRTY_MEMORY_MIGRATION 2
-#define DIRTY_MEMORY_NUM       3        /* num of dirty bits */
+#define DIRTY_MEMORY_EXCLUSIVE 3
+#define DIRTY_MEMORY_NUM       4        /* num of dirty bits */
 
 #include <stdint.h>
 #include <stdbool.h>
diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h
index c113f21..6b678d6 100644
--- a/include/exec/ram_addr.h
+++ b/include/exec/ram_addr.h
@@ -21,6 +21,7 @@
 
 #ifndef CONFIG_USER_ONLY
 #include "hw/xen/xen.h"
+#include "sysemu/sysemu.h"
 
 ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
                                     bool share, const char *mem_path,
@@ -135,6 +136,10 @@ static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
     if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
         bitmap_set_atomic(d[DIRTY_MEMORY_CODE], page, end - page);
     }
+    if (unlikely(mask & (1 << DIRTY_MEMORY_EXCLUSIVE))) {
+        bitmap_set_atomic(d[DIRTY_MEMORY_EXCLUSIVE],
+                          page * smp_cpus, (end - page) * smp_cpus);
+    }
     xen_modified_memory(start, length);
 }
 
@@ -249,5 +254,68 @@ uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest,
     return num_dirty;
 }
 
+/* Exclusive bitmap support. */
+#define EXCL_BITMAP_GET_OFFSET(addr) (smp_cpus * (addr >> TARGET_PAGE_BITS))
+static inline void cpu_physical_memory_set_excl_dirty(ram_addr_t addr,
+                                                      uint32_t cpu_index)
+{
+    set_bit_atomic(EXCL_BITMAP_GET_OFFSET(addr) + cpu_index,
+                   ram_list.dirty_memory[DIRTY_MEMORY_EXCLUSIVE]);
+}
+
+static inline int cpu_physical_memory_excl_atleast_one_clean(ram_addr_t addr)
+{
+    unsigned long *bitmap = ram_list.dirty_memory[DIRTY_MEMORY_EXCLUSIVE];
+    unsigned long next, end;
+
+    if (likely(smp_cpus <= BITS_PER_LONG)) {
+        unsigned long mask = (1 << smp_cpus) - 1;
+
+        return
+            (mask & (bitmap[BIT_WORD(EXCL_BITMAP_GET_OFFSET(addr))] >>
+            (EXCL_BITMAP_GET_OFFSET(addr) & (BITS_PER_LONG-1)))) != mask;
+    }
+
+    end = BIT_WORD(EXCL_BITMAP_GET_OFFSET(addr)) + smp_cpus;
+    next = find_next_zero_bit(bitmap, end,
+                              BIT_WORD(EXCL_BITMAP_GET_OFFSET(addr)));
+
+    return next < end;
+}
+
+static inline int cpu_physical_memory_excl_is_dirty(ram_addr_t addr,
+                                                    unsigned long cpu)
+{
+    unsigned long *bitmap = ram_list.dirty_memory[DIRTY_MEMORY_EXCLUSIVE];
+    unsigned long end, next;
+    uint32_t add;
+
+    assert(cpu <= smp_cpus);
+
+    if (likely(smp_cpus <= BITS_PER_LONG)) {
+        cpu = (cpu == smp_cpus) ? (1 << cpu) - 1 : (1 << cpu);
+
+        return cpu & (bitmap[BIT_WORD(EXCL_BITMAP_GET_OFFSET(addr))] >>
+                     (EXCL_BITMAP_GET_OFFSET(addr) & (BITS_PER_LONG-1)));
+    }
+
+    add = (cpu == smp_cpus) ? 0 : 1;
+    end = BIT_WORD(EXCL_BITMAP_GET_OFFSET(addr)) + cpu + add;
+    next = find_next_bit(bitmap, end, BIT_WORD(EXCL_BITMAP_GET_OFFSET(addr))
+                         + (cpu % smp_cpus));
+
+    return next < end;
+}
+
+static inline bool cpu_physical_memory_clear_excl_dirty(ram_addr_t addr,
+                                                        uint32_t cpu_index)
+{
+    return bitmap_test_and_clear_atomic(
+                                ram_list.dirty_memory[DIRTY_MEMORY_EXCLUSIVE],
+                                EXCL_BITMAP_GET_OFFSET(addr) + cpu_index, 1);
+}
+
+
+
 #endif
 #endif
-- 
2.5.0