[Qemu-devel] [PATCH v6 0/6] Obtain dirty bitmap via VM logging

[Qemu-devel] [PATCH v6 0/6] Obtain dirty bitmap via VM logging

[Sanidhya Kashyap]

Hi,

The following patches introduce the support of the dirty bitmap logging and
dumping to a specified file. This patch series addresses the previous issues
raised by David and Eric.

v5 --> v6
* The shared function between migration and log bitmap has been again modified
  (David's advice).
* Again changed the naming conventions for shared state between migration and
  log dirty bitmap (David's and Eric's advice).
* Removed acronyms (Eric's advice).
* Fixed grammatical mistakes and macros (Eric's advice).
* Modified the python script to utilize less memory as well as dump the snapshot
  into a file instead of separate window (David's advice).

v4 --> v5
* Removed the runstates code (Eric's and David's advice).
* Now, using global variable to resolve the issue between migration and
  bitmap dump (David's advice).
* Merged the hmp and qmp interface's patch (Eric's advice).
* Modified the code to support the dynamic change in last_ram_offset (David's
  advice).
* Formatted the strings in the python code i.e. length followed by text
  (David's advice).
* Added some code to generate black and white figure based on the bitmap.
* Modified the shared function to be more generic (David's advice).

v3 --> v4
* Added new qmp interface for information extraction from the bitmap process

v2 --> v3
* Reformatted the code and removed some unnecessary parts.
* Printing block info along with length and offset.
* Changed the functions that were directly using RUN_STATE_RUNNING as state.

v1 --> v2:
* Added two new run states to avoid simultaneous execution of both migration and
  bitmap dump process.
* Removed FILE pointer usage.
* Dumping the data only in machine-readable format.
* Tried to rectify mistakes of the previous version.



Sanidhya Kashyap (6):
  generic function between migration and bitmap dump
  BitmapLog: bitmap dump code
  BitmapLog: get the information about the parameters
  BitmapLog: cancel mechanism for an already running dump bitmap process
  BitmapLog: set the period of the dump bitmap process
  BitmapLog: python script for extracting bitmap from a binary file

 arch_init.c               |  19 ++-
 hmp-commands.hx           |  47 ++++++
 hmp.c                     |  56 ++++++
 hmp.h                     |   4 +
 include/exec/cpu-all.h    |   5 +-
 include/exec/ram_addr.h   |   4 +
 include/sysemu/sysemu.h   |   5 +
 migration.c               |  12 ++
 monitor.c                 |   7 +
 qapi-schema.json          |  86 ++++++++++
 qmp-commands.hx           | 101 +++++++++++
 savevm.c                  | 423 ++++++++++++++++++++++++++++++++++++++++++++++
 scripts/extract-bitmap.py | 213 +++++++++++++++++++++++
 vl.c                      |  23 +++
 14 files changed, 996 insertions(+), 9 deletions(-)
 create mode 100755 scripts/extract-bitmap.py

-- 
1.9.1

[Qemu-devel] [PATCH v6 1/6] generic function between migration and bitmap dump

[Sanidhya Kashyap]

Now, the counter variable is available to all the functions that will call
qemu_bitmap_sync_range. I have tried to make the function along with the
variables as generic as possible.

Signed-off-by: Sanidhya Kashyap <sanidhya.iiith@gmail.com>
---
 arch_init.c             | 19 +++++++++++--------
 include/exec/ram_addr.h |  4 ++++
 2 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/arch_init.c b/arch_init.c
index c974f3f..269c669 100644
--- a/arch_init.c
+++ b/arch_init.c
@@ -436,20 +436,22 @@ ram_addr_t migration_bitmap_find_and_reset_dirty(MemoryRegion *mr,
     return (next - base) << TARGET_PAGE_BITS;
 }
 
-static inline bool migration_bitmap_set_dirty(ram_addr_t addr)
+static inline bool qemu_bitmap_set_dirty(ram_addr_t addr, unsigned long *bitmap,
+                                         uint64_t *counter)
 {
     bool ret;
     int nr = addr >> TARGET_PAGE_BITS;
 
-    ret = test_and_set_bit(nr, migration_bitmap);
+    ret = test_and_set_bit(nr, bitmap);
 
     if (!ret) {
-        migration_dirty_pages++;
+        (*counter)++;
     }
     return ret;
 }
 
-static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)
+void qemu_bitmap_sync_range(ram_addr_t start, ram_addr_t length,
+                            unsigned long *bitmap, uint64_t *counter)
 {
     ram_addr_t addr;
     unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
@@ -463,8 +465,8 @@ static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)
         for (k = page; k < page + nr; k++) {
             if (src[k]) {
                 unsigned long new_dirty;
-                new_dirty = ~migration_bitmap[k];
-                migration_bitmap[k] |= src[k];
+                new_dirty = ~bitmap[k];
+                bitmap[k] |= src[k];
                 new_dirty &= src[k];
                 migration_dirty_pages += ctpopl(new_dirty);
                 src[k] = 0;
@@ -478,7 +480,7 @@ static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)
                 cpu_physical_memory_reset_dirty(start + addr,
                                                 TARGET_PAGE_SIZE,
                                                 DIRTY_MEMORY_MIGRATION);
-                migration_bitmap_set_dirty(start + addr);
+                qemu_bitmap_set_dirty(start + addr, bitmap, counter);
             }
         }
     }
@@ -514,7 +516,8 @@ static void migration_bitmap_sync(void)
     address_space_sync_dirty_bitmap(&address_space_memory);
 
     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        migration_bitmap_sync_range(block->mr->ram_addr, block->length);
+        qemu_bitmap_sync_range(block->mr->ram_addr, block->length,
+                               migration_bitmap, &migration_dirty_pages);
     }
     trace_migration_bitmap_sync_end(migration_dirty_pages
                                     - num_dirty_pages_init);
diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h
index 6593be1..fcc3501 100644
--- a/include/exec/ram_addr.h
+++ b/include/exec/ram_addr.h
@@ -162,5 +162,9 @@ static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length,
                                      unsigned client);
 
+
+void qemu_bitmap_sync_range(ram_addr_t start, ram_addr_t length,
+                            unsigned long *bitmap, uint64_t *counter);
+
 #endif
 #endif
-- 
1.9.1

[Qemu-devel] [PATCH v6 2/6] BitmapLog: bitmap dump code

[Sanidhya Kashyap]

Signed-off-by: Sanidhya Kashyap <sanidhya.iiith@gmail.com>
---

No functional change, except:
- naming convention i.e. QemuProcess has been changed to QemuDirtyBitmapUser.
- rectified mistakes in documentation in qapi-schema.json.
- removed acronyms

 hmp-commands.hx         |  16 ++
 hmp.c                   |  18 +++
 hmp.h                   |   1 +
 include/exec/cpu-all.h  |   5 +-
 include/sysemu/sysemu.h |   5 +
 migration.c             |  12 ++
 qapi-schema.json        |  37 +++++
 qmp-commands.hx         |  32 ++++
 savevm.c                | 378 ++++++++++++++++++++++++++++++++++++++++++++++++
 vl.c                    |  23 +++
 10 files changed, 526 insertions(+), 1 deletion(-)

diff --git a/hmp-commands.hx b/hmp-commands.hx
index f859f8d..d104232 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -1786,6 +1786,22 @@ STEXI
 show available trace events and their state
 ETEXI
 
+     {
+        .name       = "log_dirty_bitmap",
+        .args_type  = "filename:s,iterations:i?,period:i?",
+        .params     = "filename iterations period",
+        .help       = "dumps the memory's dirty bitmap to file\n\t\t\t"
+                      "filename: name of the file in which the bitmap will be saved\n\t\t\t"
+                      "iterations: number of times the memory will be logged\n\t\t\t"
+                      "period: time difference in milliseconds between each iteration",
+        .mhandler.cmd = hmp_log_dirty_bitmap,
+    },
+STEXI
+@item log_dirty_bitmap @var{filename}
+@findex log_dirty_bitmap
+dumps the writable working set of a VM's memory to a file
+ETEXI
+
 STEXI
 @end table
 ETEXI
diff --git a/hmp.c b/hmp.c
index 4d1838e..d067420 100644
--- a/hmp.c
+++ b/hmp.c
@@ -1318,6 +1318,24 @@ void hmp_device_del(Monitor *mon, const QDict *qdict)
     hmp_handle_error(mon, &err);
 }
 
+void hmp_log_dirty_bitmap(Monitor *mon, const QDict *qdict)
+{
+    const char *filename = qdict_get_str(qdict, "filename");
+    bool has_iterations = qdict_haskey(qdict, "iterations");
+    int64_t iterations = qdict_get_try_int(qdict, "iterations", 3);
+    bool has_period = qdict_haskey(qdict, "period");
+    int64_t period = qdict_get_try_int(qdict, "period", 10);
+    Error *err = NULL;
+
+    qmp_log_dirty_bitmap(filename, has_iterations, iterations,
+                         has_period, period, &err);
+    if (err) {
+        monitor_printf(mon, "log_dirty_bitmap: %s\n", error_get_pretty(err));
+        error_free(err);
+        return;
+    }
+}
+
 void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict)
 {
     Error *err = NULL;
diff --git a/hmp.h b/hmp.h
index 4fd3c4a..0895182 100644
--- a/hmp.h
+++ b/hmp.h
@@ -94,6 +94,7 @@ void hmp_cpu_add(Monitor *mon, const QDict *qdict);
 void hmp_object_add(Monitor *mon, const QDict *qdict);
 void hmp_object_del(Monitor *mon, const QDict *qdict);
 void hmp_info_memdev(Monitor *mon, const QDict *qdict);
+void hmp_log_dirty_bitmap(Monitor *mon, const QDict *qdict);
 void object_add_completion(ReadLineState *rs, int nb_args, const char *str);
 void object_del_completion(ReadLineState *rs, int nb_args, const char *str);
 void device_add_completion(ReadLineState *rs, int nb_args, const char *str);
diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
index f9d132f..4824d36 100644
--- a/include/exec/cpu-all.h
+++ b/include/exec/cpu-all.h
@@ -299,13 +299,16 @@ CPUArchState *cpu_copy(CPUArchState *env);
 
 /* memory API */
 
+/* global name which is used with both migration and bitmap dump */
+#define RAMBLOCK_NAME_LENGTH 256
+
 typedef struct RAMBlock {
     struct MemoryRegion *mr;
     uint8_t *host;
     ram_addr_t offset;
     ram_addr_t length;
     uint32_t flags;
-    char idstr[256];
+    char idstr[RAMBLOCK_NAME_LENGTH];
     /* Reads can take either the iothread or the ramlist lock.
      * Writes must take both locks.
      */
diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index d8539fd..b5525ea 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -227,4 +227,9 @@ extern QemuOptsList qemu_net_opts;
 extern QemuOptsList qemu_global_opts;
 extern QemuOptsList qemu_mon_opts;
 
+/* migration vs dirty bitmap process */
+bool qemu_process_check(QemuDirtyBitmapUser user);
+void qemu_process_set(QemuDirtyBitmapUser user);
+const char *get_qemu_dirty_bitmap_user_as_string(void);
+
 #endif
diff --git a/migration.c b/migration.c
index 8d675b3..6a02b40 100644
--- a/migration.c
+++ b/migration.c
@@ -117,6 +117,7 @@ static void process_incoming_migration_co(void *opaque)
     } else {
         runstate_set(RUN_STATE_PAUSED);
     }
+    qemu_process_set(QEMU_DIRTY_BITMAP_USER_NONE);
 }
 
 void process_incoming_migration(QEMUFile *f)
@@ -317,6 +318,7 @@ static void migrate_fd_cleanup(void *opaque)
     }
 
     notifier_list_notify(&migration_state_notifiers, s);
+    qemu_process_set(QEMU_DIRTY_BITMAP_USER_NONE);
 }
 
 void migrate_fd_error(MigrationState *s)
@@ -326,6 +328,7 @@ void migrate_fd_error(MigrationState *s)
     s->state = MIG_STATE_ERROR;
     trace_migrate_set_state(MIG_STATE_ERROR);
     notifier_list_notify(&migration_state_notifiers, s);
+    qemu_process_set(QEMU_DIRTY_BITMAP_USER_NONE);
 }
 
 static void migrate_fd_cancel(MigrationState *s)
@@ -436,6 +439,15 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
         return;
     }
 
+    if (!qemu_process_check(QEMU_DIRTY_BITMAP_USER_NONE) &&
+        !qemu_process_check(QEMU_DIRTY_BITMAP_USER_MIGRATION)) {
+        error_setg(errp, "Migration not possible, since %s "
+                   "is in progress.", get_qemu_dirty_bitmap_user_as_string());
+        return;
+    }
+
+    qemu_process_set(QEMU_DIRTY_BITMAP_USER_MIGRATION);
+
     s = migrate_init(&params);
 
     if (strstart(uri, "tcp:", &p)) {
diff --git a/qapi-schema.json b/qapi-schema.json
index 689b548..f96e959 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -3481,3 +3481,40 @@
 # Since: 2.1
 ##
 { 'command': 'rtc-reset-reinjection' }
+
+##
+# QemuDirtyBitmapUser
+#
+# @none: no other process is being executed besides a simple VM execution.
+#
+# @migration: migration process is going on.
+#
+# @bitmap-dump: bitmap dump process is being executed.
+#
+# Since 2.2
+##
+{ 'enum': 'QemuDirtyBitmapUser',
+  'data': [ 'none', 'migration', 'bitmap-dump' ] }
+
+##
+# @log-dirty-bitmap
+#
+# This command will dump the dirty bitmap to a file by logging the
+# memory for a specified number of times with a defined time difference
+#
+# @filename: name of the file in which the bitmap will be saved.
+#
+# @iterations: #optional number of times the memory will be logged. The
+# min and max values are 3 and 100000 respectively. 3 is the default value,
+# if the iterations parameter is not stated.
+#
+# @period: #optional time difference in milliseconds between each iteration.
+# The min and max values are 10 and 100000 respectively. 10 is the default
+# value if the period parameter is not stated.
+#
+# Since 2.2
+##
+{ 'command' : 'log-dirty-bitmap',
+  'data'    : { 'filename'      : 'str',
+                '*iterations'   : 'int',
+                '*period'       : 'int' } }
diff --git a/qmp-commands.hx b/qmp-commands.hx
index 7658d4b..db0a8ed 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -3752,5 +3752,37 @@ Example:
 
 -> { "execute": "rtc-reset-reinjection" }
 <- { "return": {} }
+EQMP
 
+    {
+        .name       = "log-dirty-bitmap",
+        .args_type  = "filename:s,iterations:i?,period:i?",
+        .mhandler.cmd_new = qmp_marshal_input_log_dirty_bitmap,
+    },
+
+SQMP
+log-dirty-bitmap
+----------------
+
+start logging the memory of the VM for writable working set
+
+Arguments:
+
+- "filename": name of the file, in which the bitmap will be saved.
+
+- "iterations": #optional number of times, the memory will be logged.
+  The min and max values are 3 and 100000 respectively. 3 is the default value.
+
+- "period": #optional time difference in milliseconds between each iteration.
+   The min and max values are 10 and 100000 respectively. 10 is the default
+   value.
+
+Examples:
+-> { "execute": "log-dirty-bitmap",
+     "arguments": {
+         "filename": "/tmp/fileXXX",
+         "iterations": 3,
+         "period": 10 } }
+
+<- { "return": {} }
 EQMP
diff --git a/savevm.c b/savevm.c
index e19ae0a..19f7b0c 100644
--- a/savevm.c
+++ b/savevm.c
@@ -42,6 +42,9 @@
 #include "qemu/iov.h"
 #include "block/snapshot.h"
 #include "block/qapi.h"
+#include "exec/address-spaces.h"
+#include "exec/ram_addr.h"
+#include "qemu/bitmap.h"
 
 
 #ifndef ETH_P_RARP
@@ -1137,6 +1140,381 @@ void do_savevm(Monitor *mon, const QDict *qdict)
     }
 }
 
+/*
+ * Adding the functionality of continuous logging of the
+ * dirty bitmap which is almost similar to the migration
+ * thread
+ */
+
+enum {
+    LOG_BITMAP_STATE_ERROR = -1,
+    LOG_BITMAP_STATE_NONE,
+    LOG_BITMAP_STATE_ACTIVE,
+    LOG_BITMAP_STATE_CANCELING,
+    LOG_BITMAP_STATE_COMPLETED
+};
+
+typedef struct BitmapLogState BitmapLogState;
+static int64_t MIN_ITERATION_VALUE = 3;
+static int64_t MIN_PERIOD_VALUE = 10;
+static int64_t MAX_ITERATION_VALUE = 100000;
+static int64_t MAX_PERIOD_VALUE = 100000;
+
+struct BitmapLogState {
+    int state;
+    int fd;
+    int64_t current_period;
+    int64_t current_iteration;
+    int64_t iterations;
+    unsigned long *log_bitmap_array;
+    QemuThread thread;
+};
+
+/*
+ * helper functions
+ */
+
+static inline void log_bitmap_lock(void)
+{
+    qemu_mutex_lock_iothread();
+    qemu_mutex_lock_ramlist();
+}
+
+static inline void log_bitmap_unlock(void)
+{
+    qemu_mutex_unlock_ramlist();
+    qemu_mutex_unlock_iothread();
+}
+
+static inline void log_bitmap_set_dirty(ram_addr_t addr,
+                                        unsigned long *log_bitmap_array)
+{
+    long nr  = addr >> TARGET_PAGE_BITS;
+    set_bit(nr, log_bitmap_array);
+}
+
+static bool log_bitmap_set_status(BitmapLogState *b,
+                                  int old_state,
+                                  int new_state)
+{
+    return atomic_cmpxchg(&b->state, old_state, new_state);
+}
+
+/*
+ * inspired from migration mechanism
+ */
+
+static BitmapLogState *log_bitmap_get_current_state(void)
+{
+    static BitmapLogState current_bitmaplogstate = {
+        .state = LOG_BITMAP_STATE_NONE,
+        .log_bitmap_array = NULL,
+    };
+
+    return &current_bitmaplogstate;
+}
+
+/*
+ * syncing the log_bitmap with the ram_list dirty bitmap
+ */
+
+static void log_bitmap_dirty_bitmap_sync(unsigned long *log_bitmap_array)
+{
+    RAMBlock *block;
+    uint64_t counter = 0; /* 0 means log bitmap */
+    address_space_sync_dirty_bitmap(&address_space_memory);
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+        qemu_bitmap_sync_range(block->mr->ram_addr, block->length,
+                               log_bitmap_array, &counter);
+    }
+}
+
+static inline bool value_in_range(int64_t value, int64_t min_value,
+                                  int64_t max_value, const char *str,
+                                  Error **errp)
+{
+    if (value < min_value) {
+        error_setg(errp, "%s's value must be greater than %ld",
+                         str, min_value);
+        return false;
+    }
+    if (value > max_value) {
+        error_setg(errp, "%s's value must be less than %ld",
+                         str, max_value);
+        return false;
+    }
+    return true;
+}
+
+static inline void log_bitmap_close(BitmapLogState *b)
+{
+    log_bitmap_lock();
+    memory_global_dirty_log_stop();
+    log_bitmap_unlock();
+
+    g_free(b->log_bitmap_array);
+    b->log_bitmap_array = NULL;
+    qemu_close(b->fd);
+    b->fd = -1;
+}
+
+static bool log_bitmap_ram_block_info_dump(int fd, int64_t ram_bitmap_pages,
+                                           bool dump_blocks_info)
+{
+    int block_count = 0;
+    int block_name_length;
+    RAMBlock *block;
+    int ret;
+
+    if (qemu_write_full(fd, &ram_bitmap_pages, sizeof(int64_t)) < 0) {
+        return true;
+    }
+
+    if (dump_blocks_info) {
+
+        QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+            block_count++;
+        }
+
+        ret = qemu_write_full(fd, &block_count, sizeof(int));
+        if (ret < sizeof(int)) {
+            return true;
+        }
+
+        QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+            block_name_length = strlen(block->idstr) + 1;
+            ret = qemu_write_full(fd, &block_name_length, sizeof(int));
+            if (ret < sizeof(int)) {
+                return true;
+            }
+
+            ret = qemu_write_full(fd, &(block->idstr), sizeof(char) *
+                                  block_name_length);
+            if (ret < sizeof(char) * block_name_length) {
+                return true;
+            }
+
+            ret = qemu_write_full(fd, &(block->offset), sizeof(ram_addr_t));
+            if (ret < sizeof(ram_addr_t)) {
+                return true;
+            }
+
+            ret = qemu_write_full(fd, &(block->length), sizeof(ram_addr_t));
+            if (ret < sizeof(ram_addr_t)) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+static void log_bitmap_update_status(BitmapLogState *b)
+{
+    int s = b->state;
+    switch (s) {
+    case LOG_BITMAP_STATE_ACTIVE:
+    case LOG_BITMAP_STATE_CANCELING:
+    case LOG_BITMAP_STATE_ERROR:
+        log_bitmap_set_status(b, s, LOG_BITMAP_STATE_COMPLETED);
+    }
+    return;
+}
+
+static void *bitmap_logging_thread(void *opaque)
+{
+    /*
+     * setup basic structures
+     */
+
+    BitmapLogState *b = opaque;
+    int fd = b->fd;
+    int64_t current_ram_bitmap_pages, prev_ram_bitmap_pages;
+    size_t bitmap_size = 0;
+    unsigned long *temp_log_bitmap_array = NULL;
+    char marker = 'M';
+    int ret;
+
+    b->current_iteration = 1;
+    log_bitmap_set_status(b, LOG_BITMAP_STATE_NONE,
+                             LOG_BITMAP_STATE_ACTIVE);
+
+    current_ram_bitmap_pages = 0;
+    prev_ram_bitmap_pages = 1;
+
+    /*
+     *  start the logging period
+     */
+
+    /*
+     * need lock for getting the information about the ram pages.
+     * This does not change on acquiring the lock
+     */
+    log_bitmap_lock();
+    current_ram_bitmap_pages = last_ram_offset() >> TARGET_PAGE_BITS;
+    bitmap_size = BITS_TO_LONGS(current_ram_bitmap_pages) *
+                                sizeof(unsigned long);
+    b->log_bitmap_array = bitmap_new(current_ram_bitmap_pages);
+    if (b->log_bitmap_array == NULL) {
+        b->state = LOG_BITMAP_STATE_ERROR;
+        log_bitmap_unlock();
+        goto log_thread_end;
+    }
+
+    memory_global_dirty_log_start();
+    log_bitmap_dirty_bitmap_sync(b->log_bitmap_array);
+    log_bitmap_unlock();
+
+    /*
+     * sync the dirty bitmap along with saving it
+     * using the QEMUFile pointer.
+     */
+    while (b->current_iteration <= b->iterations) {
+        if (!runstate_is_running() ||
+            b->state != LOG_BITMAP_STATE_ACTIVE) {
+            goto log_thread_end;
+        }
+
+        /*
+         * Need to calculate the ram pages again as there is a
+         * possibility of the change in the memory
+         */
+        log_bitmap_lock();
+        current_ram_bitmap_pages = last_ram_offset() >> TARGET_PAGE_BITS;
+        if (current_ram_bitmap_pages != prev_ram_bitmap_pages) {
+            temp_log_bitmap_array = bitmap_new(current_ram_bitmap_pages);
+            if (temp_log_bitmap_array == NULL) {
+                b->state = LOG_BITMAP_STATE_ERROR;
+                log_bitmap_unlock();
+                goto log_thread_end;
+            }
+            log_bitmap_dirty_bitmap_sync(temp_log_bitmap_array);
+        } else {
+            log_bitmap_dirty_bitmap_sync(b->log_bitmap_array);
+        }
+        log_bitmap_unlock();
+
+        if (current_ram_bitmap_pages != prev_ram_bitmap_pages) {
+            prev_ram_bitmap_pages = current_ram_bitmap_pages;
+            bitmap_size = BITS_TO_LONGS(current_ram_bitmap_pages) *
+                                        sizeof(unsigned long);
+            if (b->log_bitmap_array) {
+                g_free(b->log_bitmap_array);
+            }
+            b->log_bitmap_array = temp_log_bitmap_array;
+            temp_log_bitmap_array = NULL;
+            if (log_bitmap_ram_block_info_dump(fd, current_ram_bitmap_pages,
+                                               true)) {
+                b->state = LOG_BITMAP_STATE_ERROR;
+                goto log_thread_end;
+            }
+        } else {
+            if (log_bitmap_ram_block_info_dump(fd, current_ram_bitmap_pages,
+                                               false)) {
+                b->state = LOG_BITMAP_STATE_ERROR;
+                goto log_thread_end;
+            }
+        }
+
+        ret = qemu_write_full(fd, b->log_bitmap_array, bitmap_size);
+        if (ret < bitmap_size) {
+            b->state = LOG_BITMAP_STATE_ERROR;
+            goto log_thread_end;
+        }
+
+        ret = qemu_write_full(fd, &marker, sizeof(char));
+        if (ret < sizeof(char)) {
+            b->state = LOG_BITMAP_STATE_ERROR;
+            goto log_thread_end;
+        }
+        g_usleep(b->current_period * 1000);
+        b->current_iteration++;
+        bitmap_zero(b->log_bitmap_array, current_ram_bitmap_pages);
+    }
+
+    /*
+     * stop the logging period.
+     */
+ log_thread_end:
+    log_bitmap_close(b);
+    log_bitmap_update_status(b);
+    qemu_process_set(QEMU_DIRTY_BITMAP_USER_NONE);
+    return NULL;
+}
+
+void qmp_log_dirty_bitmap(const char *filename, bool has_iterations,
+                          int64_t iterations, bool has_period,
+                          int64_t period, Error **errp)
+{
+    int fd = -1;
+    BitmapLogState *b = log_bitmap_get_current_state();
+    Error *local_err = NULL;
+
+    if (!runstate_is_running()) {
+        error_setg(errp, "Guest is not in a running state");
+        return;
+    }
+
+    if (!qemu_process_check(QEMU_DIRTY_BITMAP_USER_NONE) &&
+        !qemu_process_check(QEMU_DIRTY_BITMAP_USER_BITMAP_DUMP)) {
+        error_setg(errp, "Dirty bitmap dumping not possible, since %s "
+                   "is in progress.", get_qemu_dirty_bitmap_user_as_string());
+        return;
+    }
+
+    qemu_process_set(QEMU_DIRTY_BITMAP_USER_BITMAP_DUMP);
+
+    if (b->state == LOG_BITMAP_STATE_ACTIVE ||
+        b->state == LOG_BITMAP_STATE_CANCELING) {
+        error_setg(errp, "dirty bitmap dump in progress");
+        return;
+    }
+
+    b->state = LOG_BITMAP_STATE_NONE;
+
+    /*
+     * checking the iteration range
+     */
+    if (!has_iterations) {
+        b->iterations = MIN_ITERATION_VALUE;
+    } else if (!value_in_range(iterations, MIN_ITERATION_VALUE,
+                               MAX_ITERATION_VALUE, "iterations", &local_err)) {
+        if (local_err) {
+            error_propagate(errp, local_err);
+        }
+        return;
+    } else {
+        b->iterations = iterations;
+    }
+
+    /*
+     * checking the period range
+     */
+    if (!has_period) {
+        b->current_period = MIN_PERIOD_VALUE;
+    } else if (!value_in_range(period, MIN_PERIOD_VALUE,
+                               MAX_PERIOD_VALUE, "period", &local_err)) {
+        if (local_err) {
+            error_propagate(errp, local_err);
+        }
+        return;
+    }  else {
+        b->current_period = period;
+    }
+
+    fd = qemu_open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRUSR);
+    if (fd < 0) {
+        error_setg_file_open(errp, errno, filename);
+        return;
+    }
+
+    b->fd = fd;
+    qemu_thread_create(&b->thread, "dirty-bitmap-dump",
+                       bitmap_logging_thread, b,
+                       QEMU_THREAD_JOINABLE);
+
+    return;
+}
+
 void qmp_xen_save_devices_state(const char *filename, Error **errp)
 {
     QEMUFile *f;
diff --git a/vl.c b/vl.c
index 95be92d..a858752 100644
--- a/vl.c
+++ b/vl.c
@@ -206,6 +206,8 @@ bool qemu_uuid_set;
 static QEMUBootSetHandler *boot_set_handler;
 static void *boot_set_opaque;
 
+int dirty_bitmap_user;
+
 static NotifierList exit_notifiers =
     NOTIFIER_LIST_INITIALIZER(exit_notifiers);
 
@@ -768,7 +770,27 @@ void vm_start(void)
 
     qapi_event_send_resume(&error_abort);
 }
+/*
+ * A global variable to decide which process will only
+ * execute migration or bitmap dump
+ */
+
+static QemuDirtyBitmapUser dbu = QEMU_DIRTY_BITMAP_USER_NONE;
+
+bool qemu_process_check(QemuDirtyBitmapUser user)
+{
+    return user == dbu;
+}
+
+void qemu_process_set(QemuDirtyBitmapUser user)
+{
+    dbu = user;
+}
 
+const char *get_qemu_dirty_bitmap_user_as_string(void)
+{
+    return QemuDirtyBitmapUser_lookup[dbu];
+}
 
 /***********************************************************/
 /* real time host monotonic timer */
@@ -4545,6 +4567,7 @@ int main(int argc, char **argv, char **envp)
     }
 
     if (incoming) {
+        qemu_process_set(QEMU_DIRTY_BITMAP_USER_MIGRATION);
         Error *local_err = NULL;
         qemu_start_incoming_migration(incoming, &local_err);
         if (local_err) {
-- 
1.9.1

[Qemu-devel] [PATCH v6 3/6] BitmapLog: get the information about the parameters

[Sanidhya Kashyap]

Signed-off-by: Sanidhya Kashyap <sanidhya.iiith@gmail.com>
---

Removed acronyms, no functional change.

 hmp-commands.hx  |  2 ++
 hmp.c            | 21 +++++++++++++++++++++
 hmp.h            |  1 +
 monitor.c        |  7 +++++++
 qapi-schema.json | 28 ++++++++++++++++++++++++++++
 qmp-commands.hx  | 25 +++++++++++++++++++++++++
 savevm.c         | 17 +++++++++++++++++
 7 files changed, 101 insertions(+)

diff --git a/hmp-commands.hx b/hmp-commands.hx
index d104232..d336f20 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -1778,6 +1778,8 @@ show qdev device model list
 show roms
 @item info tpm
 show the TPM device
+@item info log_dirty_bitmap
+show the current parameters values
 @end table
 ETEXI
 
diff --git a/hmp.c b/hmp.c
index d067420..4533dcd 100644
--- a/hmp.c
+++ b/hmp.c
@@ -1732,3 +1732,24 @@ void hmp_info_memdev(Monitor *mon, const QDict *qdict)
 
     monitor_printf(mon, "\n");
 }
+
+void hmp_info_log_dirty_bitmap(Monitor *mon, const QDict *qdict)
+{
+    Error *err = NULL;
+    BitmapLogStateInfo *info = qmp_query_log_dirty_bitmap(&err);
+
+    if (info) {
+        monitor_printf(mon, "current iteration: %" PRId64 "\n",
+                       info->current_iteration);
+        monitor_printf(mon, "total iterations: %" PRId64 "\n",
+                       info->iterations);
+        monitor_printf(mon, "current period value: %" PRId64 "\n",
+                       info->period);
+    }
+
+    if (err) {
+        hmp_handle_error(mon, &err);
+    }
+
+    qapi_free_BitmapLogStateInfo(info);
+}
diff --git a/hmp.h b/hmp.h
index 0895182..02e8ee4 100644
--- a/hmp.h
+++ b/hmp.h
@@ -38,6 +38,7 @@ void hmp_info_balloon(Monitor *mon, const QDict *qdict);
 void hmp_info_pci(Monitor *mon, const QDict *qdict);
 void hmp_info_block_jobs(Monitor *mon, const QDict *qdict);
 void hmp_info_tpm(Monitor *mon, const QDict *qdict);
+void hmp_info_log_dirty_bitmap(Monitor *mon, const QDict *qdict);
 void hmp_quit(Monitor *mon, const QDict *qdict);
 void hmp_stop(Monitor *mon, const QDict *qdict);
 void hmp_system_reset(Monitor *mon, const QDict *qdict);
diff --git a/monitor.c b/monitor.c
index 34cee74..ba79375 100644
--- a/monitor.c
+++ b/monitor.c
@@ -2921,6 +2921,13 @@ static mon_cmd_t info_cmds[] = {
         .mhandler.cmd = hmp_info_memdev,
     },
     {
+        .name       = "log_dirty_bitmap",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the current parameters values",
+        .mhandler.cmd = hmp_info_log_dirty_bitmap,
+    },
+    {
         .name       = NULL,
     },
 };
diff --git a/qapi-schema.json b/qapi-schema.json
index f96e959..d1b44f4 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -3518,3 +3518,31 @@
   'data'    : { 'filename'      : 'str',
                 '*iterations'   : 'int',
                 '*period'       : 'int' } }
+##
+# @BitmapLogStateInfo
+#
+# Provides information for the bitmap logging process
+#
+# @current-iteration: stores current iteration value
+#
+# @iterations: total iterations value
+#
+# @period: the time difference in milliseconds between each iteration
+#
+# Since 2.2
+##
+{ 'type': 'BitmapLogStateInfo',
+  'data': { 'current-iteration' : 'int',
+            'iterations'        : 'int',
+            'period'            : 'int' } }
+
+##
+# @query-log-dirty-bitmap
+#
+# Get the current values of the parameters involved in bitmap logging process
+#
+# This command returns the BitmapLogStateInfo
+#
+# Since 2.2
+##
+{ 'command': 'query-log-dirty-bitmap', 'returns': 'BitmapLogStateInfo' }
diff --git a/qmp-commands.hx b/qmp-commands.hx
index db0a8ed..9582bc7 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -3786,3 +3786,28 @@ Examples:
 
 <- { "return": {} }
 EQMP
+
+    {
+        .name       = "query-log-dirty-bitmap",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_marshal_input_query_log_dirty_bitmap,
+    },
+
+SQMP
+query-log-dirty-bitmap
+----------------------
+
+Get the parameters information
+
+- "current-iteration": stores current iteration value
+- "iterations": total iterations value
+- "period": the time difference in milliseconds between each iteration
+
+Example:
+
+-> { "execute": "query-log-dirty-bitmap" }
+<- { "return": {
+            "current-iteration": 3,
+            "iterations": 10,
+            "period": 100 } }
+EQMP
diff --git a/savevm.c b/savevm.c
index 19f7b0c..75fdd04 100644
--- a/savevm.c
+++ b/savevm.c
@@ -1515,6 +1515,23 @@ void qmp_log_dirty_bitmap(const char *filename, bool has_iterations,
     return;
 }
 
+BitmapLogStateInfo *qmp_query_log_dirty_bitmap(Error **errp)
+{
+    BitmapLogState *b = log_bitmap_get_current_state();
+    BitmapLogStateInfo *info = NULL;
+
+    if (b->state != LOG_BITMAP_STATE_ACTIVE) {
+        return info;
+    }
+
+    info = g_malloc0(sizeof(BitmapLogStateInfo));
+    info->current_iteration = b->current_iteration;
+    info->iterations = b->iterations;
+    info->period = b->current_period;
+
+    return info;
+}
+
 void qmp_xen_save_devices_state(const char *filename, Error **errp)
 {
     QEMUFile *f;
-- 
1.9.1

[Qemu-devel] [PATCH v6 4/6] BitmapLog: cancel mechanism for an already running dump bitmap process

[Sanidhya Kashyap]

Signed-off-by: Sanidhya Kashyap <sanidhya.iiith@gmail.com>
---

No functional change, except acronyms have been removed.

Removed acronyms, no functional change.

 hmp-commands.hx  | 14 ++++++++++++++
 hmp.c            |  5 +++++
 hmp.h            |  1 +
 qapi-schema.json |  9 +++++++++
 qmp-commands.hx  | 20 ++++++++++++++++++++
 savevm.c         | 14 ++++++++++++++
 6 files changed, 63 insertions(+)

diff --git a/hmp-commands.hx b/hmp-commands.hx
index d336f20..b253239 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -1804,6 +1804,20 @@ STEXI
 dumps the writable working set of a VM's memory to a file
 ETEXI
 
+	{
+	.name       = "log_dirty_bitmap_cancel",
+	.args_type  = "",
+	.params     = "",
+	.help       = "cancel the current bitmap dump process",
+	.mhandler.cmd = hmp_log_dirty_bitmap_cancel,
+},
+
+STEXI
+@item log_dirty_bitmap_cancel
+@findex log_dirty_bitmap_cancel
+Cancel the current bitmap dump process
+ETEXI
+
 STEXI
 @end table
 ETEXI
diff --git a/hmp.c b/hmp.c
index 4533dcd..80b4e5d 100644
--- a/hmp.c
+++ b/hmp.c
@@ -1336,6 +1336,11 @@ void hmp_log_dirty_bitmap(Monitor *mon, const QDict *qdict)
     }
 }
 
+void hmp_log_dirty_bitmap_cancel(Monitor *mon, const QDict *qdict)
+{
+    qmp_log_dirty_bitmap_cancel(NULL);
+}
+
 void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict)
 {
     Error *err = NULL;
diff --git a/hmp.h b/hmp.h
index 02e8ee4..fcfb10f 100644
--- a/hmp.h
+++ b/hmp.h
@@ -96,6 +96,7 @@ void hmp_object_add(Monitor *mon, const QDict *qdict);
 void hmp_object_del(Monitor *mon, const QDict *qdict);
 void hmp_info_memdev(Monitor *mon, const QDict *qdict);
 void hmp_log_dirty_bitmap(Monitor *mon, const QDict *qdict);
+void hmp_log_dirty_bitmap_cancel(Monitor *mon, const QDict *qdict);
 void object_add_completion(ReadLineState *rs, int nb_args, const char *str);
 void object_del_completion(ReadLineState *rs, int nb_args, const char *str);
 void device_add_completion(ReadLineState *rs, int nb_args, const char *str);
diff --git a/qapi-schema.json b/qapi-schema.json
index d1b44f4..0e90e9a 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -3546,3 +3546,12 @@
 # Since 2.2
 ##
 { 'command': 'query-log-dirty-bitmap', 'returns': 'BitmapLogStateInfo' }
+
+##
+# @log-dirty-bitmap-cancel
+#
+# cancel the dirty bitmap logging process
+#
+# Since 2.2
+##
+{ 'command': 'log-dirty-bitmap-cancel' }
diff --git a/qmp-commands.hx b/qmp-commands.hx
index 9582bc7..890a393 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -3811,3 +3811,23 @@ Example:
             "iterations": 10,
             "period": 100 } }
 EQMP
+
+	{
+        .name       = "log-dirty-bitmap-cancel",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_marshal_input_log_dirty_bitmap_cancel,
+    },
+
+SQMP
+log-dirty-bitmap-cancel
+-----------------------
+
+Cancel the current bitmap dump process.
+
+Arguments: None.
+
+Example:
+
+-> { "execute": "log-dirty-bitmap-cancel" }
+<- { "return": {} }
+EQMP
diff --git a/savevm.c b/savevm.c
index 75fdd04..51995b4 100644
--- a/savevm.c
+++ b/savevm.c
@@ -1532,6 +1532,20 @@ BitmapLogStateInfo *qmp_query_log_dirty_bitmap(Error **errp)
     return info;
 }
 
+void qmp_log_dirty_bitmap_cancel(Error **errp)
+{
+    BitmapLogState *b = log_bitmap_get_current_state();
+    int old_state;
+    do {
+        old_state = b->state;
+        if (old_state != LOG_BITMAP_STATE_ACTIVE) {
+            break;
+        }
+        log_bitmap_set_status(b, old_state,
+                                 LOG_BITMAP_STATE_CANCELING);
+    } while (b->state != LOG_BITMAP_STATE_CANCELING);
+}
+
 void qmp_xen_save_devices_state(const char *filename, Error **errp)
 {
     QEMUFile *f;
-- 
1.9.1

[Qemu-devel] [PATCH v6 5/6] BitmapLog: set the period of the dump bitmap process

[Sanidhya Kashyap]

Signed-off-by: Sanidhya Kashyap <sanidhya.iiith@gmail.com>
---

Removed acronyms, no functional change.

 hmp-commands.hx  | 15 +++++++++++++++
 hmp.c            | 12 ++++++++++++
 hmp.h            |  1 +
 qapi-schema.json | 12 ++++++++++++
 qmp-commands.hx  | 24 ++++++++++++++++++++++++
 savevm.c         | 14 ++++++++++++++
 6 files changed, 78 insertions(+)

diff --git a/hmp-commands.hx b/hmp-commands.hx
index b253239..c480309 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -1818,6 +1818,21 @@ STEXI
 Cancel the current bitmap dump process
 ETEXI
 
+    {
+        .name       = "log_dirty_bitmap_set_period",
+        .args_type  = "period:i",
+        .params     = "period",
+        .help       = "set the period for bitmap dump process\n\t\t\t"
+                      "period: the new period value to replace the existing",
+        .mhandler.cmd = hmp_log_dirty_bitmap_set_period,
+    },
+
+STEXI
+@item log_dirty_bitmap_set_period @var{period}
+@findex log_dirty_bitmap_set_period
+Set the period to @var{period} (int) for bitmap dump process.
+ETEXI
+
 STEXI
 @end table
 ETEXI
diff --git a/hmp.c b/hmp.c
index 80b4e5d..4f9b807 100644
--- a/hmp.c
+++ b/hmp.c
@@ -1341,6 +1341,18 @@ void hmp_log_dirty_bitmap_cancel(Monitor *mon, const QDict *qdict)
     qmp_log_dirty_bitmap_cancel(NULL);
 }
 
+void hmp_log_dirty_bitmap_set_period(Monitor *mon, const QDict *qdict)
+{
+    int64_t period = qdict_get_int(qdict, "period");
+    Error *err = NULL;
+    qmp_log_dirty_bitmap_set_period(period, &err);
+    if (err) {
+        monitor_printf(mon, "log_dirty_bitmap_set_period: %s\n",
+                       error_get_pretty(err));
+        error_free(err);
+    }
+}
+
 void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict)
 {
     Error *err = NULL;
diff --git a/hmp.h b/hmp.h
index fcfb10f..a5a0571 100644
--- a/hmp.h
+++ b/hmp.h
@@ -97,6 +97,7 @@ void hmp_object_del(Monitor *mon, const QDict *qdict);
 void hmp_info_memdev(Monitor *mon, const QDict *qdict);
 void hmp_log_dirty_bitmap(Monitor *mon, const QDict *qdict);
 void hmp_log_dirty_bitmap_cancel(Monitor *mon, const QDict *qdict);
+void hmp_log_dirty_bitmap_set_period(Monitor *mon, const QDict *qdict);
 void object_add_completion(ReadLineState *rs, int nb_args, const char *str);
 void object_del_completion(ReadLineState *rs, int nb_args, const char *str);
 void device_add_completion(ReadLineState *rs, int nb_args, const char *str);
diff --git a/qapi-schema.json b/qapi-schema.json
index 0e90e9a..87b9297 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -3555,3 +3555,15 @@
 # Since 2.2
 ##
 { 'command': 'log-dirty-bitmap-cancel' }
+
+##
+# @log-dirty-bitmap-set-period
+#
+# sets the period of the dirty bitmap logging process
+# @frequency: the updated period value (in milliseconds).
+# The min and max values are 10 and 100000 respectively.
+#
+# Since 2.2
+##
+{ 'command': 'log-dirty-bitmap-set-period',
+  'data': { 'period': 'int' } }
diff --git a/qmp-commands.hx b/qmp-commands.hx
index 890a393..f229a2f 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -3831,3 +3831,27 @@ Example:
 -> { "execute": "log-dirty-bitmap-cancel" }
 <- { "return": {} }
 EQMP
+
+    {
+        .name       = "log-dirty-bitmap-set-period",
+        .args_type  = "period:i",
+        .mhandler.cmd_new = qmp_marshal_input_log_dirty_bitmap_set_period,
+    },
+
+SQMP
+log-dirty-bitmap-set-period
+---------------------------
+
+Update the period for the remaining iterations.
+
+Arguments:
+
+- "period": The updated period (json-int) (in milliseconds).
+            The min and max values are 10 and 100000 respectively.
+
+Example:
+
+-> { "execute": "log-dirty-bitmap-set-period", "arguments": { "period": 1024 } }
+<- { "return": {} }
+
+EQMP
diff --git a/savevm.c b/savevm.c
index 51995b4..17af116 100644
--- a/savevm.c
+++ b/savevm.c
@@ -1546,6 +1546,20 @@ void qmp_log_dirty_bitmap_cancel(Error **errp)
     } while (b->state != LOG_BITMAP_STATE_CANCELING);
 }
 
+void qmp_log_dirty_bitmap_set_period(int64_t period, Error **errp)
+{
+    BitmapLogState *b = log_bitmap_get_current_state();
+    Error *local_err = NULL;
+    if (value_in_range(period, MIN_PERIOD_VALUE,
+                       MAX_PERIOD_VALUE, "period", &local_err)) {
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+    b->current_period = period;
+}
+
 void qmp_xen_save_devices_state(const char *filename, Error **errp)
 {
     QEMUFile *f;
-- 
1.9.1

[Qemu-devel] [PATCH v6 6/6] BitmapLog: python script for extracting bitmap from a binary file

[Sanidhya Kashyap]

I have modified the script to support the dump of the images to the file. Earlier,
everything was saved to the memory and later the processing was taking place. Now,
I have tried to solve that issue with only using the required memory.

After discussion with David, I have tried to select a base 2 matrix dimension like
512 X 512 or 1024 X 512 etc for the dumping of the bitmap. But, I am still supporting
the default square root based method of dimension selection.


Signed-off-by: Sanidhya Kashyap <sanidhya.iiith@gmail.com>
---

 scripts/extract-bitmap.py | 213 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 213 insertions(+)
 create mode 100755 scripts/extract-bitmap.py

diff --git a/scripts/extract-bitmap.py b/scripts/extract-bitmap.py
new file mode 100755
index 0000000..9a5a481
--- /dev/null
+++ b/scripts/extract-bitmap.py
@@ -0,0 +1,213 @@
+#!/usr/bin/python
+# This python script helps in extracting the dirty bitmap present
+# in the file after executing the log-dirty-bitmap command either
+# from the qmp or hmp interface. This file only processes binary
+# file obtained via command.
+#
+# Copyright (C) 2014 Sanidhya Kashyap <sanidhya.iiith@gmail.com>
+#
+# Authors:
+#       Sanidhya Kashyap
+#
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or later.
+
+import struct
+import argparse
+from functools import partial
+from math import sqrt
+from numpy import array
+from pylab import figure, imshow, savefig, gray, xlim, ylim
+from os import path, makedirs
+
+long_bytes = 8
+byte_size = 8
+int_bytes = 4
+block_list = []
+total_blocks = 0
+
+def get_unsigned_long_integer(value):
+	return struct.unpack('<Q', value)[0]
+
+def get_long_integer(value):
+	return struct.unpack('<q', value)[0]
+
+def get_integer(value):
+	return struct.unpack('<i', value)[0]
+
+def get_char(value):
+	return struct.unpack('<c', value)[0]
+
+def get_string(value, length):
+	name = struct.unpack('<'+str(length)+'s', value)[0]
+	for i in range(len(name)):
+		if name[i] == '\x00':
+			return name[:i]
+
+def dec2bin(decimal):
+    bin_value = bin(decimal)[2:]
+    if len(bin_value) < long_bytes * byte_size:
+        add_zeroes = long_bytes * byte_size - len(bin_value)
+        for i in range(add_zeroes):
+            bin_value += "0"
+    return str(bin_value)
+
+def get_bitmap_length(ram_bitmap_pages):
+    bitmap_length = ram_bitmap_pages / (long_bytes * byte_size)
+    if ram_bitmap_pages % (long_bytes * byte_size) != 0:
+        bitmap_length += 1
+    return bitmap_length
+
+def get_block_info():
+    print "Select any one of the following:"
+    for i in range(len(block_list)):
+        print str(i) + " " + block_list[i]['name']
+    return int(raw_input('Enter the number: '))
+
+def get_matrix(all_digits, y, x):
+    v = []
+    xlim(xmin = 0, xmax = x)
+    ylim(ymin = 0, ymax = y)
+    for i in range(y):
+        v1 = []
+        for j in range(x):
+            v1.append(int(all_digits[i * x + j]))
+        v.append(v1)
+    return v
+
+def get_matrix_base2(all_digits):
+    l = len(all_digits)
+    sqrtvalue = int(sqrt(l))
+    x = 2 ** (sqrtvalue.bit_length() - 1)
+    y = x * 2
+    if (x * y - l < 0):
+        x = y
+    for i in range(x * y - l):
+        all_digits += "0"
+
+    return get_matrix(all_digits, x, y);
+
+def get_matrix_sqrt(all_digits):
+    l = len(all_digits)
+    sqrtvalue = int(sqrt(l))
+    for i in range(sqrtvalue * (sqrtvalue + 1) - l):
+        all_digits += "0"
+
+    if sqrtvalue * sqrtvalue == l:
+        return get_matrix(all_digits, sqrtvalue, sqrtvalue)
+    else:
+        return get_matrix(all_digits, sqrtvalue, sqrtvalue + 1)
+
+def dump_ram_block_info(infile):
+    total_blocks = get_integer(infile.read(int_bytes))
+    for i in range(total_blocks):
+        block_name_length = get_integer(infile.read(int_bytes))
+        block_name = get_string(infile.read(block_name_length), block_name_length)
+        block_offset = get_unsigned_long_integer(infile.read(long_bytes))
+        block_length = get_unsigned_long_integer(infile.read(long_bytes))
+        block_list.append(dict(name=block_name, offset=block_offset, length=block_length))
+
+def generate_image(all_digits, num, debug, dir, base):
+    v = []
+
+    if base is False:
+        v = get_matrix_sqrt(all_digits)
+    else:
+        v = get_matrix_base2(all_digits)
+
+    im_array = array(v)
+    figure(num)
+    imshow(im_array, cmap=gray())
+
+    filename=dir + "/" + "out_" + str(num) + ".png"
+    del v
+    savefig(filename)
+    if debug is True:
+        print 'file ' + filename + ' dumped'
+
+
+def dump_bitmap(args):
+    marker = 'M'
+    count = 0
+    value = ' '
+    block_offset = 0
+    block_length = 0
+    block_num = 0
+    current_ram_bitmap_pages = 0
+    prev_ram_bitmap_pages = 0
+    infile = open(format(args.infile), 'rb')
+    debug = args.debug
+    dump_all = args.dump_all
+    dir = args.dir
+    base = args.base
+
+    if not path.exists(dir):
+        makedirs(dir)
+
+    while True:
+        if len(value) == 0  or marker != 'M':
+            print "issue with the dump"
+            return
+        bitmap_page_raw_value = infile.read(long_bytes)
+        if not bitmap_page_raw_value:
+            break
+        current_ram_bitmap_pages = get_long_integer(bitmap_page_raw_value)
+        if current_ram_bitmap_pages != prev_ram_bitmap_pages:
+            prev_ram_bitmap_pages = current_ram_bitmap_pages
+            dump_ram_block_info(infile)
+            # asking what should be printed
+            if dump_all is False:
+                if count == 0:
+                    block_num = get_block_info()
+                    if debug is True:
+                        print block_list[block_num]['name'] + ' selected'
+                else:
+                    x = block_num
+                    y = len(block_list) / total_blocks
+                    block_num = total_blocks * (y-1) + x
+                block_offset = block_list[block_num]['offset']
+                block_length = block_list[block_num]['length']
+                if debug is True:
+                    print 'total ram bitmap pages: ' + str(current_ram_bitmap_pages)
+                    print block_list[block_num]['name'] + ' offset: ' + str(block_offset)
+                    print block_list[block_num]['name'] + ' length: ' + str(block_length)
+
+        bitmap_length = get_bitmap_length(current_ram_bitmap_pages)
+        bitmap_raw_value = infile.read(long_bytes * bitmap_length)
+        if not bitmap_raw_value:
+            break
+
+        count+=1
+        all_digits=""
+        for i in range(bitmap_length):
+            mark = i * long_bytes
+            all_digits += dec2bin(get_unsigned_long_integer(bitmap_raw_value[mark : mark + long_bytes]))
+
+        if dump_all is False:
+            generate_image(all_digits[block_offset : block_offset + block_length], count, debug, dir, base)
+        else:
+            generate_image(all_digits, count, debug, dir, base)
+        value = infile.read(1)
+        marker = get_char(value)
+    infile.close()
+
+def main():
+    extracter = argparse.ArgumentParser(description='Extract dirty bitmap from binary file.')
+    extracter.add_argument('-f', '--file', dest='infile',
+            help='Input file to extract the bitmap', metavar='FILE')
+    extracter.add_argument('-a', '--all', action='store_true', dest='dump_all', default=False,
+            help='Use all memory blocks for the figure')
+    extracter.add_argument('-d', '--debug', action='store_true', dest='debug', default=False,
+            help='print the debug info')
+    extracter.add_argument('-dir', dest='dir', default='.',
+            help='directory to store the image files')
+    extracter.add_argument('-b', '--base2', dest='base', action='store_true', default=False,
+            help='dump the bitmap array in powers of 2')
+
+    args = extracter.parse_args()
+    print 'The filename is {}'.format(args.infile)
+
+    dump_bitmap(args)
+
+if __name__ == '__main__':
+    main()
-- 
1.9.1