[Qemu-devel] [PATCH 6/9 v2] Add API to create page

[qiaonuohan@cn.fujitsu.com]

From: Qiao Nuohan <qiaonuohan@cn.fujitsu.com>

Add API to get data of page desc and page data and save them into tmp files.
The following patch will use these functions to gather data of page desc and
page data, then write them into vmcore

Signed-off-by: Qiao Nuohan <qiaonuohan@cn.fujitsu.com>
Reviewed-by: Zhang Xiaohe <zhangxh@cn.fujitsu.com>
---
 dump.c                |  259 +++++++++++++++++++++++++++++++++++++++++++++++++
 include/sysemu/dump.h |   32 ++++++
 2 files changed, 291 insertions(+), 0 deletions(-)

diff --git a/dump.c b/dump.c
index 998d71e..ebfb190 100644
--- a/dump.c
+++ b/dump.c
@@ -877,6 +877,265 @@ static int create_dump_bitmap(DumpState *s)
     return 0;
 }
 
+/*
+ * create two tmpfile and save page_desc and page_data
+ */
+static int prepare_pages(DumpState *s)
+{
+    int ret;
+    struct cache_data *page_desc;
+    struct cache_data *page_data;
+
+    page_desc = g_malloc0(sizeof(struct cache_data));
+
+    page_data = g_malloc0(sizeof(struct cache_data));
+
+    ret = init_cache_data(page_desc, FILENAME_PAGE_DESC);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to init page_desc.\n");
+        return -1;
+    }
+    s->page_desc = page_desc;
+
+    ret = init_cache_data(page_data, FILENAME_PAGE_DATA);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to init page_desc.\n");
+        return -1;
+    }
+    s->page_data = page_data;
+
+    return 0;
+}
+
+/*
+ * memory should be read page by page, or it may exceed the boundary and
+ * fail to read
+ */
+static int readmem(void *bufptr, ram_addr_t addr, size_t size, DumpState *s)
+{
+    RAMBlock *block;
+
+    block = s->block;
+
+    while (block) {
+        if ((addr >= block->offset) &&
+            (addr + size <= block->offset + block->length)) {
+            memcpy(bufptr, block->host + (addr - block->offset), size);
+            return 0;
+        } else {
+            block = QTAILQ_NEXT(block, next);
+        }
+    }
+
+    return -1;
+}
+
+/*
+ * check if the page is all 0
+ */
+static inline int is_zero_page(unsigned char *buf, long page_size)
+{
+    size_t i;
+
+    for (i = 0; i < page_size; i++) {
+        if (buf[i]) {
+            return 0;
+        }
+    }
+
+    return 1;
+}
+
+static int create_pages(DumpState *s)
+{
+    int ret;
+    unsigned long long pfn;
+    unsigned char buf[s->page_size];
+    unsigned char *buf_out = NULL;
+    unsigned long len_buf_out;
+    unsigned long size_out;
+    int zero_page;
+    struct page_desc pd, pd_zero;
+    off_t offset_desc, offset_data;
+    unsigned long len_buf_out_zlib, len_buf_out_lzo, len_buf_out_snappy;
+
+    ret = 0;
+
+    ret = prepare_pages(s);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to prepare pages.\n");
+        goto out;
+    }
+
+    /* init buf_out */
+    len_buf_out_zlib = len_buf_out_lzo = len_buf_out_snappy = 0;
+
+    /* buf size for zlib */
+    len_buf_out_zlib = compressBound(s->page_size);
+
+    /* buf size for lzo */
+#ifdef CONFIG_LZO
+    if (s->flag_compress & DUMP_DH_COMPRESSED_LZO) {
+        if (lzo_init() != LZO_E_OK) {
+            ret = -1;
+            dump_error(s, "dump: failed to use lzo compression");
+            goto out;
+        }
+    }
+
+    lzo_bytep wrkmem;
+
+    wrkmem = g_malloc(LZO1X_1_MEM_COMPRESS);
+
+    len_buf_out_lzo = s->page_size + s->page_size / 16 + 64 + 3;
+#endif
+
+    /* buf size for snappy */
+#ifdef CONFIG_SNAPPY
+    len_buf_out_snappy = snappy_max_compressed_length(s->page_size);
+#endif
+
+    /* get the biggest that can store all kinds of compressed page */
+    len_buf_out = MAX(len_buf_out_zlib,
+                    MAX(len_buf_out_lzo, len_buf_out_snappy));
+
+    buf_out = g_malloc(len_buf_out);
+
+    /* get offset of page_desc and page_data in dump file */
+    offset_desc = s->offset_page;
+    offset_data = offset_desc + sizeof(page_desc_t) * s->num_dumpable;
+
+    /*
+     * init zero page's page_desc and page_data, and all zero pages
+     * will use the same page_data
+     */
+    pd_zero.size = s->page_size;
+    pd_zero.flags = 0;
+    pd_zero.offset = offset_data;
+    pd_zero.page_flags = 0;
+    memset(buf, 0, pd_zero.size);
+    write_cache(s->page_data, buf, pd_zero.size);
+    offset_data += pd_zero.size;
+
+    for (pfn = 0; pfn < s->max_mapnr; pfn++) {
+        /* check whether the page is dumpable */
+        if (!is_bit_set(s->dump_bitmap2, pfn)) {
+            continue;
+        }
+
+        memset(buf, 0, s->page_size);
+        ret = readmem(buf, pfn_to_paddr(pfn, s->page_shift), s->page_size, s);
+        if (ret < 0) {
+            dump_error(s, "dump: failed to read memory ");
+            goto out;
+        }
+
+        /* check zero page */
+        zero_page = is_zero_page(buf, s->page_size);
+        if (zero_page) {
+            write_cache(s->page_desc, &pd_zero, sizeof(page_desc_t));
+        } else {
+            /*
+             * not zero page, then:
+             * 1. compress the page
+             * 2. write the compressed page into the cache of page_data
+             * 3. get page desc of the compressed page and write it into the
+             *    cache of page_desc
+             */
+            size_out = len_buf_out;
+            if ((s->flag_compress & DUMP_DH_COMPRESSED_ZLIB) &&
+                    (compress2(buf_out, &size_out, buf, s->page_size,
+                    Z_BEST_SPEED) == Z_OK) && (size_out < s->page_size)) {
+                pd.flags = DUMP_DH_COMPRESSED_ZLIB;
+                pd.size  = size_out;
+
+                ret = write_cache(s->page_data, buf_out, pd.size);
+                if (ret < 0) {
+                    dump_error(s, "dump: failed to write page data(zlib).\n");
+                    goto out;
+                }
+#ifdef CONFIG_LZO
+            } else if ((s->flag_compress & DUMP_DH_COMPRESSED_LZO) &&
+                    (lzo1x_1_compress(buf, s->page_size, buf_out, &size_out,
+                    wrkmem) == LZO_E_OK) && (size_out < s->page_size)) {
+                pd.flags = DUMP_DH_COMPRESSED_LZO;
+                pd.size  = size_out;
+
+                ret = write_cache(s->page_data, buf_out, pd.size);
+                if (ret < 0) {
+                    dump_error(s, "dump: failed to write page data(lzo).\n");
+                    goto out;
+                }
+#endif
+#ifdef CONFIG_SNAPPY
+            } else if ((s->flag_compress & DUMP_DH_COMPRESSED_SNAPPY) &&
+                    (snappy_compress((char *)buf, s->page_size, (char *)buf_out,
+                    (size_t *)&size_out) == SNAPPY_OK) &&
+                    (size_out < s->page_size)) {
+                pd.flags = DUMP_DH_COMPRESSED_SNAPPY;
+                pd.size  = size_out;
+
+                ret = write_cache(s->page_data, buf_out, pd.size);
+                if (ret < 0) {
+                    dump_error(s, "dump: failed to write page data(snappy).\n");
+                    goto out;
+                }
+#endif
+            } else {
+                pd.flags = 0;
+                pd.size = s->page_size;
+
+                ret = write_cache(s->page_data, buf, pd.size);
+                if (ret < 0) {
+                    dump_error(s, "dump: failed to write page data.\n");
+                    goto out;
+                }
+            }
+
+            /* get and write page desc here */
+            pd.page_flags = 0;
+            pd.offset = offset_data;
+            offset_data += pd.size;
+
+            ret = write_cache(s->page_desc, &pd, sizeof(page_desc_t));
+            if (ret < 0) {
+                dump_error(s, "dump: failed to write page desc.\n");
+                goto out;
+            }
+        }
+    }
+
+    ret = sync_cache(s->page_desc);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to sync cache for page_desc.\n");
+        goto out;
+    }
+    ret = sync_cache(s->page_data);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to sync cache for page_data.\n");
+        goto out;
+    }
+
+   /* get size of page_desc and page_data, then reset their offset */
+    s->page_desc_size = s->page_desc->offset;
+    s->page_data_size = s->page_data->offset;
+    s->page_desc->offset = 0;
+    s->page_data->offset = 0;
+
+out:
+#ifdef CONFIG_LZO
+    if (wrkmem) {
+        g_free(wrkmem);
+    }
+#endif
+
+    if (buf_out) {
+        g_free(buf_out);
+    }
+
+    return ret;
+}
+
 static int dump_init(DumpState *s, int fd, bool paging, bool has_filter,
                      int64_t begin, int64_t length, Error **errp)
 {
diff --git a/include/sysemu/dump.h b/include/sysemu/dump.h
index 93c35bc..7cb74d6 100644
--- a/include/sysemu/dump.h
+++ b/include/sysemu/dump.h
@@ -25,8 +25,23 @@
 #include "qapi/error.h"
 #include "qmp-commands.h"
 #include "dump_bitmap.h"
+#include "cache_data.h"
 
 #include <sys/utsname.h>
+#include <zlib.h>
+#ifdef CONFIG_LZO
+#include <lzo/lzo1x.h>
+#endif
+#ifdef CONFIG_SNAPPY
+#include <snappy-c.h>
+#endif
+
+/*
+ * flag used in page desc of kdump-compressed format
+ */
+#define DUMP_DH_COMPRESSED_ZLIB     (0x1)
+#define DUMP_DH_COMPRESSED_LZO      (0x2)
+#define DUMP_DH_COMPRESSED_SNAPPY   (0x4)
 
 #define KDUMP_SIGNATURE             "KDUMP   "
 #define SIG_LEN                     (sizeof(KDUMP_SIGNATURE) - 1)
@@ -36,10 +51,14 @@
 #define ARCH_PFN_OFFSET             (0)
 #define FILENAME_BITMAP1            "kdump_bitmap1_XXXXXX"
 #define FILENAME_BITMAP2            "kdump_bitmap2_XXXXXX"
+#define FILENAME_PAGE_DESC          "kdump_page_desc_XXXXXX"
+#define FILENAME_PAGE_DATA          "kdump_page_data_XXXXXX"
 
 #define divideup(x, y)              (((x) + ((y) - 1)) / (y))
 #define paddr_to_pfn(X, page_shift) \
     (((unsigned long long)(X) >> (page_shift)) - ARCH_PFN_OFFSET)
+#define pfn_to_paddr(X, page_shift) \
+    (((unsigned long long)(X) + ARCH_PFN_OFFSET) << (page_shift))
 
 typedef struct ArchDumpInfo {
     int d_machine;  /* Architecture */
@@ -120,6 +139,14 @@ struct kdump_sub_header64 {
     uint64_t size_eraseinfo;        /* header_version 5 and later */
 };
 
+/* descriptor of each page for vmcore */
+typedef struct page_desc {
+    off_t offset;                   /* the offset of the page data*/
+    unsigned int size;              /* the size of this dump page */
+    unsigned int flags;             /* flags */
+    unsigned long long page_flags;  /* page flags */
+} page_desc_t;
+
 typedef struct DumpState {
     ArchDumpInfo dump_info;
     MemoryMappingList list;
@@ -146,6 +173,7 @@ typedef struct DumpState {
     void *kh;
     unsigned long long num_dumpable;
     off_t offset_sub_header;
+    int flag_compress;
 
     off_t offset_dump_bitmap;
     unsigned long len_dump_bitmap;
@@ -153,6 +181,10 @@ typedef struct DumpState {
     struct dump_bitmap *dump_bitmap2;
 
     off_t offset_page;
+    unsigned long long page_desc_size;
+    unsigned long long page_data_size;
+    struct cache_data *page_desc;
+    struct cache_data *page_data;
 } DumpState;
 
 int cpu_get_dump_info(ArchDumpInfo *info);
-- 
1.7.1