* [RFC PATCH 4/6] migration: Add zstd support in multi-thread compression
2020-11-09 9:08 [RFC PATCH 0/6] migration: Multi-thread compression with zstd method Zeyu Jin
@ 2020-11-09 9:08 ` Zeyu Jin
0 siblings, 0 replies; 4+ messages in thread
From: Zeyu Jin @ 2020-11-09 9:08 UTC (permalink / raw)
To: quintela, dgilbert; +Cc: Ying Fang, qemu-devel, Zeyu Jin, zhang.zhanghailiang
This patch enables zstd option in multi-thread compression.
Signed-off-by: Zeyu Jin <jinzeyu@huawei.com>
Signed-off-by: Ying Fang <fangying1@huawei.com>
---
hw/core/qdev-properties-system.c | 2 +-
migration/ram.c | 128 ++++++++++++++++++++++++++++++-
qapi/migration.json | 2 +-
3 files changed, 129 insertions(+), 3 deletions(-)
diff --git a/hw/core/qdev-properties-system.c b/hw/core/qdev-properties-system.c
index d757b2cd70..54760d94b3 100644
--- a/hw/core/qdev-properties-system.c
+++ b/hw/core/qdev-properties-system.c
@@ -667,7 +667,7 @@ const PropertyInfo qdev_prop_multifd_compression = {
const PropertyInfo qdev_prop_compress_method = {
.name = "CompressMethod",
.description = "multi-thread compression method, "
- "zlib",
+ "zlib/zstd",
.enum_table = &CompressMethod_lookup,
.get = qdev_propinfo_get_enum,
.set = qdev_propinfo_set_enum,
diff --git a/migration/ram.c b/migration/ram.c
index 94a7422204..a732d80db2 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -57,6 +57,10 @@
#include "qemu/iov.h"
#include "multifd.h"
+#ifdef CONFIG_ZSTD
+#include <zstd.h>
+#include <zstd_errors.h>
+#endif
/***********************************************************/
/* ram save/restore */
@@ -424,6 +428,11 @@ struct CompressParam {
/* for zlib compression */
z_stream stream;
+#ifdef CONFIG_ZSTD
+ ZSTD_CStream *zstd_cs;
+ ZSTD_inBuffer in;
+ ZSTD_outBuffer out;
+#endif
};
typedef struct CompressParam CompressParam;
@@ -438,6 +447,12 @@ struct DecompressParam {
/* for zlib compression */
z_stream stream;
+
+#ifdef CONFIG_ZSTD
+ ZSTD_DStream *zstd_ds;
+ ZSTD_inBuffer in;
+ ZSTD_outBuffer out;
+#endif
};
typedef struct DecompressParam DecompressParam;
@@ -571,6 +586,102 @@ static int zlib_check_len(int len)
return len < 0 || len > compressBound(TARGET_PAGE_SIZE);
}
+#ifdef CONFIG_ZSTD
+static int zstd_save_setup(CompressParam *param)
+{
+ int res;
+ param->zstd_cs = ZSTD_createCStream();
+ if (!param->zstd_cs) {
+ return -1;
+ }
+ res = ZSTD_initCStream(param->zstd_cs, migrate_compress_level());
+ if (ZSTD_isError(res)) {
+ return -1;
+ }
+ return 0;
+}
+static void zstd_save_cleanup(CompressParam *param)
+{
+ ZSTD_freeCStream(param->zstd_cs);
+ param->zstd_cs = NULL;
+}
+static ssize_t zstd_compress_data(CompressParam *param, size_t size)
+{
+ int ret;
+ uint8_t *dest = NULL;
+ uint8_t *p = param->originbuf;
+ QEMUFile *f = f = param->file;
+ ssize_t blen = qemu_put_compress_start(f, &dest);
+ if (blen < ZSTD_compressBound(size)) {
+ return -1;
+ }
+ param->out.dst = dest;
+ param->out.size = blen;
+ param->out.pos = 0;
+ param->in.src = p;
+ param->in.size = size;
+ param->in.pos = 0;
+ do {
+ ret = ZSTD_compressStream2(param->zstd_cs, ¶m->out,
+ ¶m->in, ZSTD_e_end);
+ } while (ret > 0 && (param->in.size - param->in.pos > 0)
+ && (param->out.size - param->out.pos > 0));
+ if (ret > 0 && (param->in.size - param->in.pos > 0)) {
+ return -1;
+ }
+ if (ZSTD_isError(ret)) {
+ return -1;
+ }
+ blen = param->out.pos;
+ qemu_put_compress_end(f, blen);
+ return blen + sizeof(int32_t);
+}
+static int zstd_load_setup(DecompressParam *param)
+{
+ int ret;
+ param->zstd_ds = ZSTD_createDStream();
+ if (!param->zstd_ds) {
+ return -1;
+ }
+ ret = ZSTD_initDStream(param->zstd_ds);
+ if (ZSTD_isError(ret)) {
+ return -1;
+ }
+ return 0;
+}
+static void zstd_load_cleanup(DecompressParam *param)
+{
+ ZSTD_freeDStream(param->zstd_ds);
+ param->zstd_ds = NULL;
+}
+static int
+zstd_decompress_data(DecompressParam *param, uint8_t *dest, size_t size)
+{
+ int ret;
+ param->out.dst = dest;
+ param->out.size = size;
+ param->out.pos = 0;
+ param->in.src = param->compbuf;
+ param->in.size = param->len;
+ param->in.pos = 0;
+ do {
+ ret = ZSTD_decompressStream(param->zstd_ds, ¶m->out, ¶m->in);
+ } while (ret > 0 && (param->in.size - param->in.pos > 0)
+ && (param->out.size - param->out.pos > 0));
+ if (ret > 0 && (param->in.size - param->in.pos > 0)) {
+ return -1;
+ }
+ if (ZSTD_isError(ret)) {
+ return -1;
+ }
+ return ret;
+}
+static int zstd_check_len(int len)
+{
+ return len < 0 || len > ZSTD_compressBound(TARGET_PAGE_SIZE);
+}
+#endif
+
static int set_compress_ops(void)
{
compress_ops = g_new0(MigrationCompressOps, 1);
@@ -581,9 +692,16 @@ static int set_compress_ops(void)
compress_ops->save_cleanup = zlib_save_cleanup;
compress_ops->compress_data = zlib_compress_data;
break;
+#ifdef CONFIG_ZSTD
+ case COMPRESS_METHOD_ZSTD:
+ compress_ops->save_setup = zstd_save_setup;
+ compress_ops->save_cleanup = zstd_save_cleanup;
+ compress_ops->compress_data = zstd_compress_data;
+ break;
+#endif
default:
return -1;
- }
+ }
return 0;
}
@@ -599,6 +717,14 @@ static int set_decompress_ops(void)
decompress_ops->decompress_data = zlib_decompress_data;
decompress_ops->check_len = zlib_check_len;
break;
+#ifdef CONFIG_ZSTD
+ case COMPRESS_METHOD_ZSTD:
+ decompress_ops->load_setup = zstd_load_setup;
+ decompress_ops->load_cleanup = zstd_load_cleanup;
+ decompress_ops->decompress_data = zstd_decompress_data;
+ decompress_ops->check_len = zstd_check_len;
+ break;
+#endif
default:
return -1;
}
diff --git a/qapi/migration.json b/qapi/migration.json
index d262683a38..ac6d06c683 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -536,7 +536,7 @@
#
##
{ 'enum': 'CompressMethod',
- 'data': [ 'zlib' ] }
+ 'data': [ 'zlib', { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } ] }
##
# @BitmapMigrationBitmapAlias:
--
2.23.0
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [RFC PATCH 4/6] migration: Add zstd support in multi-thread compression
@ 2020-11-27 7:42 Zeyu Jin
2020-11-27 12:10 ` Markus Armbruster
0 siblings, 1 reply; 4+ messages in thread
From: Zeyu Jin @ 2020-11-27 7:42 UTC (permalink / raw)
To: quintela, dgilbert; +Cc: Ying Fang, qemu-devel, Zeyu Jin, zhang.zhanghailiang
This patch enables zstd option in multi-thread compression.
Signed-off-by: Zeyu Jin <jinzeyu@huawei.com>
Signed-off-by: Ying Fang <fangying1@huawei.com>
---
hw/core/qdev-properties-system.c | 2 +-
migration/ram.c | 128 ++++++++++++++++++++++++++++++-
qapi/migration.json | 2 +-
3 files changed, 129 insertions(+), 3 deletions(-)
diff --git a/hw/core/qdev-properties-system.c b/hw/core/qdev-properties-system.c
index d757b2cd70..54760d94b3 100644
--- a/hw/core/qdev-properties-system.c
+++ b/hw/core/qdev-properties-system.c
@@ -667,7 +667,7 @@ const PropertyInfo qdev_prop_multifd_compression = {
const PropertyInfo qdev_prop_compress_method = {
.name = "CompressMethod",
.description = "multi-thread compression method, "
- "zlib",
+ "zlib/zstd",
.enum_table = &CompressMethod_lookup,
.get = qdev_propinfo_get_enum,
.set = qdev_propinfo_set_enum,
diff --git a/migration/ram.c b/migration/ram.c
index 94a7422204..a732d80db2 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -57,6 +57,10 @@
#include "qemu/iov.h"
#include "multifd.h"
+#ifdef CONFIG_ZSTD
+#include <zstd.h>
+#include <zstd_errors.h>
+#endif
/***********************************************************/
/* ram save/restore */
@@ -424,6 +428,11 @@ struct CompressParam {
/* for zlib compression */
z_stream stream;
+#ifdef CONFIG_ZSTD
+ ZSTD_CStream *zstd_cs;
+ ZSTD_inBuffer in;
+ ZSTD_outBuffer out;
+#endif
};
typedef struct CompressParam CompressParam;
@@ -438,6 +447,12 @@ struct DecompressParam {
/* for zlib compression */
z_stream stream;
+
+#ifdef CONFIG_ZSTD
+ ZSTD_DStream *zstd_ds;
+ ZSTD_inBuffer in;
+ ZSTD_outBuffer out;
+#endif
};
typedef struct DecompressParam DecompressParam;
@@ -571,6 +586,102 @@ static int zlib_check_len(int len)
return len < 0 || len > compressBound(TARGET_PAGE_SIZE);
}
+#ifdef CONFIG_ZSTD
+static int zstd_save_setup(CompressParam *param)
+{
+ int res;
+ param->zstd_cs = ZSTD_createCStream();
+ if (!param->zstd_cs) {
+ return -1;
+ }
+ res = ZSTD_initCStream(param->zstd_cs, migrate_compress_level());
+ if (ZSTD_isError(res)) {
+ return -1;
+ }
+ return 0;
+}
+static void zstd_save_cleanup(CompressParam *param)
+{
+ ZSTD_freeCStream(param->zstd_cs);
+ param->zstd_cs = NULL;
+}
+static ssize_t zstd_compress_data(CompressParam *param, size_t size)
+{
+ int ret;
+ uint8_t *dest = NULL;
+ uint8_t *p = param->originbuf;
+ QEMUFile *f = f = param->file;
+ ssize_t blen = qemu_put_compress_start(f, &dest);
+ if (blen < ZSTD_compressBound(size)) {
+ return -1;
+ }
+ param->out.dst = dest;
+ param->out.size = blen;
+ param->out.pos = 0;
+ param->in.src = p;
+ param->in.size = size;
+ param->in.pos = 0;
+ do {
+ ret = ZSTD_compressStream2(param->zstd_cs, ¶m->out,
+ ¶m->in, ZSTD_e_end);
+ } while (ret > 0 && (param->in.size - param->in.pos > 0)
+ && (param->out.size - param->out.pos > 0));
+ if (ret > 0 && (param->in.size - param->in.pos > 0)) {
+ return -1;
+ }
+ if (ZSTD_isError(ret)) {
+ return -1;
+ }
+ blen = param->out.pos;
+ qemu_put_compress_end(f, blen);
+ return blen + sizeof(int32_t);
+}
+static int zstd_load_setup(DecompressParam *param)
+{
+ int ret;
+ param->zstd_ds = ZSTD_createDStream();
+ if (!param->zstd_ds) {
+ return -1;
+ }
+ ret = ZSTD_initDStream(param->zstd_ds);
+ if (ZSTD_isError(ret)) {
+ return -1;
+ }
+ return 0;
+}
+static void zstd_load_cleanup(DecompressParam *param)
+{
+ ZSTD_freeDStream(param->zstd_ds);
+ param->zstd_ds = NULL;
+}
+static int
+zstd_decompress_data(DecompressParam *param, uint8_t *dest, size_t size)
+{
+ int ret;
+ param->out.dst = dest;
+ param->out.size = size;
+ param->out.pos = 0;
+ param->in.src = param->compbuf;
+ param->in.size = param->len;
+ param->in.pos = 0;
+ do {
+ ret = ZSTD_decompressStream(param->zstd_ds, ¶m->out, ¶m->in);
+ } while (ret > 0 && (param->in.size - param->in.pos > 0)
+ && (param->out.size - param->out.pos > 0));
+ if (ret > 0 && (param->in.size - param->in.pos > 0)) {
+ return -1;
+ }
+ if (ZSTD_isError(ret)) {
+ return -1;
+ }
+ return ret;
+}
+static int zstd_check_len(int len)
+{
+ return len < 0 || len > ZSTD_compressBound(TARGET_PAGE_SIZE);
+}
+#endif
+
static int set_compress_ops(void)
{
compress_ops = g_new0(MigrationCompressOps, 1);
@@ -581,9 +692,16 @@ static int set_compress_ops(void)
compress_ops->save_cleanup = zlib_save_cleanup;
compress_ops->compress_data = zlib_compress_data;
break;
+#ifdef CONFIG_ZSTD
+ case COMPRESS_METHOD_ZSTD:
+ compress_ops->save_setup = zstd_save_setup;
+ compress_ops->save_cleanup = zstd_save_cleanup;
+ compress_ops->compress_data = zstd_compress_data;
+ break;
+#endif
default:
return -1;
- }
+ }
return 0;
}
@@ -599,6 +717,14 @@ static int set_decompress_ops(void)
decompress_ops->decompress_data = zlib_decompress_data;
decompress_ops->check_len = zlib_check_len;
break;
+#ifdef CONFIG_ZSTD
+ case COMPRESS_METHOD_ZSTD:
+ decompress_ops->load_setup = zstd_load_setup;
+ decompress_ops->load_cleanup = zstd_load_cleanup;
+ decompress_ops->decompress_data = zstd_decompress_data;
+ decompress_ops->check_len = zstd_check_len;
+ break;
+#endif
default:
return -1;
}
diff --git a/qapi/migration.json b/qapi/migration.json
index d262683a38..ac6d06c683 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -536,7 +536,7 @@
#
##
{ 'enum': 'CompressMethod',
- 'data': [ 'zlib' ] }
+ 'data': [ 'zlib', { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } ] }
##
# @BitmapMigrationBitmapAlias:
--
2.23.0
^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [RFC PATCH 4/6] migration: Add zstd support in multi-thread compression
2020-11-27 7:42 [RFC PATCH 4/6] migration: Add zstd support in multi-thread compression Zeyu Jin
@ 2020-11-27 12:10 ` Markus Armbruster
2020-11-28 7:06 ` Zeyu Jin
0 siblings, 1 reply; 4+ messages in thread
From: Markus Armbruster @ 2020-11-27 12:10 UTC (permalink / raw)
To: Zeyu Jin; +Cc: Ying Fang, zhang.zhanghailiang, qemu-devel, dgilbert, quintela
Zeyu Jin <jinzeyu@huawei.com> writes:
> This patch enables zstd option in multi-thread compression.
>
> Signed-off-by: Zeyu Jin <jinzeyu@huawei.com>
> Signed-off-by: Ying Fang <fangying1@huawei.com>
[...]
> diff --git a/qapi/migration.json b/qapi/migration.json
> index d262683a38..ac6d06c683 100644
> --- a/qapi/migration.json
> +++ b/qapi/migration.json
> @@ -536,7 +536,7 @@
> #
> ##
> { 'enum': 'CompressMethod',
> - 'data': [ 'zlib' ] }
> + 'data': [ 'zlib', { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } ] }
>
> ##
> # @BitmapMigrationBitmapAlias:
Please document the new enum value @zstd.
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [RFC PATCH 4/6] migration: Add zstd support in multi-thread compression
2020-11-27 12:10 ` Markus Armbruster
@ 2020-11-28 7:06 ` Zeyu Jin
0 siblings, 0 replies; 4+ messages in thread
From: Zeyu Jin @ 2020-11-28 7:06 UTC (permalink / raw)
To: Markus Armbruster
Cc: Ying Fang, zhang.zhanghailiang, qemu-devel, dgilbert, quintela
On 2020/11/27 20:10, Markus Armbruster wrote:
> Zeyu Jin <jinzeyu@huawei.com> writes:
>
>> This patch enables zstd option in multi-thread compression.
>>
>> Signed-off-by: Zeyu Jin <jinzeyu@huawei.com>
>> Signed-off-by: Ying Fang <fangying1@huawei.com>
> [...]
>> diff --git a/qapi/migration.json b/qapi/migration.json
>> index d262683a38..ac6d06c683 100644
>> --- a/qapi/migration.json
>> +++ b/qapi/migration.json
>> @@ -536,7 +536,7 @@
>> #
>> ##
>> { 'enum': 'CompressMethod',
>> - 'data': [ 'zlib' ] }
>> + 'data': [ 'zlib', { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } ] }
>>
>> ##
>> # @BitmapMigrationBitmapAlias:
>
> Please document the new enum value @zstd.
I will document that. Thank you.
>
> .
>
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2020-11-28 7:07 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2020-11-27 7:42 [RFC PATCH 4/6] migration: Add zstd support in multi-thread compression Zeyu Jin
2020-11-27 12:10 ` Markus Armbruster
2020-11-28 7:06 ` Zeyu Jin
-- strict thread matches above, loose matches on Subject: below --
2020-11-09 9:08 [RFC PATCH 0/6] migration: Multi-thread compression with zstd method Zeyu Jin
2020-11-09 9:08 ` [RFC PATCH 4/6] migration: Add zstd support in multi-thread compression Zeyu Jin
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.