* [PATCH RESEND 1/2] qtest/migration-test.c: Add test with compress enabled @ 2023-04-02 17:47 Lukas Straub 2023-04-02 17:48 ` [PATCH RESEND 2/2] migration/ram.c: Fix migration " Lukas Straub 2023-04-03 21:17 ` [PATCH RESEND 1/2] qtest/migration-test.c: Add test " Peter Xu 0 siblings, 2 replies; 5+ messages in thread From: Lukas Straub @ 2023-04-02 17:47 UTC (permalink / raw) To: qemu-devel Cc: Dr. David Alan Gilbert, Juan Quintela, Peter Xu, Thomas Huth, Laurent Vivier, Paolo Bonzini [-- Attachment #1: Type: text/plain, Size: 3574 bytes --] There has never been a test for migration with compress enabled. Add a suitable test, testing with compress-wait-thread = false too. iterations = 2 is intentional, so it also tests that no invalid thread state is left over from the previous iteration. Signed-off-by: Lukas Straub <lukasstraub2@web.de> --- tests/qtest/migration-test.c | 67 ++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/qtest/migration-test.c b/tests/qtest/migration-test.c index 3b615b0da9..dbcab2e8ae 100644 --- a/tests/qtest/migration-test.c +++ b/tests/qtest/migration-test.c @@ -406,6 +406,41 @@ static void migrate_set_parameter_str(QTestState *who, const char *parameter, migrate_check_parameter_str(who, parameter, value); } +static long long migrate_get_parameter_bool(QTestState *who, + const char *parameter) +{ + QDict *rsp; + int result; + + rsp = wait_command(who, "{ 'execute': 'query-migrate-parameters' }"); + result = qdict_get_bool(rsp, parameter); + qobject_unref(rsp); + return !!result; +} + +static void migrate_check_parameter_bool(QTestState *who, const char *parameter, + int value) +{ + int result; + + result = migrate_get_parameter_bool(who, parameter); + g_assert_cmpint(result, ==, value); +} + +static void migrate_set_parameter_bool(QTestState *who, const char *parameter, + int value) +{ + QDict *rsp; + + rsp = qtest_qmp(who, + "{ 'execute': 'migrate-set-parameters'," + "'arguments': { %s: %i } }", + parameter, value); + g_assert(qdict_haskey(rsp, "return")); + qobject_unref(rsp); + migrate_check_parameter_bool(who, parameter, value); +} + static void migrate_ensure_non_converge(QTestState *who) { /* Can't converge with 1ms downtime + 3 mbs bandwidth limit */ @@ -1524,6 +1559,36 @@ static void test_precopy_unix_xbzrle(void) test_precopy_common(&args); } +static void * +test_migrate_compress_start(QTestState *from, + QTestState *to) +{ + migrate_set_parameter_int(from, "compress-level", 9); + migrate_set_parameter_int(from, "compress-threads", 1); + migrate_set_parameter_bool(from, "compress-wait-thread", false); + migrate_set_parameter_int(to, "decompress-threads", 1); + + migrate_set_capability(from, "compress", true); + migrate_set_capability(to, "compress", true); + + return NULL; +} + +static void test_precopy_unix_compress(void) +{ + g_autofree char *uri = g_strdup_printf("unix:%s/migsocket", tmpfs); + MigrateCommon args = { + .connect_uri = uri, + .listen_uri = uri, + + .start_hook = test_migrate_compress_start, + + .iterations = 2, + }; + + test_precopy_common(&args); +} + static void test_precopy_tcp_plain(void) { MigrateCommon args = { @@ -2515,6 +2580,8 @@ int main(int argc, char **argv) qtest_add_func("/migration/bad_dest", test_baddest); qtest_add_func("/migration/precopy/unix/plain", test_precopy_unix_plain); qtest_add_func("/migration/precopy/unix/xbzrle", test_precopy_unix_xbzrle); + qtest_add_func("/migration/precopy/unix/compress", + test_precopy_unix_compress); #ifdef CONFIG_GNUTLS qtest_add_func("/migration/precopy/unix/tls/psk", test_precopy_unix_tls_psk); -- 2.30.2 [-- Attachment #2: OpenPGP digital signature --] [-- Type: application/pgp-signature, Size: 833 bytes --] ^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH RESEND 2/2] migration/ram.c: Fix migration with compress enabled 2023-04-02 17:47 [PATCH RESEND 1/2] qtest/migration-test.c: Add test with compress enabled Lukas Straub @ 2023-04-02 17:48 ` Lukas Straub 2023-04-03 21:19 ` Peter Xu 2023-04-03 21:17 ` [PATCH RESEND 1/2] qtest/migration-test.c: Add test " Peter Xu 1 sibling, 1 reply; 5+ messages in thread From: Lukas Straub @ 2023-04-02 17:48 UTC (permalink / raw) To: qemu-devel Cc: Dr. David Alan Gilbert, Juan Quintela, Peter Xu, Thomas Huth, Laurent Vivier, Paolo Bonzini [-- Attachment #1: Type: text/plain, Size: 4375 bytes --] Since ec6f3ab9, migration with compress enabled was broken, because the compress threads use a dummy QEMUFile which just acts as a buffer and that commit accidentally changed it to use the outgoing migration channel instead. Fix this by using the dummy file again in the compress threads. Signed-off-by: Lukas Straub <lukasstraub2@web.de> --- migration/ram.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/migration/ram.c b/migration/ram.c index 96e8a19a58..9d1817ab7b 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -688,12 +688,11 @@ exit: * @offset: offset inside the block for the page * in the lower bits, it contains flags */ -static size_t save_page_header(PageSearchStatus *pss, RAMBlock *block, - ram_addr_t offset) +static size_t save_page_header(PageSearchStatus *pss, QEMUFile *f, + RAMBlock *block, ram_addr_t offset) { size_t size, len; bool same_block = (block == pss->last_sent_block); - QEMUFile *f = pss->pss_channel; if (same_block) { offset |= RAM_SAVE_FLAG_CONTINUE; @@ -867,7 +866,7 @@ static int save_xbzrle_page(RAMState *rs, PageSearchStatus *pss, } /* Send XBZRLE based compressed page */ - bytes_xbzrle = save_page_header(pss, block, + bytes_xbzrle = save_page_header(pss, pss->pss_channel, block, offset | RAM_SAVE_FLAG_XBZRLE); qemu_put_byte(file, ENCODING_FLAG_XBZRLE); qemu_put_be16(file, encoded_len); @@ -1302,15 +1301,14 @@ void ram_release_page(const char *rbname, uint64_t offset) * @block: block that contains the page we want to send * @offset: offset inside the block for the page */ -static int save_zero_page_to_file(PageSearchStatus *pss, +static int save_zero_page_to_file(PageSearchStatus *pss, QEMUFile *file, RAMBlock *block, ram_addr_t offset) { uint8_t *p = block->host + offset; - QEMUFile *file = pss->pss_channel; int len = 0; if (buffer_is_zero(p, TARGET_PAGE_SIZE)) { - len += save_page_header(pss, block, offset | RAM_SAVE_FLAG_ZERO); + len += save_page_header(pss, file, block, offset | RAM_SAVE_FLAG_ZERO); qemu_put_byte(file, 0); len += 1; ram_release_page(block->idstr, offset); @@ -1327,10 +1325,10 @@ static int save_zero_page_to_file(PageSearchStatus *pss, * @block: block that contains the page we want to send * @offset: offset inside the block for the page */ -static int save_zero_page(PageSearchStatus *pss, RAMBlock *block, +static int save_zero_page(PageSearchStatus *pss, QEMUFile *f, RAMBlock *block, ram_addr_t offset) { - int len = save_zero_page_to_file(pss, block, offset); + int len = save_zero_page_to_file(pss, f, block, offset); if (len) { stat64_add(&ram_atomic_counters.duplicate, 1); @@ -1394,7 +1392,7 @@ static int save_normal_page(PageSearchStatus *pss, RAMBlock *block, { QEMUFile *file = pss->pss_channel; - ram_transferred_add(save_page_header(pss, block, + ram_transferred_add(save_page_header(pss, pss->pss_channel, block, offset | RAM_SAVE_FLAG_PAGE)); if (async) { qemu_put_buffer_async(file, buf, TARGET_PAGE_SIZE, @@ -1473,11 +1471,11 @@ static bool do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block, uint8_t *p = block->host + offset; int ret; - if (save_zero_page_to_file(pss, block, offset)) { + if (save_zero_page_to_file(pss, f, block, offset)) { return true; } - save_page_header(pss, block, offset | RAM_SAVE_FLAG_COMPRESS_PAGE); + save_page_header(pss, f, block, offset | RAM_SAVE_FLAG_COMPRESS_PAGE); /* * copy it to a internal buffer to avoid it being modified by VM @@ -2355,7 +2353,7 @@ static int ram_save_target_page_legacy(RAMState *rs, PageSearchStatus *pss) return 1; } - res = save_zero_page(pss, block, offset); + res = save_zero_page(pss, pss->pss_channel, block, offset); if (res > 0) { /* Must let xbzrle know, otherwise a previous (now 0'd) cached * page would be stale -- 2.30.2 [-- Attachment #2: OpenPGP digital signature --] [-- Type: application/pgp-signature, Size: 833 bytes --] ^ permalink raw reply related [flat|nested] 5+ messages in thread
* Re: [PATCH RESEND 2/2] migration/ram.c: Fix migration with compress enabled 2023-04-02 17:48 ` [PATCH RESEND 2/2] migration/ram.c: Fix migration " Lukas Straub @ 2023-04-03 21:19 ` Peter Xu 0 siblings, 0 replies; 5+ messages in thread From: Peter Xu @ 2023-04-03 21:19 UTC (permalink / raw) To: Lukas Straub Cc: qemu-devel, Dr. David Alan Gilbert, Juan Quintela, Thomas Huth, Laurent Vivier, Paolo Bonzini On Sun, Apr 02, 2023 at 05:48:38PM +0000, Lukas Straub wrote: > Since ec6f3ab9, migration with compress enabled was broken, because > the compress threads use a dummy QEMUFile which just acts as a > buffer and that commit accidentally changed it to use the outgoing > migration channel instead. Sorry. :( > Fix this by using the dummy file again in the compress threads. > > Signed-off-by: Lukas Straub <lukasstraub2@web.de> Reviewed-by: Peter Xu <peterx@redhat.com> Let's also add a Fixes tag to be clear: ec6f3ab9f4 ("migration: Move last_sent_block into PageSearchStatus") Even though it may not need to copy stable. I think this is for 8.0, am I right? It should be very clear if so, otherwise it's easy to get overlooked. -- Peter Xu ^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH RESEND 1/2] qtest/migration-test.c: Add test with compress enabled 2023-04-02 17:47 [PATCH RESEND 1/2] qtest/migration-test.c: Add test with compress enabled Lukas Straub 2023-04-02 17:48 ` [PATCH RESEND 2/2] migration/ram.c: Fix migration " Lukas Straub @ 2023-04-03 21:17 ` Peter Xu 2023-04-04 7:40 ` Lukas Straub 1 sibling, 1 reply; 5+ messages in thread From: Peter Xu @ 2023-04-03 21:17 UTC (permalink / raw) To: Lukas Straub Cc: qemu-devel, Dr. David Alan Gilbert, Juan Quintela, Thomas Huth, Laurent Vivier, Paolo Bonzini On Sun, Apr 02, 2023 at 05:47:45PM +0000, Lukas Straub wrote: > There has never been a test for migration with compress enabled. > > Add a suitable test, testing with compress-wait-thread = false > too. > > iterations = 2 is intentional, so it also tests that no invalid > thread state is left over from the previous iteration. > > Signed-off-by: Lukas Straub <lukasstraub2@web.de> Overall looks good to me: Reviewed-by: Peter Xu <peterx@redhat.com> A few nitpicks below. > --- > tests/qtest/migration-test.c | 67 ++++++++++++++++++++++++++++++++++++ > 1 file changed, 67 insertions(+) > > diff --git a/tests/qtest/migration-test.c b/tests/qtest/migration-test.c > index 3b615b0da9..dbcab2e8ae 100644 > --- a/tests/qtest/migration-test.c > +++ b/tests/qtest/migration-test.c > @@ -406,6 +406,41 @@ static void migrate_set_parameter_str(QTestState *who, const char *parameter, > migrate_check_parameter_str(who, parameter, value); > } > > +static long long migrate_get_parameter_bool(QTestState *who, > + const char *parameter) > +{ > + QDict *rsp; > + int result; > + > + rsp = wait_command(who, "{ 'execute': 'query-migrate-parameters' }"); > + result = qdict_get_bool(rsp, parameter); > + qobject_unref(rsp); > + return !!result; > +} > + > +static void migrate_check_parameter_bool(QTestState *who, const char *parameter, > + int value) > +{ > + int result; > + > + result = migrate_get_parameter_bool(who, parameter); > + g_assert_cmpint(result, ==, value); > +} > + > +static void migrate_set_parameter_bool(QTestState *who, const char *parameter, > + int value) > +{ > + QDict *rsp; > + > + rsp = qtest_qmp(who, > + "{ 'execute': 'migrate-set-parameters'," > + "'arguments': { %s: %i } }", > + parameter, value); > + g_assert(qdict_haskey(rsp, "return")); > + qobject_unref(rsp); > + migrate_check_parameter_bool(who, parameter, value); > +} > + > static void migrate_ensure_non_converge(QTestState *who) > { > /* Can't converge with 1ms downtime + 3 mbs bandwidth limit */ > @@ -1524,6 +1559,36 @@ static void test_precopy_unix_xbzrle(void) > test_precopy_common(&args); > } > > +static void * > +test_migrate_compress_start(QTestState *from, > + QTestState *to) > +{ > + migrate_set_parameter_int(from, "compress-level", 9); > + migrate_set_parameter_int(from, "compress-threads", 1); > + migrate_set_parameter_bool(from, "compress-wait-thread", false); May worth trying both true/false (can split into two tests)? > + migrate_set_parameter_int(to, "decompress-threads", 1); Why not set both compress/decompress threads to something >1 to check arace conditions between the threads? > + > + migrate_set_capability(from, "compress", true); > + migrate_set_capability(to, "compress", true); > + > + return NULL; > +} > + > +static void test_precopy_unix_compress(void) > +{ > + g_autofree char *uri = g_strdup_printf("unix:%s/migsocket", tmpfs); > + MigrateCommon args = { > + .connect_uri = uri, > + .listen_uri = uri, > + Empty line. > + .start_hook = test_migrate_compress_start, > + Empty line. > + .iterations = 2, Maybe move the comment in commit message over here? > + }; > + > + test_precopy_common(&args); > +} > + > static void test_precopy_tcp_plain(void) > { > MigrateCommon args = { > @@ -2515,6 +2580,8 @@ int main(int argc, char **argv) > qtest_add_func("/migration/bad_dest", test_baddest); > qtest_add_func("/migration/precopy/unix/plain", test_precopy_unix_plain); > qtest_add_func("/migration/precopy/unix/xbzrle", test_precopy_unix_xbzrle); > + qtest_add_func("/migration/precopy/unix/compress", > + test_precopy_unix_compress); > #ifdef CONFIG_GNUTLS > qtest_add_func("/migration/precopy/unix/tls/psk", > test_precopy_unix_tls_psk); > -- > 2.30.2 > -- Peter Xu ^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH RESEND 1/2] qtest/migration-test.c: Add test with compress enabled 2023-04-03 21:17 ` [PATCH RESEND 1/2] qtest/migration-test.c: Add test " Peter Xu @ 2023-04-04 7:40 ` Lukas Straub 0 siblings, 0 replies; 5+ messages in thread From: Lukas Straub @ 2023-04-04 7:40 UTC (permalink / raw) To: Peter Xu Cc: qemu-devel, Dr. David Alan Gilbert, Juan Quintela, Thomas Huth, Laurent Vivier, Paolo Bonzini [-- Attachment #1: Type: text/plain, Size: 3587 bytes --] On Mon, 3 Apr 2023 17:17:52 -0400 Peter Xu <peterx@redhat.com> wrote: > On Sun, Apr 02, 2023 at 05:47:45PM +0000, Lukas Straub wrote: > > There has never been a test for migration with compress enabled. > > > > Add a suitable test, testing with compress-wait-thread = false > > too. > > > > iterations = 2 is intentional, so it also tests that no invalid > > thread state is left over from the previous iteration. > > > > Signed-off-by: Lukas Straub <lukasstraub2@web.de> > > Overall looks good to me: > > Reviewed-by: Peter Xu <peterx@redhat.com> > > A few nitpicks below. > > > --- > > tests/qtest/migration-test.c | 67 ++++++++++++++++++++++++++++++++++++ > > 1 file changed, 67 insertions(+) > > > > diff --git a/tests/qtest/migration-test.c b/tests/qtest/migration-test.c > > index 3b615b0da9..dbcab2e8ae 100644 > > --- a/tests/qtest/migration-test.c > > +++ b/tests/qtest/migration-test.c > > > > [...] > > > > static void migrate_ensure_non_converge(QTestState *who) > > { > > /* Can't converge with 1ms downtime + 3 mbs bandwidth limit */ > > @@ -1524,6 +1559,36 @@ static void test_precopy_unix_xbzrle(void) > > test_precopy_common(&args); > > } > > > > +static void * > > +test_migrate_compress_start(QTestState *from, > > + QTestState *to) > > +{ > > + migrate_set_parameter_int(from, "compress-level", 9); > > + migrate_set_parameter_int(from, "compress-threads", 1); > > + migrate_set_parameter_bool(from, "compress-wait-thread", false); > > May worth trying both true/false (can split into two tests)? Maybe, I just wasn't sure with your CI resources being tight, whether I should add more tests. I think this test gives the most "bang for the buck". > > + migrate_set_parameter_int(to, "decompress-threads", 1); > > Why not set both compress/decompress threads to something >1 to check arace > conditions between the threads? I just wanted to make sure that it won't get too fast so it really sends some pages uncompressed. But I guess this can be fixed when splitting it into 2 tests. > > + > > + migrate_set_capability(from, "compress", true); > > + migrate_set_capability(to, "compress", true); > > + > > + return NULL; > > +} > > + > > +static void test_precopy_unix_compress(void) > > +{ > > + g_autofree char *uri = g_strdup_printf("unix:%s/migsocket", tmpfs); > > + MigrateCommon args = { > > + .connect_uri = uri, > > + .listen_uri = uri, > > + > > Empty line. > > > + .start_hook = test_migrate_compress_start, > > + > > Empty line. > > > + .iterations = 2, > > Maybe move the comment in commit message over here? Good idea, will fix in the next version. Regards, Lukas Straub > > + }; > > + > > + test_precopy_common(&args); > > +} > > + > > static void test_precopy_tcp_plain(void) > > { > > MigrateCommon args = { > > @@ -2515,6 +2580,8 @@ int main(int argc, char **argv) > > qtest_add_func("/migration/bad_dest", test_baddest); > > qtest_add_func("/migration/precopy/unix/plain", test_precopy_unix_plain); > > qtest_add_func("/migration/precopy/unix/xbzrle", test_precopy_unix_xbzrle); > > + qtest_add_func("/migration/precopy/unix/compress", > > + test_precopy_unix_compress); > > #ifdef CONFIG_GNUTLS > > qtest_add_func("/migration/precopy/unix/tls/psk", > > test_precopy_unix_tls_psk); > > -- > > 2.30.2 > > > > > -- [-- Attachment #2: OpenPGP digital signature --] [-- Type: application/pgp-signature, Size: 833 bytes --] ^ permalink raw reply [flat|nested] 5+ messages in thread
end of thread, other threads:[~2023-04-04 7:41 UTC | newest] Thread overview: 5+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2023-04-02 17:47 [PATCH RESEND 1/2] qtest/migration-test.c: Add test with compress enabled Lukas Straub 2023-04-02 17:48 ` [PATCH RESEND 2/2] migration/ram.c: Fix migration " Lukas Straub 2023-04-03 21:19 ` Peter Xu 2023-04-03 21:17 ` [PATCH RESEND 1/2] qtest/migration-test.c: Add test " Peter Xu 2023-04-04 7:40 ` Lukas Straub
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for NNTP newsgroup(s).