[PATCH 05/13] migration/rdma: Change io_create_watch() to return immediately

[Peter Xu <peterx@redhat.com>]

The old RDMA's io_create_watch() isn't really doing much work anyway.  For
G_IO_OUT, it already does return immediately.  For G_IO_IN, it will try to
detect some RDMA context length however normally nobody will be able to set
it at all.

Simplify the code so that RDMA iochannels simply always rely on synchronous
reads and writes.  It is highly likely what 6ddd2d76ca6f86f was talking
about, that the async model isn't really working well.

To be eplicit, incoming migration should always have marked the iochannel
to be nonblocking.  For non-RDMA channels, what happens with current master
branch is when we have nothing to read, QEMU yields the coroutine at
qemu_fill_buffer().  For RDMA, what I see is it always polls on its own and
it yields at qemu_rdma_wait_comp_channel(). A sample stack:

  #0  qemu_coroutine_yield
  #1  0x0000562e46e51f77 in yield_until_fd_readable
  #2  0x0000562e46927823 in qemu_rdma_wait_comp_channel
  #3  0x0000562e46927b35 in qemu_rdma_block_for_wrid
  #4  0x0000562e46927e6f in qemu_rdma_post_send_control
  #5  0x0000562e4692857f in qemu_rdma_exchange_recv
  #6  0x0000562e4692ab5e in qio_channel_rdma_readv
  #7  0x0000562e46c1f2d7 in qio_channel_readv_full
  #8  0x0000562e46c13a6e in qemu_fill_buffer
  #9  0x0000562e46c14ba8 in qemu_peek_byte
  #10 0x0000562e46c14c09 in qemu_get_byte
  #11 0x0000562e46c14e2a in qemu_get_be32
  #12 0x0000562e46c14e8a in qemu_get_be64
  #13 0x0000562e46913f08 in ram_load_precopy
  #14 0x0000562e46914448 in ram_load
  #15 0x0000562e469186e3 in vmstate_load
  #16 0x0000562e4691ce6d in qemu_loadvm_section_part_end
  #17 0x0000562e4691d99b in qemu_loadvm_state_main
  #18 0x0000562e4691db87 in qemu_loadvm_state
  #19 0x0000562e468f2e87 in process_incoming_migration_co

This patch may or may not help in reality, the whole IO watch may or may
not be working at all for RDMA iochannels.  In all cases, this patch makes
sure above will be the only place that RDMA can poll on IOs.

Tested-by: Zhijian Li (Fujitsu) <lizhijian@fujitsu.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
---
 migration/rdma.c | 69 +++---------------------------------------------
 1 file changed, 3 insertions(+), 66 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 13dd391c14..0e5e02cdca 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2776,56 +2776,14 @@ static gboolean
 qio_channel_rdma_source_prepare(GSource *source,
                                 gint *timeout)
 {
-    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
-    RDMAContext *rdma;
-    GIOCondition cond = 0;
     *timeout = -1;
-
-    RCU_READ_LOCK_GUARD();
-    if (rsource->condition == G_IO_IN) {
-        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
-    } else {
-        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
-    }
-
-    if (!rdma) {
-        error_report("RDMAContext is NULL when prepare Gsource");
-        return FALSE;
-    }
-
-    if (rdma->wr_data[0].control_len) {
-        cond |= G_IO_IN;
-    }
-    cond |= G_IO_OUT;
-
-    return cond & rsource->condition;
+    return TRUE;
 }
 
 static gboolean
 qio_channel_rdma_source_check(GSource *source)
 {
-    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
-    RDMAContext *rdma;
-    GIOCondition cond = 0;
-
-    RCU_READ_LOCK_GUARD();
-    if (rsource->condition == G_IO_IN) {
-        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
-    } else {
-        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
-    }
-
-    if (!rdma) {
-        error_report("RDMAContext is NULL when check Gsource");
-        return FALSE;
-    }
-
-    if (rdma->wr_data[0].control_len) {
-        cond |= G_IO_IN;
-    }
-    cond |= G_IO_OUT;
-
-    return cond & rsource->condition;
+    return TRUE;
 }
 
 static gboolean
@@ -2835,29 +2793,8 @@ qio_channel_rdma_source_dispatch(GSource *source,
 {
     QIOChannelFunc func = (QIOChannelFunc)callback;
     QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
-    RDMAContext *rdma;
-    GIOCondition cond = 0;
-
-    RCU_READ_LOCK_GUARD();
-    if (rsource->condition == G_IO_IN) {
-        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
-    } else {
-        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
-    }
-
-    if (!rdma) {
-        error_report("RDMAContext is NULL when dispatch Gsource");
-        return FALSE;
-    }
-
-    if (rdma->wr_data[0].control_len) {
-        cond |= G_IO_IN;
-    }
-    cond |= G_IO_OUT;
 
-    return (*func)(QIO_CHANNEL(rsource->rioc),
-                   (cond & rsource->condition),
-                   user_data);
+    return (*func)(QIO_CHANNEL(rsource->rioc), rsource->condition, user_data);
 }
 
 static void
-- 
2.50.1