[PATCH v4 07/11] migration: add support for fault thread to load pages from disk

[Aadeshveer Singh <aadeshveer07@gmail.com>]

In fast snapshot load, we would like to serve faults as soon as possible
hence loading pages directly instead of requesting a source

Add postcopy_mapped_ram_load_page() function which serves single page
fault. It uses bitmap_test_and_clear_atomic() on pending_bmap to prevent
multiple threads from loading same page. It loads a page or pages
depending on size of guest pages and host pages, loading exactly the
larger of two. If the entire page is zero postcopy_place_page_zero() is
used for efficiency and in case some part is non zero it is part by part
loaded on loop using new function postcopy_mapped_ram_load_guest_page()
which loads a single guest page in a buffer which is then placed using
postcopy_place_page(). This covers all possible cases for various page
sizes of host and guest.

Update postcopy_ram_fault_thread to call postcopy_mapped_ram_load_page
instead of requesting source in case of fast snapshot load. to_src_file
check is bypassed in fast snapshot load case as there is no source.

Call try_mark_postcopy_blocktime_begin on every page fault to support
postcopy-blocktime.

Allocate another channel in postcopy_temp_pages_setup(like the preempt
case), for both the fault thread and eager thread to load pages
independently.

Signed-off-by: Aadeshveer Singh <aadeshveer07@gmail.com>
---
 migration/postcopy-ram.c | 174 +++++++++++++++++++++++++++++++++++----
 1 file changed, 157 insertions(+), 17 deletions(-)

diff --git a/migration/postcopy-ram.c b/migration/postcopy-ram.c
index 2e2c9fae10..1da9c4309b 100644
--- a/migration/postcopy-ram.c
+++ b/migration/postcopy-ram.c
@@ -949,6 +949,121 @@ int postcopy_wake_shared(struct PostCopyFD *pcfd,
                        pagesize);
 }
 
+/*
+ * Load a single guest page from source file into the buffer.
+ * NOTE: This is not an atomic operation and should not be used to directly load
+ * pages on page faults in postcopy. It is meant to fill in buffer that can then
+ * be copied into the faulting location using UFFDIO_COPY.
+ */
+static bool postcopy_mapped_ram_load_guest_page(MigrationIncomingState *mis,
+                                                RAMBlock *rb,
+                                                ram_addr_t rb_offset, void *buf,
+                                                Error **errp)
+{
+    ERRP_GUARD();
+    size_t page = rb_offset / qemu_target_page_size();
+    size_t read;
+
+    if (test_bit(page, rb->file_bmap)) {
+        /*
+         * This can happen concurrently, but it's thread-safe because
+         * qemu_get_buffer_at() is thread-safe, and the caller will be using
+         * different temporary buffers.
+         */
+        read =
+            qemu_get_buffer_at(mis->from_src_file, buf, qemu_target_page_size(),
+                               rb->pages_offset + rb_offset, errp);
+
+        if (read != qemu_target_page_size()) {
+            error_prepend(errp,
+                          "Could not read page %zu from RAM Block %s: ", page,
+                          rb->idstr);
+            return false;
+        }
+    } else {
+        memset(buf, '\0', qemu_target_page_size());
+    }
+    return true;
+}
+
+/**
+ * postcopy_mapped_ram_load_page() - Load pages required to access host address.
+ * @mis: Migration Incoming State.
+ * @rb: RAMBlock from where page is loaded.
+ * @rb_offset: Offset of target page in RAMBlock.
+ * @haddr: Base of target page where to load in page.
+ * @channel: Used to identify between threads and use corresponding temp.
+ * @errp: Set error in case of failure
+ *
+ * Load page(s) from RAMBlock covering the faulting address. We might need to
+ * load multiple pages in the case when host page size is greater than guest
+ * page size. As userfaultfd works on granularity of host pages, we might need
+ * to load guest pages in single operation.
+ *
+ * Return: True on success.
+ */
+static bool postcopy_mapped_ram_load_page(MigrationIncomingState *mis,
+                                          RAMBlock *rb, ram_addr_t rb_offset,
+                                          uint64_t haddr, int channel,
+                                          Error **errp)
+{
+    void *place_source = mis->postcopy_tmp_pages[channel].tmp_huge_page;
+    char *buffer_ptr = (char *)place_source;
+    size_t guest_pages_to_load =
+        MAX(1, qemu_ram_pagesize(rb) / qemu_target_page_size());
+    size_t host_page;
+    size_t page;
+
+    /*
+     * If guest page size is greater than host page size uffd needs to load one
+     * guest page and multiple host pages, hence the offsets need to aligned
+     * with guest pages (which is automatically aligned with host pages). In the
+     * same case we need to check range of bits on pending_bmap(bit per host
+     * page) to decide whether all the page have been loaded
+     */
+    rb_offset = ROUND_DOWN(rb_offset, qemu_target_page_size());
+    haddr = ROUND_DOWN(haddr, qemu_target_page_size());
+    host_page = rb_offset / qemu_ram_pagesize(rb);
+    page = rb_offset >> qemu_target_page_bits();
+
+    if (bitmap_test_and_clear_atomic(
+            rb->pending_bmap, host_page,
+            MAX(1, qemu_target_page_size() / qemu_ram_pagesize(rb)))) {
+        if (find_next_bit(rb->file_bmap, page + guest_pages_to_load, page) ==
+            page + guest_pages_to_load) {
+            /* It is efficient to use UFFDIO_ZERO if all pages are zero */
+            if (postcopy_place_page_zero(mis, (void *)haddr, rb)) {
+                error_setg(errp,
+                           "Failed to place zero page %zu from RAM Block %s at "
+                           "address %" PRIu64,
+                           page, rb->idstr, haddr);
+                return false;
+            }
+        } else {
+            size_t load_size = guest_pages_to_load * qemu_target_page_size();
+            size_t offset;
+
+            for (offset = 0; offset < load_size;
+                 offset += qemu_target_page_size()) {
+                if (!postcopy_mapped_ram_load_guest_page(
+                        mis, rb, rb_offset + offset, buffer_ptr + offset,
+                        errp)) {
+                    return false;
+                }
+            }
+
+            if (postcopy_place_page(mis, (void *)haddr, place_source, rb)) {
+                error_setg(errp,
+                           "Failed to place page %zu from RAM Block %s at "
+                           "address %" PRIu64,
+                           page, rb->idstr, haddr);
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
 /*
  * NOTE: @tid is only used when postcopy-blocktime feature is enabled, and
  * also optional: when zero is provided, the fault accounting will be ignored.
@@ -1310,6 +1425,7 @@ static void *postcopy_ram_fault_thread(void *opaque)
     int ret;
     size_t index;
     RAMBlock *rb = NULL;
+    Error *local_err = NULL;
 
     trace_postcopy_ram_fault_thread_entry();
     rcu_register_thread();
@@ -1351,11 +1467,13 @@ static void *postcopy_ram_fault_thread(void *opaque)
             break;
         }
 
-        if (!mis->to_src_file) {
+        if (!migrate_mapped_ram() && !mis->to_src_file) {
             /*
-             * Possibly someone tells us that the return path is
-             * broken already using the event. We should hold until
-             * the channel is rebuilt.
+             * Possibly someone tells us that the return path is broken already
+             * using the event. We should hold until the channel is rebuilt.
+             * Fast snapshot load doesn't support pause and recover, because
+             * it's not necessary: we can fail right away when QEMU just booted
+             * with nothing to lose.
              */
             postcopy_pause_fault_thread(mis);
         }
@@ -1418,18 +1536,37 @@ static void *postcopy_ram_fault_thread(void *opaque)
                                                 qemu_ram_get_idstr(rb),
                                                 rb_offset,
                                                 msg.arg.pagefault.feat.ptid);
+
+            if (migrate_mapped_ram()) {
+                /* Load page directly in case of fast snapshot load */
+
+                uintptr_t aligned = (uintptr_t)ROUND_DOWN(
+                    msg.arg.pagefault.address, qemu_ram_pagesize(rb));
+
+                if (try_mark_postcopy_blocktime_begin(
+                        mis, rb, rb_offset, (uintptr_t)aligned,
+                        msg.arg.pagefault.feat.ptid)) {
+                    if (!postcopy_mapped_ram_load_page(
+                            mis, rb, rb_offset, aligned, RAM_CHANNEL_POSTCOPY,
+                            &local_err)) {
+                        error_report_err(local_err);
+                        break;
+                    }
+                }
+            } else {
 retry:
-            /*
-             * Send the request to the source - we want to request one
-             * of our host page sizes (which is >= TPS)
-             */
-            ret = postcopy_request_page(mis, rb, rb_offset,
-                                        msg.arg.pagefault.address,
-                                        msg.arg.pagefault.feat.ptid);
-            if (ret) {
-                /* May be network failure, try to wait for recovery */
-                postcopy_pause_fault_thread(mis);
-                goto retry;
+                /*
+                 * Send the request to the source - we want to request one
+                 * of our host page sizes (which is >= TPS)
+                 */
+                ret = postcopy_request_page(mis, rb, rb_offset,
+                                            msg.arg.pagefault.address,
+                                            msg.arg.pagefault.feat.ptid);
+                if (ret) {
+                    /* May be network failure, try to wait for recovery */
+                    postcopy_pause_fault_thread(mis);
+                    goto retry;
+                }
             }
         }
 
@@ -1501,8 +1638,11 @@ static int postcopy_temp_pages_setup(MigrationIncomingState *mis, Error **errp)
     unsigned i, channels;
     void *temp_page;
 
-    if (migrate_postcopy_preempt()) {
-        /* If preemption enabled, need extra channel for urgent requests */
+    if (migrate_postcopy_preempt() || migrate_mapped_ram()) {
+        /*
+         * If preemption enabled or it is fast snapshot load, need extra channel
+         * for urgent requests/faults
+         */
         mis->postcopy_channels = RAM_CHANNEL_MAX;
     } else {
         /* Both precopy/postcopy on the same channel */
-- 
2.55.0