[Qemu-devel] [PATCH 41/46] Handle userfault requests (although userfaultfd not done yet)

["Dr. David Alan Gilbert (git)" <dgilbert@redhat.com>]

From: "Dr. David Alan Gilbert" <dgilbert@redhat.com>

Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
 include/migration/migration.h |  1 +
 postcopy-ram.c                | 93 +++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 90 insertions(+), 4 deletions(-)

diff --git a/include/migration/migration.h b/include/migration/migration.h
index 1a33b05..46fc37b 100644
--- a/include/migration/migration.h
+++ b/include/migration/migration.h
@@ -89,6 +89,7 @@ struct MigrationIncomingState {
 
     QemuThread     fault_thread;
     QemuSemaphore  fault_thread_sem;
+    int            userfault_fd;
 
     QEMUFile *return_path;
     QemuMutex      rp_mutex;    /* We send replies from multiple threads */
diff --git a/postcopy-ram.c b/postcopy-ram.c
index de3534f..8d0a225 100644
--- a/postcopy-ram.c
+++ b/postcopy-ram.c
@@ -183,7 +183,6 @@ static void postcopy_pmi_dump(MigrationIncomingState *mis)
     ram_debug_dump_bitmap(mis->postcopy_pmi.received_map, true);
 }
 
-/* ---------------------------------------------------------------------- */
 int postcopy_ram_hosttest(void)
 {
     /* TODO: Needs guarding with CONFIG_ once we have libc's that have the defs
@@ -367,12 +366,97 @@ static int postcopy_ram_sensitise_area(const char *block_name, void *host_addr,
 static void *postcopy_ram_fault_thread(void *opaque)
 {
     MigrationIncomingState *mis = (MigrationIncomingState *)opaque;
+    void *hostaddr;
+    int ret;
+    size_t hostpagesize = getpagesize();
+    RAMBlock *rb = NULL;
+    RAMBlock *last_rb = NULL;
 
-    fprintf(stderr, "postcopy_ram_fault_thread\n");
-    /* TODO: In later patch */
+    DPRINTF("%s", __func__);
     qemu_sem_post(&mis->fault_thread_sem);
     while (1) {
-        /* TODO: In later patch */
+        PostcopyPMIState old_state, tmp_state;
+        ram_addr_t rb_offset;
+        ram_addr_t in_raspace;
+        unsigned long bitmap_index;
+
+        /* Read a faulting HVA from the kernel */
+        ret = read(mis->userfault_fd, &hostaddr, sizeof(hostaddr));
+        if (ret != sizeof(hostaddr)) {
+            if (ret < 0) {
+                perror("Failed to read full userfault hostaddr");
+                break;
+            } else {
+                error_report("%s: Read %d bytes from userfaultfd expected %ld",
+                             __func__, ret, sizeof(hostaddr));
+                break; /* Lost alignment, don't know what we'd read next */
+            }
+        }
+
+        /* TODO: We want to be marking host-page-size areas of the bitmaps? */
+        last_rb = rb;
+        rb = qemu_ram_block_from_host(hostaddr, &in_raspace, &rb_offset,
+                                      &bitmap_index);
+        if (!rb) {
+            error_report("postcopy_ram_fault_thread: Fault outside guest: %p",
+                         hostaddr);
+            break;
+        }
+
+        DPRINTF("%s: Request for HVA=%p index=%lx rb=%s offset=%zx",
+                __func__, hostaddr, bitmap_index, qemu_ram_get_idstr(rb),
+                rb_offset);
+
+        tmp_state = postcopy_pmi_get_state(mis, bitmap_index);
+        do {
+            old_state = tmp_state;
+
+            switch (old_state) {
+            case POSTCOPY_PMI_REQUESTED:
+                /* Do nothing - it's already requested */
+                break;
+
+            case POSTCOPY_PMI_RECEIVED:
+                /* Already arrived - no state change, just kick the kernel */
+                DPRINTF("postcopy_ram_fault_thread: notify pre of %p",
+                        hostaddr);
+                /* TODO! Send ack
+                if (ack_userfault(mis, hostaddr, hostpagesize)) {
+                    assert(0);
+                } */
+                break;
+
+            case POSTCOPY_PMI_MISSING:
+
+                tmp_state = postcopy_pmi_change_state(mis, bitmap_index,
+                                           old_state, POSTCOPY_PMI_REQUESTED);
+                if (tmp_state == POSTCOPY_PMI_MISSING) {
+                    /*
+                     * Send the request to the source - we want to request one
+                     * of our host page sizes (which is >= TPS)
+                     */
+                    if (rb != last_rb) {
+                        migrate_send_rp_reqpages(mis, qemu_ram_get_idstr(rb),
+                                                 rb_offset, hostpagesize);
+                    } else {
+                        /* Save some space */
+                        migrate_send_rp_reqpages(mis, NULL,
+                                                 rb_offset, hostpagesize);
+                    }
+
+                    if (mis->postcopy_ram_state == POSTCOPY_RAM_INCOMING_END) {
+                        /* This shouldn't happen - the command to close the
+                         * postcopy stream should be after the last page of RAM
+                         * so we're not going to get an answer
+                         */
+                        error_report("postcopy_ram_fault_thread: UF after end");
+                        postcopy_pmi_dump(mis);
+                        assert(0);
+                    }
+                }
+                break;
+           }
+        } while (tmp_state != old_state);
     }
 
     return NULL;
@@ -381,6 +465,7 @@ static void *postcopy_ram_fault_thread(void *opaque)
 int postcopy_ram_enable_notify(MigrationIncomingState *mis)
 {
     /* Create the fault handler thread and wait for it to be ready */
+    mis->userfault_fd = -1; /* TODO */
     qemu_sem_init(&mis->fault_thread_sem, 0);
     qemu_thread_create(&mis->fault_thread, "postcopy/fault",
                        postcopy_ram_fault_thread, mis, QEMU_THREAD_JOINABLE);
-- 
1.9.3