[Qemu-devel] [RFC PATCH 02/22]

["Pavel Dovgaluk" <Pavel.Dovgaluk@ispras.ru>]

These patches modify virtual hardware to make them deterministic.
We also implemented saving/restoring for several devices or fixed vm state
of them to save the fields that we needed for deterministically restoring
behavior of the devices.

Signed-off-by: Pavel Dovgalyuk <pavel.dovgaluk@gmail.com>
---

diff --git a/arch_init.c b/arch_init.c
index 8ddaf35..f496f82
--- a/arch_init.c
+++ b/arch_init.c
@@ -52,6 +52,7 @@
 #include "exec/ram_addr.h"
 #include "hw/acpi/acpi.h"
 #include "qemu/host-utils.h"
+#include "replay/replay.h"
 
 #ifdef DEBUG_ARCH_INIT
 #define DPRINTF(fmt, ...) \
@@ -1092,7 +1093,15 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
 
                 total_ram_bytes -= length;
             }
+            if (replay_mode == REPLAY_PLAY) {
+                RAMBlock *block;
+                // Clear the blocks' memory instead of resetting the machine
+                QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+                    memset(block->host, 0, block->length);
+                }
+            }
         } else if (flags & RAM_SAVE_FLAG_COMPRESS) {
+               
             void *host;
             uint8_t ch;
 
diff --git a/hw/acpi/core.c b/hw/acpi/core.c
index a7368fb..5df071a 100644
--- a/hw/acpi/core.c
+++ b/hw/acpi/core.c
@@ -376,8 +376,11 @@ static void acpi_notify_wakeup(Notifier *notifier, void *data)
 /* ACPI PM1a EVT */
 uint16_t acpi_pm1_evt_get_sts(ACPIREGS *ar)
 {
-    int64_t d = acpi_pm_tmr_get_clock();
-    if (d >= ar->tmr.overflow_time) {
+    // Compare ns-clock, not PM timer ticks, because 
+    // acpi_pm_tmr_update function uses ns for setting the timer.
+    int64_t d = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+    if (d >= muldiv64(ar->tmr.overflow_time, 
+                      get_ticks_per_sec(), PM_TIMER_FREQUENCY)) {
         ar->pm1.evt.sts |= ACPI_BITMASK_TIMER_STATUS;
     }
     return ar->pm1.evt.sts;
diff --git a/hw/arm/integratorcp.c b/hw/arm/integratorcp.c
index 0e476c3..496f84e
--- a/hw/arm/integratorcp.c
+++ b/hw/arm/integratorcp.c
@@ -42,6 +42,27 @@ typedef struct IntegratorCMState {
     uint32_t fiq_enabled;
 } IntegratorCMState;
 
+static const VMStateDescription vmstate_integratorcm = {
+    .name = "integratorcm",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField []) {
+        VMSTATE_UINT32(cm_osc, IntegratorCMState),
+        VMSTATE_UINT32(cm_ctrl, IntegratorCMState),
+        VMSTATE_UINT32(cm_lock, IntegratorCMState),
+        VMSTATE_UINT32(cm_auxosc, IntegratorCMState),
+        VMSTATE_UINT32(cm_sdram, IntegratorCMState),
+        VMSTATE_UINT32(cm_init, IntegratorCMState),
+        VMSTATE_UINT32(cm_flags, IntegratorCMState),
+        VMSTATE_UINT32(cm_nvflags, IntegratorCMState),
+        VMSTATE_UINT32(int_level, IntegratorCMState),
+        VMSTATE_UINT32(irq_enabled, IntegratorCMState),
+        VMSTATE_UINT32(fiq_enabled, IntegratorCMState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static uint8_t integrator_spd[128] = {
    128, 8, 4, 11, 9, 1, 64, 0,  2, 0xa0, 0xa0, 0, 0, 8, 0, 1,
    0xe, 4, 0x1c, 1, 2, 0x20, 0xc0, 0, 0, 0, 0, 0x30, 0x28, 0x30, 0x28, 0x40
@@ -272,7 +293,7 @@ static int integratorcm_init(SysBusDevice *dev)
     sysbus_init_mmio(dev, &s->iomem);
 
     integratorcm_do_remap(s);
-    /* ??? Save/restore.  */
+    vmstate_register(NULL, -1, &vmstate_integratorcm, s);
     return 0;
 }
 
@@ -296,6 +317,20 @@ typedef struct icp_pic_state {
     qemu_irq parent_fiq;
 } icp_pic_state;
 
+
+static const VMStateDescription vmstate_icp_pic = {
+    .name = "icp_pic_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField []) {
+        VMSTATE_UINT32(level, icp_pic_state),
+        VMSTATE_UINT32(irq_enabled, icp_pic_state),
+        VMSTATE_UINT32(fiq_enabled, icp_pic_state),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static void icp_pic_update(icp_pic_state *s)
 {
     uint32_t flags;
@@ -399,6 +434,7 @@ static int icp_pic_init(SysBusDevice *sbd)
     memory_region_init_io(&s->iomem, OBJECT(s), &icp_pic_ops, s,
                           "icp-pic", 0x00800000);
     sysbus_init_mmio(sbd, &s->iomem);
+    vmstate_register(NULL, -1, &vmstate_icp_pic, s);
     return 0;
 }
 
diff --git a/hw/audio/pcspk.c b/hw/audio/pcspk.c
index 1d81bbe..2afcffb
--- a/hw/audio/pcspk.c
+++ b/hw/audio/pcspk.c
@@ -50,8 +50,8 @@ typedef struct {
     unsigned int pit_count;
     unsigned int samples;
     unsigned int play_pos;
-    int data_on;
-    int dummy_refresh_clock;
+    uint8_t data_on;
+    uint8_t dummy_refresh_clock;
 } PCSpkState;
 
 static const char *s_spk = "pcspk";
@@ -163,6 +163,19 @@ static const MemoryRegionOps pcspk_io_ops = {
     },
 };
 
+static const VMStateDescription vmstate_spk = {
+    .name = "pcspk",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField []) {
+        //VMSTATE_UINT32(play_pos, PCSpkState),
+        VMSTATE_UINT8(data_on, PCSpkState),
+        VMSTATE_UINT8(dummy_refresh_clock, PCSpkState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static void pcspk_initfn(Object *obj)
 {
     PCSpkState *s = PC_SPEAKER(obj);
@@ -175,6 +188,8 @@ static void pcspk_realizefn(DeviceState *dev, Error **errp)
     ISADevice *isadev = ISA_DEVICE(dev);
     PCSpkState *s = PC_SPEAKER(dev);
 
+    vmstate_register(NULL, 0, &vmstate_spk, s);
+
     isa_register_ioport(isadev, &s->ioport, s->iobase);
 
     pcspk_state = s;
diff --git a/hw/block/fdc.c b/hw/block/fdc.c
index 490d127..132310a 100644
--- a/hw/block/fdc.c
+++ b/hw/block/fdc.c
@@ -697,12 +697,17 @@ static const VMStateDescription vmstate_fdrive_media_rate = {
 
 static const VMStateDescription vmstate_fdrive = {
     .name = "fdrive",
-    .version_id = 1,
+    .version_id = 2,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(head, FDrive),
         VMSTATE_UINT8(track, FDrive),
         VMSTATE_UINT8(sect, FDrive),
+        VMSTATE_UINT8_V(last_sect, FDrive, 2),
+        VMSTATE_UINT8_V(max_track, FDrive, 2),
+        VMSTATE_UINT16_V(bps, FDrive, 2),
+        VMSTATE_UINT8_V(ro, FDrive, 2),
+        VMSTATE_UINT8_V(perpendicular, FDrive, 2),
         VMSTATE_END_OF_LIST()
     },
     .subsections = (VMStateSubsection[]) {
@@ -736,7 +741,7 @@ static int fdc_post_load(void *opaque, int version_id)
 
 static const VMStateDescription vmstate_fdc = {
     .name = "fdc",
-    .version_id = 2,
+    .version_id = 3,
     .minimum_version_id = 2,
     .pre_save = fdc_pre_save,
     .post_load = fdc_post_load,
@@ -769,6 +774,8 @@ static const VMStateDescription vmstate_fdc = {
         VMSTATE_UINT8_EQUAL(num_floppies, FDCtrl),
         VMSTATE_STRUCT_ARRAY(drives, FDCtrl, MAX_FD, 1,
                              vmstate_fdrive, FDrive),
+        VMSTATE_INT32_V(reset_sensei, FDCtrl, 3),
+        VMSTATE_TIMER_V(result_timer, FDCtrl, 3),
         VMSTATE_END_OF_LIST()
     }
 };
diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
index e59ebc9..fdabd27 100644
--- a/hw/block/virtio-blk.c
+++ b/hw/block/virtio-blk.c
@@ -28,6 +28,7 @@
 #endif
 #include "hw/virtio/virtio-bus.h"
 #include "hw/virtio/virtio-access.h"
+#include "replay/replay.h"
 
 static VirtIOBlockReq *virtio_blk_alloc_request(VirtIOBlock *s)
 {
@@ -265,7 +266,7 @@ void virtio_submit_multiwrite(BlockDriverState *bs, MultiReqBuffer *mrb)
         return;
     }
 
-    ret = bdrv_aio_multiwrite(bs, mrb->blkreq, mrb->num_writes);
+    ret = bdrv_aio_multiwrite(bs, mrb->blkreq, mrb->num_writes, true);
     if (ret != 0) {
         for (i = 0; i < mrb->num_writes; i++) {
             if (mrb->blkreq[i].error) {
@@ -285,7 +286,7 @@ static void virtio_blk_handle_flush(VirtIOBlockReq *req, MultiReqBuffer *mrb)
      * Make sure all outstanding writes are posted to the backing device.
      */
     virtio_submit_multiwrite(req->dev->bs, mrb);
-    bdrv_aio_flush(req->dev->bs, virtio_blk_flush_complete, req);
+    bdrv_aio_flush_replay(req->dev->bs, virtio_blk_flush_complete, req);
 }
 
 static void virtio_blk_handle_write(VirtIOBlockReq *req, MultiReqBuffer *mrb)
@@ -341,11 +342,12 @@ static void virtio_blk_handle_read(VirtIOBlockReq *req)
         virtio_blk_rw_complete(req, -EIO);
         return;
     }
-    bdrv_aio_readv(req->dev->bs, sector, &req->qiov,
+    bdrv_aio_readv_replay(req->dev->bs, sector, &req->qiov,
                    req->qiov.size / BDRV_SECTOR_SIZE,
                    virtio_blk_rw_complete, req);
 }
 
+
 void virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb)
 {
     uint32_t type;
diff --git a/hw/char/parallel.c b/hw/char/parallel.c
index 7ac90a5..bc34e55
--- a/hw/char/parallel.c
+++ b/hw/char/parallel.c
@@ -477,6 +477,26 @@ static const MemoryRegionPortio isa_parallel_portio_sw_list[] = {
     PORTIO_END_OF_LIST(),
 };
 
+
+static const VMStateDescription vmstate_parallel_isa = {
+    .name = "parallel_isa",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField []) {
+        VMSTATE_UINT8(state.dataw, ISAParallelState),
+        VMSTATE_UINT8(state.datar, ISAParallelState),
+        VMSTATE_UINT8(state.status, ISAParallelState),
+        VMSTATE_UINT8(state.control, ISAParallelState),
+        VMSTATE_INT32(state.irq_pending, ISAParallelState),
+        VMSTATE_INT32(state.hw_driver, ISAParallelState),
+        VMSTATE_INT32(state.epp_timeout, ISAParallelState),
+        VMSTATE_INT32(state.it_shift, ISAParallelState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+
 static void parallel_isa_realizefn(DeviceState *dev, Error **errp)
 {
     static int index;
@@ -518,6 +538,8 @@ static void parallel_isa_realizefn(DeviceState *dev, Error **errp)
                               ? &isa_parallel_portio_hw_list[0]
                               : &isa_parallel_portio_sw_list[0]),
                              s, "parallel");
+
+    vmstate_register(NULL, -1, &vmstate_parallel_isa, isa);
 }
 
 /* Memory mapped interface */
diff --git a/hw/char/serial.c b/hw/char/serial.c
index d17da16..4e54287
--- a/hw/char/serial.c
+++ b/hw/char/serial.c
@@ -267,6 +267,61 @@ static gboolean serial_xmit(GIOChannel *chan, GIOCondition cond, void *opaque)
 }
 
 
+/* Setter for FCR.
+   is_load flag means, that value is set while loading VM state
+   and interrupt should not be invoked */
+static void serial_write_fcr(void *opaque, uint32_t val, int is_load)
+{
+    SerialState *s = opaque;
+    val = val & 0xFF;
+
+    if (s->fcr == val)
+        return;
+
+    /* Did the enable/disable flag change? If so, make sure FIFOs get flushed */
+    if ((val ^ s->fcr) & UART_FCR_FE)
+        val |= UART_FCR_XFR | UART_FCR_RFR;
+
+    /* FIFO clear */
+
+    if (val & UART_FCR_RFR) {
+        timer_del(s->fifo_timeout_timer);
+        s->timeout_ipending=0;
+        fifo8_reset(&s->recv_fifo);
+    }
+
+    if (val & UART_FCR_XFR) {
+        fifo8_reset(&s->xmit_fifo);
+    }
+
+    if (val & UART_FCR_FE) {
+        s->iir |= UART_IIR_FE;
+        /* Set recv_fifo trigger Level */
+        switch (val & 0xC0) {
+        case UART_FCR_ITL_1:
+            s->recv_fifo_itl = 1;
+            break;
+        case UART_FCR_ITL_2:
+            s->recv_fifo_itl = 4;
+            break;
+        case UART_FCR_ITL_3:
+            s->recv_fifo_itl = 8;
+            break;
+        case UART_FCR_ITL_4:
+            s->recv_fifo_itl = 14;
+            break;
+        }
+    } else
+        s->iir &= ~UART_IIR_FE;
+
+    /* Set fcr - or at least the bits in it that are supposed to "stick" */
+    s->fcr = val & 0xC9;
+
+    if (!is_load) {
+        serial_update_irq(s);
+    }
+}
+
 static void serial_ioport_write(void *opaque, hwaddr addr, uint64_t val,
                                 unsigned size)
 {
@@ -320,50 +375,7 @@ static void serial_ioport_write(void *opaque, hwaddr addr, uint64_t val,
         }
         break;
     case 2:
-        val = val & 0xFF;
-
-        if (s->fcr == val)
-            break;
-
-        /* Did the enable/disable flag change? If so, make sure FIFOs get flushed */
-        if ((val ^ s->fcr) & UART_FCR_FE)
-            val |= UART_FCR_XFR | UART_FCR_RFR;
-
-        /* FIFO clear */
-
-        if (val & UART_FCR_RFR) {
-            timer_del(s->fifo_timeout_timer);
-            s->timeout_ipending=0;
-            fifo8_reset(&s->recv_fifo);
-        }
-
-        if (val & UART_FCR_XFR) {
-            fifo8_reset(&s->xmit_fifo);
-        }
-
-        if (val & UART_FCR_FE) {
-            s->iir |= UART_IIR_FE;
-            /* Set recv_fifo trigger Level */
-            switch (val & 0xC0) {
-            case UART_FCR_ITL_1:
-                s->recv_fifo_itl = 1;
-                break;
-            case UART_FCR_ITL_2:
-                s->recv_fifo_itl = 4;
-                break;
-            case UART_FCR_ITL_3:
-                s->recv_fifo_itl = 8;
-                break;
-            case UART_FCR_ITL_4:
-                s->recv_fifo_itl = 14;
-                break;
-            }
-        } else
-            s->iir &= ~UART_IIR_FE;
-
-        /* Set fcr - or at least the bits in it that are supposed to "stick" */
-        s->fcr = val & 0xC9;
-        serial_update_irq(s);
+        serial_write_fcr(s, val, 0);
         break;
     case 3:
         {
@@ -591,20 +603,35 @@ static int serial_post_load(void *opaque, int version_id)
         s->fcr_vmstate = 0;
     }
     /* Initialize fcr via setter to perform essential side-effects */
-    serial_ioport_write(s, 0x02, s->fcr_vmstate, 1);
+    serial_write_fcr(s, s->fcr_vmstate, 1);
     serial_update_parameters(s);
     return 0;
 }
 
+/*static const VMStateDescription vmstate_fifo = {
+    .name = "serial FIFO",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField []) {
+        VMSTATE_PARTIAL_VBUFFER(data, Fifo8, capacity),
+        VMSTATE_INT32(capacity, Fifo8),
+        VMSTATE_UINT32(head, Fifo8),
+        VMSTATE_UINT32(num, Fifo8),
+        VMSTATE_END_OF_LIST()
+    }
+};*/
+
 const VMStateDescription vmstate_serial = {
     .name = "serial",
-    .version_id = 3,
+    .version_id = 4,
     .minimum_version_id = 2,
     .pre_save = serial_pre_save,
     .post_load = serial_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT16_V(divider, SerialState, 2),
         VMSTATE_UINT8(rbr, SerialState),
+        VMSTATE_UINT8_V(thr, SerialState, 4),
+        VMSTATE_UINT8_V(tsr, SerialState, 4),
         VMSTATE_UINT8(ier, SerialState),
         VMSTATE_UINT8(iir, SerialState),
         VMSTATE_UINT8(lcr, SerialState),
@@ -613,6 +640,15 @@ const VMStateDescription vmstate_serial = {
         VMSTATE_UINT8(msr, SerialState),
         VMSTATE_UINT8(scr, SerialState),
         VMSTATE_UINT8_V(fcr_vmstate, SerialState, 3),
+        VMSTATE_INT32_V(thr_ipending, SerialState, 4),
+        VMSTATE_INT32_V(last_break_enable, SerialState, 4),
+        VMSTATE_INT32_V(tsr_retry, SerialState, 4),
+        VMSTATE_STRUCT(recv_fifo, SerialState, 4, vmstate_fifo8, Fifo8),
+        VMSTATE_STRUCT(xmit_fifo, SerialState, 4, vmstate_fifo8, Fifo8),
+        VMSTATE_TIMER_V(fifo_timeout_timer, SerialState, 4),
+        VMSTATE_INT32_V(timeout_ipending, SerialState, 4),
+        VMSTATE_INT32_V(poll_msl, SerialState, 4),
+        VMSTATE_TIMER_V(modem_status_poll, SerialState, 4),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -670,11 +706,11 @@ void serial_exit_core(SerialState *s)
 }
 
 /* Change the main reference oscillator frequency. */
-void serial_set_frequency(SerialState *s, uint32_t frequency)
+/*void serial_set_frequency(SerialState *s, uint32_t frequency)
 {
     s->baudbase = frequency;
     serial_update_parameters(s);
-}
+}*/
 
 const MemoryRegionOps serial_io_ops = {
     .read = serial_ioport_read,
diff --git a/hw/display/vga.c b/hw/display/vga.c
index 4b089a3..6c65ca1
--- a/hw/display/vga.c
+++ b/hw/display/vga.c
@@ -1304,7 +1304,8 @@ static void vga_draw_text(VGACommonState *s, int full_update)
     uint32_t *ch_attr_ptr;
     vga_draw_glyph8_func *vga_draw_glyph8;
     vga_draw_glyph9_func *vga_draw_glyph9;
-    int64_t now = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
+    /* TODO: find better solution instead of changing vm to rt */
+    int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
 
     /* compute font data address (in plane 2) */
     v = s->sr[VGA_SEQ_CHARACTER_MAP];
@@ -1906,7 +1907,8 @@ static void vga_update_display(void *opaque)
         }
         if (graphic_mode != s->graphic_mode) {
             s->graphic_mode = graphic_mode;
-            s->cursor_blink_time = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
+            /* TODO: find better solution instead of changing vm to rt */
+            s->cursor_blink_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
             full_update = 1;
         }
         switch(graphic_mode) {
diff --git a/hw/i386/kvmvapic.c b/hw/i386/kvmvapic.c
index cb855c7..b9f68aa
--- a/hw/i386/kvmvapic.c
+++ b/hw/i386/kvmvapic.c
@@ -351,6 +351,24 @@ static int get_kpcr_number(X86CPU *cpu)
     return kpcr.number;
 }
 
+static int vapic_enable_post_load(VAPICROMState *s, X86CPU *cpu)
+{
+    int cpu_number = get_kpcr_number(cpu);
+    hwaddr vapic_paddr;
+    static const uint8_t enabled = 1;
+
+    if (cpu_number < 0) {
+        return -1;
+    }
+    vapic_paddr = s->vapic_paddr +
+        (((hwaddr)cpu_number) << VAPIC_CPU_SHIFT);
+    cpu_physical_memory_rw(vapic_paddr + offsetof(VAPICState, enabled),
+                           (void *)&enabled, sizeof(enabled), 1);
+    s->state = VAPIC_ACTIVE;
+
+    return 0;
+}
+
 static int vapic_enable(VAPICROMState *s, X86CPU *cpu)
 {
     int cpu_number = get_kpcr_number(cpu);
@@ -531,7 +549,7 @@ static int patch_hypercalls(VAPICROMState *s)
     int patches = 0;
     off_t pos;
     uint8_t *rom;
-
+    
     rom = g_malloc(s->rom_size);
     cpu_physical_memory_read(rom_paddr, rom, s->rom_size);
 
@@ -559,11 +577,10 @@ static int patch_hypercalls(VAPICROMState *s)
     }
 
     g_free(rom);
-
+    
     if (patches != 0 && patches != 2) {
         return -1;
     }
-
     return 0;
 }
 
@@ -729,9 +746,9 @@ static void vapic_realize(DeviceState *dev, Error **errp)
 static void do_vapic_enable(void *data)
 {
     VAPICROMState *s = data;
-    X86CPU *cpu = X86_CPU(first_cpu);
-
-    vapic_enable(s, cpu);
+    // Do not synchronize with APIC, because it was not loaded yet.
+    // Just call the enable function which does not have synchronization.
+    vapic_enable_post_load(s, X86_CPU(first_cpu));
 }
 
 static int vapic_post_load(void *opaque, int version_id)
@@ -746,7 +763,6 @@ static int vapic_post_load(void *opaque, int version_id)
     if (s->state == VAPIC_INACTIVE && s->rom_state_paddr != 0) {
         s->state = VAPIC_STANDBY;
     }
-
     if (s->state != VAPIC_INACTIVE) {
         if (vapic_prepare(s) < 0) {
             return -1;
diff --git a/hw/ide/ahci.c b/hw/ide/ahci.c
index 9bae22e..6e7afb6
--- a/hw/ide/ahci.c
+++ b/hw/ide/ahci.c
@@ -32,6 +32,8 @@
 #include "internal.h"
 #include <hw/ide/pci.h>
 #include <hw/ide/ahci.h>
+#include "replay/replay.h"
+#include "qemu/log.h"
 
 /* #define DEBUG_AHCI */
 
@@ -612,7 +614,7 @@ static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis)
     d2h_fis[1] = (ad->hba->control_regs.irqstatus ? (1 << 6) : 0);
     d2h_fis[2] = ad->port.ifs[0].status;
     d2h_fis[3] = ad->port.ifs[0].error;
-
+    
     d2h_fis[4] = cmd_fis[4];
     d2h_fis[5] = cmd_fis[5];
     d2h_fis[6] = cmd_fis[6];
@@ -674,7 +676,7 @@ static int ahci_populate_sglist(AHCIDevice *ad, QEMUSGList *sglist, int offset)
         r = -1;
         goto out;
     }
-
+    
     /* Get entries in the PRDT, init a qemu sglist accordingly */
     if (sglist_alloc_hint > 0) {
         AHCI_SG *tbl = (AHCI_SG *)prdt;
@@ -861,7 +863,7 @@ static int handle_cmd(AHCIState *s, int port, int slot)
 
     debug_print_fis(cmd_fis, 0x90);
     //debug_print_fis(cmd_fis, (opts & AHCI_CMD_HDR_CMD_FIS_LEN) * 4);
-
+    
     switch (cmd_fis[0]) {
         case SATA_FIS_TYPE_REGISTER_H2D:
             break;
@@ -1126,7 +1128,7 @@ static int ahci_async_cmd_done(IDEDMA *dma)
 
     if (!ad->check_bh) {
         /* maybe we still have something to process, check later */
-        ad->check_bh = qemu_bh_new(ahci_check_cmd_bh, ad);
+        ad->check_bh = qemu_bh_new_replay(ahci_check_cmd_bh, ad, replay_get_current_step());
         qemu_bh_schedule(ad->check_bh);
     }
 
diff --git a/hw/input/pckbd.c b/hw/input/pckbd.c
index ca1cffc..19f6658 100644
--- a/hw/input/pckbd.c
+++ b/hw/input/pckbd.c
@@ -371,13 +371,14 @@ static void kbd_reset(void *opaque)
 
 static const VMStateDescription vmstate_kbd = {
     .name = "pckbd",
-    .version_id = 3,
+    .version_id = 4,
     .minimum_version_id = 3,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(write_cmd, KBDState),
         VMSTATE_UINT8(status, KBDState),
         VMSTATE_UINT8(mode, KBDState),
         VMSTATE_UINT8(pending, KBDState),
+        VMSTATE_UINT8_V(outport, KBDState, 4),
         VMSTATE_END_OF_LIST()
     }
 };
diff --git a/hw/intc/apic_common.c b/hw/intc/apic_common.c
index ce3d903..d93c00d 100644
--- a/hw/intc/apic_common.c
+++ b/hw/intc/apic_common.c
@@ -347,7 +347,7 @@ static int apic_dispatch_post_load(void *opaque, int version_id)
 
 static const VMStateDescription vmstate_apic_common = {
     .name = "apic",
-    .version_id = 3,
+    .version_id = 5,
     .minimum_version_id = 3,
     .minimum_version_id_old = 1,
     .load_state_old = apic_load_old,
@@ -374,6 +374,9 @@ static const VMStateDescription vmstate_apic_common = {
         VMSTATE_INT64(next_time, APICCommonState),
         VMSTATE_INT64(timer_expiry,
                       APICCommonState), /* open-coded timer state */
+        VMSTATE_INT32_V(sipi_vector, APICCommonState, 4),
+        VMSTATE_INT32_V(wait_for_sipi, APICCommonState, 4),
+        VMSTATE_UINT64_V(vapic_paddr, APICCommonState, 5),
         VMSTATE_END_OF_LIST()
     }
 };
diff --git a/hw/net/rtl8139.c b/hw/net/rtl8139.c
index 90bc5ec..992caf0 100644
--- a/hw/net/rtl8139.c
+++ b/hw/net/rtl8139.c
@@ -3289,7 +3289,7 @@ static void rtl8139_pre_save(void *opaque)
 
 static const VMStateDescription vmstate_rtl8139 = {
     .name = "rtl8139",
-    .version_id = 4,
+    .version_id = 5,
     .minimum_version_id = 3,
     .post_load = rtl8139_post_load,
     .pre_save  = rtl8139_pre_save,
@@ -3363,6 +3363,9 @@ static const VMStateDescription vmstate_rtl8139 = {
         VMSTATE_STRUCT(tally_counters, RTL8139State, 0,
                        vmstate_tally_counters, RTL8139TallyCounters),
 
+        VMSTATE_TIMER_V(timer, RTL8139State, 5),
+        VMSTATE_INT64_V(TimerExpire, RTL8139State, 5),
+
         VMSTATE_UINT32_V(cplus_enabled, RTL8139State, 4),
         VMSTATE_END_OF_LIST()
     },
diff --git a/hw/pci-host/piix.c b/hw/pci-host/piix.c
index e0e0946..86d6d20 100644
--- a/hw/pci-host/piix.c
+++ b/hw/pci-host/piix.c
@@ -409,7 +409,7 @@ static void piix3_set_irq_pic(PIIX3State *piix3, int pic_irq)
                      (pic_irq * PIIX_NUM_PIRQS))));
 }
 
-static void piix3_set_irq_level(PIIX3State *piix3, int pirq, int level)
+static void piix3_set_irq_level_internal(PIIX3State *piix3, int pirq, int level)
 {
     int pic_irq;
     uint64_t mask;
@@ -422,6 +422,18 @@ static void piix3_set_irq_level(PIIX3State *piix3, int pirq, int level)
     mask = 1ULL << ((pic_irq * PIIX_NUM_PIRQS) + pirq);
     piix3->pic_levels &= ~mask;
     piix3->pic_levels |= mask * !!level;
+}
+
+static void piix3_set_irq_level(PIIX3State *piix3, int pirq, int level)
+{
+    int pic_irq;
+
+    pic_irq = piix3->dev.config[PIIX_PIRQC + pirq];
+    if (pic_irq >= PIIX_NUM_PIC_IRQS) {
+        return;
+    }
+
+    piix3_set_irq_level_internal(piix3, pirq, level);
 
     piix3_set_irq_pic(piix3, pic_irq);
 }
@@ -527,7 +539,13 @@ static void piix3_reset(void *opaque)
 static int piix3_post_load(void *opaque, int version_id)
 {
     PIIX3State *piix3 = opaque;
-    piix3_update_irq_levels(piix3);
+    int pirq;
+
+    piix3->pic_levels = 0;
+    for (pirq = 0; pirq < PIIX_NUM_PIRQS; pirq++) {
+        piix3_set_irq_level_internal(piix3, pirq,
+                            pci_bus_get_irq_level(piix3->dev.bus, pirq));
+    }
     return 0;
 }
 
diff --git a/hw/timer/arm_timer.c b/hw/timer/arm_timer.c
index 1452910..97784a0 100644
--- a/hw/timer/arm_timer.c
+++ b/hw/timer/arm_timer.c
@@ -168,7 +168,7 @@ static arm_timer_state *arm_timer_init(uint32_t freq)
     s->freq = freq;
     s->control = TIMER_CTRL_IE;
 
-    bh = qemu_bh_new(arm_timer_tick, s);
+    bh = qemu_bh_new_replay(arm_timer_tick, s, 0);
     s->timer = ptimer_init(bh);
     vmstate_register(NULL, -1, &vmstate_arm_timer, s);
     return s;
diff --git a/hw/timer/hpet.c b/hw/timer/hpet.c
index e160e8f..2605266
--- a/hw/timer/hpet.c
+++ b/hw/timer/hpet.c
@@ -224,10 +224,11 @@ static void update_irq(struct HPETTimer *timer, int set)
 
 static void hpet_pre_save(void *opaque)
 {
-    HPETState *s = opaque;
+    //HPETState *s = opaque;
 
+    /* VM clock does not run while saving, so there is no need for saving the ticks */
     /* save current counter value */
-    s->hpet_counter = hpet_get_ticks(s);
+    //s->hpet_counter = hpet_get_ticks(s);
 }
 
 static int hpet_pre_load(void *opaque)
@@ -255,8 +256,9 @@ static int hpet_post_load(void *opaque, int version_id)
 {
     HPETState *s = opaque;
 
+    /* VM clock does not run while saving, so there is no need for saving the ticks */
     /* Recalculate the offset between the main counter and guest time */
-    s->hpet_offset = ticks_to_ns(s->hpet_counter) - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+    //s->hpet_offset = ticks_to_ns(s->hpet_counter) - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
 
     /* Push number of timers into capability returned via HPET_ID */
     s->capability &= ~HPET_ID_NUM_TIM_MASK;
@@ -315,6 +317,7 @@ static const VMStateDescription vmstate_hpet = {
         VMSTATE_UINT64(config, HPETState),
         VMSTATE_UINT64(isr, HPETState),
         VMSTATE_UINT64(hpet_counter, HPETState),
+        VMSTATE_UINT64(hpet_offset, HPETState),
         VMSTATE_UINT8_V(num_timers, HPETState, 2),
         VMSTATE_VALIDATE("num_timers in range", hpet_validate_num_timers),
         VMSTATE_STRUCT_VARRAY_UINT8(timer, HPETState, num_timers, 0,
diff --git a/hw/timer/mc146818rtc.c b/hw/timer/mc146818rtc.c
index 307732c..84d6483
--- a/hw/timer/mc146818rtc.c
+++ b/hw/timer/mc146818rtc.c
@@ -33,6 +33,8 @@
 #include "hw/i386/apic.h"
 #endif
 
+#include "replay/replay.h"
+
 //#define DEBUG_CMOS
 //#define DEBUG_COALESCED
 
@@ -703,7 +705,7 @@ static void rtc_set_date_from_host(ISADevice *dev)
     RTCState *s = MC146818_RTC(dev);
     struct tm tm;
 
-    qemu_get_timedate(&tm, 0);
+    qemu_get_timedate_no_warning(&tm, 0);
 
     s->base_rtc = mktimegm(&tm);
     s->last_update = qemu_clock_get_ns(rtc_clock);
@@ -735,7 +737,7 @@ static int rtc_post_load(void *opaque, int version_id)
 
 static const VMStateDescription vmstate_rtc = {
     .name = "mc146818rtc",
-    .version_id = 3,
+    .version_id = 4,
     .minimum_version_id = 1,
     .post_load = rtc_post_load,
     .fields = (VMStateField[]) {
@@ -752,6 +754,7 @@ static const VMStateDescription vmstate_rtc = {
         VMSTATE_INT64_V(offset, RTCState, 3),
         VMSTATE_TIMER_V(update_timer, RTCState, 3),
         VMSTATE_UINT64_V(next_alarm_time, RTCState, 3),
+        VMSTATE_UINT16_V(irq_reinject_on_ack_count, RTCState, 4),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -851,6 +854,14 @@ out:
     error_propagate(errp, err);
 }
 
+static void rtc_get_date_replay(ISADevice *dev, struct tm *tm)
+{
+    RTCState *s = MC146818_RTC(dev);
+    rtc_get_time(s, tm);
+    ++tm->tm_mon;
+    tm->tm_year += 1900;
+}
+
 static void rtc_realizefn(DeviceState *dev, Error **errp)
 {
     ISADevice *isadev = ISA_DEVICE(dev);
@@ -904,6 +915,8 @@ static void rtc_realizefn(DeviceState *dev, Error **errp)
     memory_region_init_io(&s->io, OBJECT(s), &cmos_ops, s, "rtc", 2);
     isa_register_ioport(isadev, &s->io, base);
 
+    replay_register_get_time(rtc_get_date_replay, s);
+
     qdev_set_legacy_instance_id(dev, base, 3);
     qemu_register_reset(rtc_reset, s);
 
diff --git a/hw/timer/pl031.c b/hw/timer/pl031.c
index 34d9b44..966f10f
--- a/hw/timer/pl031.c
+++ b/hw/timer/pl031.c
@@ -200,7 +200,12 @@ static int pl031_init(SysBusDevice *dev)
     sysbus_init_mmio(dev, &s->iomem);
 
     sysbus_init_irq(dev, &s->irq);
-    qemu_get_timedate(&tm, 0);
+    qemu_get_timedate_no_warning(&tm, 0);
+    if (replay_mode == REPLAY_SAVE) {
+        replay_save_tm(&tm);
+    } else if (replay_mode == REPLAY_PLAY) {
+        replay_read_tm(&tm);
+    }
     s->tick_offset = mktimegm(&tm) -
         qemu_clock_get_ns(rtc_clock) / get_ticks_per_sec();
 
@@ -212,30 +217,39 @@ static void pl031_pre_save(void *opaque)
 {
     PL031State *s = opaque;
 
-    /* tick_offset is base_time - rtc_clock base time.  Instead, we want to
-     * store the base time relative to the QEMU_CLOCK_VIRTUAL for backwards-compatibility.  */
-    int64_t delta = qemu_clock_get_ns(rtc_clock) - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    s->tick_offset_vmstate = s->tick_offset + delta / get_ticks_per_sec();
+    if (replay_mode == REPLAY_NONE) {
+        /* tick_offset is base_time - rtc_clock base time.  Instead, we want to
+         * store the base time relative to the QEMU_CLOCK_VIRTUAL for backwards-compatibility.  */
+        int64_t delta = qemu_clock_get_ns(rtc_clock) - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+        s->tick_offset_vmstate = s->tick_offset + delta / get_ticks_per_sec();
+    } else {
+        s->tick_offset_vmstate = s->tick_offset;
+    }
 }
 
 static int pl031_post_load(void *opaque, int version_id)
 {
     PL031State *s = opaque;
 
-    int64_t delta = qemu_clock_get_ns(rtc_clock) - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    s->tick_offset = s->tick_offset_vmstate - delta / get_ticks_per_sec();
-    pl031_set_alarm(s);
+    if (replay_mode == REPLAY_NONE) {
+        int64_t delta = qemu_clock_get_ns(rtc_clock) - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+        s->tick_offset = s->tick_offset_vmstate - delta / get_ticks_per_sec();
+        pl031_set_alarm(s);
+    } else {
+        s->tick_offset = s->tick_offset_vmstate;
+    }
     return 0;
 }
 
 static const VMStateDescription vmstate_pl031 = {
     .name = "pl031",
-    .version_id = 1,
+    .version_id = 2,
     .minimum_version_id = 1,
     .pre_save = pl031_pre_save,
     .post_load = pl031_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(tick_offset_vmstate, PL031State),
+	VMSTATE_TIMER_V(timer, PL031State, 2),
         VMSTATE_UINT32(mr, PL031State),
         VMSTATE_UINT32(lr, PL031State),
         VMSTATE_UINT32(cr, PL031State),

diff --git a/include/hw/char/serial.h b/include/hw/char/serial.h
index f431764..41982f5 100644
--- a/include/hw/char/serial.h
+++ b/include/hw/char/serial.h
@@ -80,7 +80,7 @@ extern const MemoryRegionOps serial_io_ops;
 
 void serial_realize_core(SerialState *s, Error **errp);
 void serial_exit_core(SerialState *s);
-void serial_set_frequency(SerialState *s, uint32_t frequency);
+//void serial_set_frequency(SerialState *s, uint32_t frequency);
 
 /* legacy pre qom */
 SerialState *serial_init(int base, qemu_irq irq, int baudbase,