[PATCH v2 0/2] hw/pflash: implement update buffer for block writes

[PATCH v2 0/2] hw/pflash: implement update buffer for block writes

[Philippe Mathieu-Daudé]

Respin of Gerd's series [*] using the LD/ST API.

v1 [*] cover:

  When running qemu with edk2 efi firmware on aarch64 the efi
  variable store in pflash can get corrupted.  qemu not doing
  proper block writes -- flush all or nothing to storage -- is
  a hot candidate for being the root cause.

  This little series tries to fix that with an update buffer
  where block writes are staged, so we can commit or discard
  the changes when the block write is completed or canceled.

[*] https://lore.kernel.org/qemu-devel/20240105135855.268064-1-kraxel@redhat.com/

Gerd Hoffmann (1):
  hw/pflash: implement update buffer for block writes

Philippe Mathieu-Daudé (1):
  hw/block/pflash_cfi01: Use the LD/ST API in pflash_data_read/write

 hw/block/pflash_cfi01.c | 165 +++++++++++++++++++++-------------------
 1 file changed, 86 insertions(+), 79 deletions(-)

-- 
2.41.0

[PATCH v2 1/2] hw/block/pflash_cfi01: Use the LD/ST API in pflash_data_read/write

[Philippe Mathieu-Daudé]

Similarly to commit c3d25271b2 ("hw/block/pflash_cfi02: Use the
ldst API in pflash_write") for CFI type 2, use the LD/ST API for
CFI type 1.

Signed-off-by: Philippe Mathieu-Daudé <philmd@linaro.org>
---
 hw/block/pflash_cfi01.c | 63 ++++++-----------------------------------
 1 file changed, 8 insertions(+), 55 deletions(-)

diff --git a/hw/block/pflash_cfi01.c b/hw/block/pflash_cfi01.c
index 3e2dc08bd7..ce63ba43b6 100644
--- a/hw/block/pflash_cfi01.c
+++ b/hw/block/pflash_cfi01.c
@@ -225,34 +225,10 @@ static uint32_t pflash_data_read(PFlashCFI01 *pfl, hwaddr offset,
     uint32_t ret;
 
     p = pfl->storage;
-    switch (width) {
-    case 1:
-        ret = p[offset];
-        break;
-    case 2:
-        if (be) {
-            ret = p[offset] << 8;
-            ret |= p[offset + 1];
-        } else {
-            ret = p[offset];
-            ret |= p[offset + 1] << 8;
-        }
-        break;
-    case 4:
-        if (be) {
-            ret = p[offset] << 24;
-            ret |= p[offset + 1] << 16;
-            ret |= p[offset + 2] << 8;
-            ret |= p[offset + 3];
-        } else {
-            ret = p[offset];
-            ret |= p[offset + 1] << 8;
-            ret |= p[offset + 2] << 16;
-            ret |= p[offset + 3] << 24;
-        }
-        break;
-    default:
-        abort();
+    if (be) {
+        ret = ldn_be_p(p, width);
+    } else {
+        ret = ldn_le_p(p, width);
     }
     trace_pflash_data_read(pfl->name, offset, width, ret);
     return ret;
@@ -406,34 +382,11 @@ static inline void pflash_data_write(PFlashCFI01 *pfl, hwaddr offset,
     uint8_t *p = pfl->storage;
 
     trace_pflash_data_write(pfl->name, offset, width, value, pfl->counter);
-    switch (width) {
-    case 1:
-        p[offset] = value;
-        break;
-    case 2:
-        if (be) {
-            p[offset] = value >> 8;
-            p[offset + 1] = value;
-        } else {
-            p[offset] = value;
-            p[offset + 1] = value >> 8;
-        }
-        break;
-    case 4:
-        if (be) {
-            p[offset] = value >> 24;
-            p[offset + 1] = value >> 16;
-            p[offset + 2] = value >> 8;
-            p[offset + 3] = value;
-        } else {
-            p[offset] = value;
-            p[offset + 1] = value >> 8;
-            p[offset + 2] = value >> 16;
-            p[offset + 3] = value >> 24;
-        }
-        break;
+    if (be) {
+        stn_be_p(p, width, value);
+    } else {
+        stn_le_p(p, width, value);
     }
-
 }
 
 static void pflash_write(PFlashCFI01 *pfl, hwaddr offset,
-- 
2.41.0

[PATCH v2 2/2] hw/pflash: implement update buffer for block writes

[Philippe Mathieu-Daudé]

From: Gerd Hoffmann <kraxel@redhat.com>

Add an update buffer where all block updates are staged.
Flush or discard updates properly, so we should never see
half-completed block writes in pflash storage.

Drop a bunch of FIXME comments ;)

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
Message-ID: <20240105135855.268064-3-kraxel@redhat.com>
Signed-off-by: Philippe Mathieu-Daudé <philmd@linaro.org>
---
 hw/block/pflash_cfi01.c | 106 ++++++++++++++++++++++++++++++----------
 1 file changed, 80 insertions(+), 26 deletions(-)

diff --git a/hw/block/pflash_cfi01.c b/hw/block/pflash_cfi01.c
index ce63ba43b6..0120462648 100644
--- a/hw/block/pflash_cfi01.c
+++ b/hw/block/pflash_cfi01.c
@@ -80,16 +80,39 @@ struct PFlashCFI01 {
     uint16_t ident3;
     uint8_t cfi_table[0x52];
     uint64_t counter;
-    unsigned int writeblock_size;
+    uint32_t writeblock_size;
     MemoryRegion mem;
     char *name;
     void *storage;
     VMChangeStateEntry *vmstate;
     bool old_multiple_chip_handling;
+
+    /* block update buffer */
+    unsigned char *blk_bytes;
+    uint32_t blk_offset;
 };
 
 static int pflash_post_load(void *opaque, int version_id);
 
+static bool pflash_blk_write_state_needed(void *opaque)
+{
+    PFlashCFI01 *pfl = opaque;
+
+    return (pfl->blk_offset != -1);
+}
+
+static const VMStateDescription vmstate_pflash_blk_write = {
+    .name = "pflash_cfi01_blk_write",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = pflash_blk_write_state_needed,
+    .fields = (const VMStateField[]) {
+        VMSTATE_VBUFFER_UINT32(blk_bytes, PFlashCFI01, 0, NULL, writeblock_size),
+        VMSTATE_UINT32(blk_offset, PFlashCFI01),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_pflash = {
     .name = "pflash_cfi01",
     .version_id = 1,
@@ -101,6 +124,10 @@ static const VMStateDescription vmstate_pflash = {
         VMSTATE_UINT8(status, PFlashCFI01),
         VMSTATE_UINT64(counter, PFlashCFI01),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription * const []) {
+        &vmstate_pflash_blk_write,
+        NULL
     }
 };
 
@@ -376,12 +403,51 @@ static void pflash_update(PFlashCFI01 *pfl, int offset,
     }
 }
 
+/* copy current flash content to block update buffer */
+static void pflash_blk_write_start(PFlashCFI01 *pfl, hwaddr offset)
+{
+    hwaddr mask = ~(pfl->writeblock_size - 1);
+
+    pfl->blk_offset = offset & mask;
+    memcpy(pfl->blk_bytes, pfl->storage + pfl->blk_offset,
+           pfl->writeblock_size);
+}
+
+/* commit block update buffer changes */
+static void pflash_blk_write_flush(PFlashCFI01 *pfl)
+{
+    g_assert(pfl->blk_offset != -1);
+    memcpy(pfl->storage + pfl->blk_offset, pfl->blk_bytes,
+           pfl->writeblock_size);
+    pflash_update(pfl, pfl->blk_offset, pfl->writeblock_size);
+    pfl->blk_offset = -1;
+}
+
+/* discard block update buffer changes */
+static void pflash_blk_write_abort(PFlashCFI01 *pfl)
+{
+    pfl->blk_offset = -1;
+}
+
 static inline void pflash_data_write(PFlashCFI01 *pfl, hwaddr offset,
                                      uint32_t value, int width, int be)
 {
-    uint8_t *p = pfl->storage;
+    uint8_t *p;
+
+    if (pfl->blk_offset != -1) {
+        /* block write: redirect writes to block update buffer */
+        if ((offset < pfl->blk_offset) ||
+            (offset + width > pfl->blk_offset + pfl->writeblock_size)) {
+            pfl->status |= 0x10; /* Programming error */
+            return;
+        }
+        p = pfl->blk_bytes + (offset - pfl->blk_offset);
+    } else {
+        /* write directly to storage */
+        trace_pflash_data_write(pfl->name, offset, width, value, pfl->counter);
+        p = pfl->storage + offset;
+    }
 
-    trace_pflash_data_write(pfl->name, offset, width, value, pfl->counter);
     if (be) {
         stn_be_p(p, width, value);
     } else {
@@ -504,6 +570,7 @@ static void pflash_write(PFlashCFI01 *pfl, hwaddr offset,
             trace_pflash_write_block(pfl->name, value);
             pfl->counter = value;
             pfl->wcycle++;
+            pflash_blk_write_start(pfl, offset);
             break;
         case 0x60:
             if (cmd == 0xd0) {
@@ -534,12 +601,7 @@ static void pflash_write(PFlashCFI01 *pfl, hwaddr offset,
         switch (pfl->cmd) {
         case 0xe8: /* Block write */
             /* FIXME check @offset, @width */
-            if (!pfl->ro) {
-                /*
-                 * FIXME writing straight to memory is *wrong*.  We
-                 * should write to a buffer, and flush it to memory
-                 * only on confirm command (see below).
-                 */
+            if (!pfl->ro && (pfl->blk_offset != -1)) {
                 pflash_data_write(pfl, offset, value, width, be);
             } else {
                 pfl->status |= 0x10; /* Programming error */
@@ -548,18 +610,8 @@ static void pflash_write(PFlashCFI01 *pfl, hwaddr offset,
             pfl->status |= 0x80;
 
             if (!pfl->counter) {
-                hwaddr mask = pfl->writeblock_size - 1;
-                mask = ~mask;
-
                 trace_pflash_write(pfl->name, "block write finished");
                 pfl->wcycle++;
-                if (!pfl->ro) {
-                    /* Flush the entire write buffer onto backing storage.  */
-                    /* FIXME premature! */
-                    pflash_update(pfl, offset & mask, pfl->writeblock_size);
-                } else {
-                    pfl->status |= 0x10; /* Programming error */
-                }
             }
 
             pfl->counter--;
@@ -571,20 +623,17 @@ static void pflash_write(PFlashCFI01 *pfl, hwaddr offset,
     case 3: /* Confirm mode */
         switch (pfl->cmd) {
         case 0xe8: /* Block write */
-            if (cmd == 0xd0) {
-                /* FIXME this is where we should write out the buffer */
+            if ((cmd == 0xd0) && !(pfl->status & 0x10)) {
+                pflash_blk_write_flush(pfl);
                 pfl->wcycle = 0;
                 pfl->status |= 0x80;
             } else {
-                qemu_log_mask(LOG_UNIMP,
-                    "%s: Aborting write to buffer not implemented,"
-                    " the data is already written to storage!\n"
-                    "Flash device reset into READ mode.\n",
-                    __func__);
+                pflash_blk_write_abort(pfl);
                 goto mode_read_array;
             }
             break;
         default:
+            pflash_blk_write_abort(pfl);
             goto error_flash;
         }
         break;
@@ -818,6 +867,9 @@ static void pflash_cfi01_realize(DeviceState *dev, Error **errp)
     pfl->cmd = 0x00;
     pfl->status = 0x80; /* WSM ready */
     pflash_cfi01_fill_cfi_table(pfl);
+
+    pfl->blk_bytes = g_malloc(pfl->writeblock_size);
+    pfl->blk_offset = -1;
 }
 
 static void pflash_cfi01_system_reset(DeviceState *dev)
@@ -837,6 +889,8 @@ static void pflash_cfi01_system_reset(DeviceState *dev)
      * This model deliberately ignores this delay.
      */
     pfl->status = 0x80;
+
+    pfl->blk_offset = -1;
 }
 
 static Property pflash_cfi01_properties[] = {
-- 
2.41.0

Re: [PATCH v2 2/2] hw/pflash: implement update buffer for block writes

[Philippe Mathieu-Daudé]

Hi Gerd,

On 8/1/24 13:53, Philippe Mathieu-Daudé wrote:
> From: Gerd Hoffmann <kraxel@redhat.com>
> 
> Add an update buffer where all block updates are staged.
> Flush or discard updates properly, so we should never see
> half-completed block writes in pflash storage.
> 
> Drop a bunch of FIXME comments ;)
> 
> Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
> Message-ID: <20240105135855.268064-3-kraxel@redhat.com>
> Signed-off-by: Philippe Mathieu-Daudé <philmd@linaro.org>
> ---
>   hw/block/pflash_cfi01.c | 106 ++++++++++++++++++++++++++++++----------
>   1 file changed, 80 insertions(+), 26 deletions(-)
> 
> diff --git a/hw/block/pflash_cfi01.c b/hw/block/pflash_cfi01.c
> index ce63ba43b6..0120462648 100644
> --- a/hw/block/pflash_cfi01.c
> +++ b/hw/block/pflash_cfi01.c
> @@ -80,16 +80,39 @@ struct PFlashCFI01 {
>       uint16_t ident3;
>       uint8_t cfi_table[0x52];
>       uint64_t counter;
> -    unsigned int writeblock_size;
> +    uint32_t writeblock_size;
>       MemoryRegion mem;
>       char *name;
>       void *storage;
>       VMChangeStateEntry *vmstate;
>       bool old_multiple_chip_handling;
> +
> +    /* block update buffer */
> +    unsigned char *blk_bytes;

I'd rather use a 'void *' type here, but then we need to
use a (uinptr_t) cast in pflash_data_write().

> +    uint32_t blk_offset;
>   };
>   
>   static int pflash_post_load(void *opaque, int version_id);
>   
> +static bool pflash_blk_write_state_needed(void *opaque)
> +{
> +    PFlashCFI01 *pfl = opaque;
> +
> +    return (pfl->blk_offset != -1);
> +}
> +
> +static const VMStateDescription vmstate_pflash_blk_write = {
> +    .name = "pflash_cfi01_blk_write",
> +    .version_id = 1,
> +    .minimum_version_id = 1,
> +    .needed = pflash_blk_write_state_needed,
> +    .fields = (const VMStateField[]) {
> +        VMSTATE_VBUFFER_UINT32(blk_bytes, PFlashCFI01, 0, NULL, writeblock_size),

I don't get the difference with VMSTATE_VBUFFER_ALLOC_UINT32() which
sets VMS_ALLOC. In this case pflash_cfi01_realize() does the alloc so
we don't need VMS_ALLOC?

> +        VMSTATE_UINT32(blk_offset, PFlashCFI01),
> +        VMSTATE_END_OF_LIST()
> +    }
> +};
> +
>   static const VMStateDescription vmstate_pflash = {
>       .name = "pflash_cfi01",
>       .version_id = 1,
> @@ -101,6 +124,10 @@ static const VMStateDescription vmstate_pflash = {
>           VMSTATE_UINT8(status, PFlashCFI01),
>           VMSTATE_UINT64(counter, PFlashCFI01),
>           VMSTATE_END_OF_LIST()
> +    },
> +    .subsections = (const VMStateDescription * const []) {
> +        &vmstate_pflash_blk_write,
> +        NULL
>       }
>   };
>   
> @@ -376,12 +403,51 @@ static void pflash_update(PFlashCFI01 *pfl, int offset,
>       }
>   }
>   
> +/* copy current flash content to block update buffer */
> +static void pflash_blk_write_start(PFlashCFI01 *pfl, hwaddr offset)
> +{
> +    hwaddr mask = ~(pfl->writeblock_size - 1);
> +
> +    pfl->blk_offset = offset & mask;
> +    memcpy(pfl->blk_bytes, pfl->storage + pfl->blk_offset,
> +           pfl->writeblock_size);
> +}
> +
> +/* commit block update buffer changes */
> +static void pflash_blk_write_flush(PFlashCFI01 *pfl)
> +{
> +    g_assert(pfl->blk_offset != -1);
> +    memcpy(pfl->storage + pfl->blk_offset, pfl->blk_bytes,
> +           pfl->writeblock_size);
> +    pflash_update(pfl, pfl->blk_offset, pfl->writeblock_size);
> +    pfl->blk_offset = -1;
> +}
> +
> +/* discard block update buffer changes */
> +static void pflash_blk_write_abort(PFlashCFI01 *pfl)
> +{
> +    pfl->blk_offset = -1;
> +}
> +
>   static inline void pflash_data_write(PFlashCFI01 *pfl, hwaddr offset,
>                                        uint32_t value, int width, int be)
>   {
> -    uint8_t *p = pfl->storage;
> +    uint8_t *p;
> +
> +    if (pfl->blk_offset != -1) {

I'd rather have a trace event in this if() ladder.

> +        /* block write: redirect writes to block update buffer */
> +        if ((offset < pfl->blk_offset) ||
> +            (offset + width > pfl->blk_offset + pfl->writeblock_size)) {
> +            pfl->status |= 0x10; /* Programming error */
> +            return;
> +        }
> +        p = pfl->blk_bytes + (offset - pfl->blk_offset);
> +    } else {
> +        /* write directly to storage */
> +        trace_pflash_data_write(pfl->name, offset, width, value, pfl->counter);
> +        p = pfl->storage + offset;
> +    }
>   
> -    trace_pflash_data_write(pfl->name, offset, width, value, pfl->counter);
>       if (be) {
>           stn_be_p(p, width, value);
>       } else {
> @@ -504,6 +570,7 @@ static void pflash_write(PFlashCFI01 *pfl, hwaddr offset,
>               trace_pflash_write_block(pfl->name, value);
>               pfl->counter = value;
>               pfl->wcycle++;
> +            pflash_blk_write_start(pfl, offset);
>               break;
>           case 0x60:
>               if (cmd == 0xd0) {
> @@ -534,12 +601,7 @@ static void pflash_write(PFlashCFI01 *pfl, hwaddr offset,
>           switch (pfl->cmd) {
>           case 0xe8: /* Block write */
>               /* FIXME check @offset, @width */
> -            if (!pfl->ro) {
> -                /*
> -                 * FIXME writing straight to memory is *wrong*.  We
> -                 * should write to a buffer, and flush it to memory
> -                 * only on confirm command (see below).
> -                 */
> +            if (!pfl->ro && (pfl->blk_offset != -1)) {
>                   pflash_data_write(pfl, offset, value, width, be);
>               } else {
>                   pfl->status |= 0x10; /* Programming error */
> @@ -548,18 +610,8 @@ static void pflash_write(PFlashCFI01 *pfl, hwaddr offset,
>               pfl->status |= 0x80;
>   
>               if (!pfl->counter) {
> -                hwaddr mask = pfl->writeblock_size - 1;
> -                mask = ~mask;
> -
>                   trace_pflash_write(pfl->name, "block write finished");
>                   pfl->wcycle++;
> -                if (!pfl->ro) {
> -                    /* Flush the entire write buffer onto backing storage.  */
> -                    /* FIXME premature! */
> -                    pflash_update(pfl, offset & mask, pfl->writeblock_size);
> -                } else {
> -                    pfl->status |= 0x10; /* Programming error */
> -                }
>               }
>   
>               pfl->counter--;
> @@ -571,20 +623,17 @@ static void pflash_write(PFlashCFI01 *pfl, hwaddr offset,
>       case 3: /* Confirm mode */
>           switch (pfl->cmd) {
>           case 0xe8: /* Block write */
> -            if (cmd == 0xd0) {
> -                /* FIXME this is where we should write out the buffer */
> +            if ((cmd == 0xd0) && !(pfl->status & 0x10)) {
> +                pflash_blk_write_flush(pfl);
>                   pfl->wcycle = 0;
>                   pfl->status |= 0x80;
>               } else {
> -                qemu_log_mask(LOG_UNIMP,
> -                    "%s: Aborting write to buffer not implemented,"
> -                    " the data is already written to storage!\n"
> -                    "Flash device reset into READ mode.\n",
> -                    __func__);
> +                pflash_blk_write_abort(pfl);
>                   goto mode_read_array;
>               }
>               break;
>           default:
> +            pflash_blk_write_abort(pfl);
>               goto error_flash;
>           }
>           break;
> @@ -818,6 +867,9 @@ static void pflash_cfi01_realize(DeviceState *dev, Error **errp)
>       pfl->cmd = 0x00;
>       pfl->status = 0x80; /* WSM ready */
>       pflash_cfi01_fill_cfi_table(pfl);
> +
> +    pfl->blk_bytes = g_malloc(pfl->writeblock_size);
> +    pfl->blk_offset = -1;
>   }
>   
>   static void pflash_cfi01_system_reset(DeviceState *dev)
> @@ -837,6 +889,8 @@ static void pflash_cfi01_system_reset(DeviceState *dev)
>        * This model deliberately ignores this delay.
>        */
>       pfl->status = 0x80;
> +
> +    pfl->blk_offset = -1;
>   }
>   
>   static Property pflash_cfi01_properties[] = {

Patch LGTM. If you want I can apply the changes suggested
and post a v3/queue.

Regards,

Phil.

Re: [PATCH v2 2/2] hw/pflash: implement update buffer for block writes

[Peter Maydell]

On Mon, 8 Jan 2024 at 13:06, Philippe Mathieu-Daudé <philmd@linaro.org> wrote:
>
> Hi Gerd,
>
> On 8/1/24 13:53, Philippe Mathieu-Daudé wrote:
> > From: Gerd Hoffmann <kraxel@redhat.com>
> >
> > Add an update buffer where all block updates are staged.
> > Flush or discard updates properly, so we should never see
> > half-completed block writes in pflash storage.
> >
> > Drop a bunch of FIXME comments ;)
> >
> > Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
> > Message-ID: <20240105135855.268064-3-kraxel@redhat.com>
> > Signed-off-by: Philippe Mathieu-Daudé <philmd@linaro.org>
> > ---
> >   hw/block/pflash_cfi01.c | 106 ++++++++++++++++++++++++++++++----------
> >   1 file changed, 80 insertions(+), 26 deletions(-)
> >
> > diff --git a/hw/block/pflash_cfi01.c b/hw/block/pflash_cfi01.c
> > index ce63ba43b6..0120462648 100644
> > --- a/hw/block/pflash_cfi01.c
> > +++ b/hw/block/pflash_cfi01.c
> > @@ -80,16 +80,39 @@ struct PFlashCFI01 {
> >       uint16_t ident3;
> >       uint8_t cfi_table[0x52];
> >       uint64_t counter;
> > -    unsigned int writeblock_size;
> > +    uint32_t writeblock_size;
> >       MemoryRegion mem;
> >       char *name;
> >       void *storage;
> >       VMChangeStateEntry *vmstate;
> >       bool old_multiple_chip_handling;
> > +
> > +    /* block update buffer */
> > +    unsigned char *blk_bytes;
>
> I'd rather use a 'void *' type here, but then we need to
> use a (uinptr_t) cast in pflash_data_write().
>
> > +    uint32_t blk_offset;
> >   };
> >
> >   static int pflash_post_load(void *opaque, int version_id);
> >
> > +static bool pflash_blk_write_state_needed(void *opaque)
> > +{
> > +    PFlashCFI01 *pfl = opaque;
> > +
> > +    return (pfl->blk_offset != -1);
> > +}
> > +
> > +static const VMStateDescription vmstate_pflash_blk_write = {
> > +    .name = "pflash_cfi01_blk_write",
> > +    .version_id = 1,
> > +    .minimum_version_id = 1,
> > +    .needed = pflash_blk_write_state_needed,
> > +    .fields = (const VMStateField[]) {
> > +        VMSTATE_VBUFFER_UINT32(blk_bytes, PFlashCFI01, 0, NULL, writeblock_size),
>
> I don't get the difference with VMSTATE_VBUFFER_ALLOC_UINT32() which
> sets VMS_ALLOC. In this case pflash_cfi01_realize() does the alloc so
> we don't need VMS_ALLOC?

Yes, that's the idea. A VMS_ALLOC vmstate type means "this
block of memory is dynamically sized at runtime, so when the
migration code is doing inbound migration it needs to
allocate a buffer of the right size first (based on some
state struct field we've already migrated) and then put the
incoming data into it". VMS_VBUFFER means "the size of the buffer
isn't a compile-time constant, so we need to fish it out of
some other state struct field". So:

 VMSTATE_VBUFFER_UINT32: we need to migrate (a pointer to) an array
 of uint32_t; the size of that is in some other struct field,
 but it's a runtime constant and we can assume the memory has
 already been allocated

 VMSTATE_VBUFFER_ALLOC_UINT32: we need to migrate an array
 of uint32_t of variable size dependent on the inbound migration
 data, and so the migration code must allocate it

thanks
-- PMM

Re: [PATCH v2 2/2] hw/pflash: implement update buffer for block writes

[Philippe Mathieu-Daudé]

On 12/1/24 17:54, Peter Maydell wrote:
> On Mon, 8 Jan 2024 at 13:06, Philippe Mathieu-Daudé <philmd@linaro.org> wrote:
>>
>> Hi Gerd,
>>
>> On 8/1/24 13:53, Philippe Mathieu-Daudé wrote:
>>> From: Gerd Hoffmann <kraxel@redhat.com>
>>>
>>> Add an update buffer where all block updates are staged.
>>> Flush or discard updates properly, so we should never see
>>> half-completed block writes in pflash storage.
>>>
>>> Drop a bunch of FIXME comments ;)
>>>
>>> Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
>>> Message-ID: <20240105135855.268064-3-kraxel@redhat.com>
>>> Signed-off-by: Philippe Mathieu-Daudé <philmd@linaro.org>
>>> ---
>>>    hw/block/pflash_cfi01.c | 106 ++++++++++++++++++++++++++++++----------
>>>    1 file changed, 80 insertions(+), 26 deletions(-)


>>> +static const VMStateDescription vmstate_pflash_blk_write = {
>>> +    .name = "pflash_cfi01_blk_write",
>>> +    .version_id = 1,
>>> +    .minimum_version_id = 1,
>>> +    .needed = pflash_blk_write_state_needed,
>>> +    .fields = (const VMStateField[]) {
>>> +        VMSTATE_VBUFFER_UINT32(blk_bytes, PFlashCFI01, 0, NULL, writeblock_size),
>>
>> I don't get the difference with VMSTATE_VBUFFER_ALLOC_UINT32() which
>> sets VMS_ALLOC. In this case pflash_cfi01_realize() does the alloc so
>> we don't need VMS_ALLOC?
> 
> Yes, that's the idea. A VMS_ALLOC vmstate type means "this
> block of memory is dynamically sized at runtime, so when the
> migration code is doing inbound migration it needs to
> allocate a buffer of the right size first (based on some
> state struct field we've already migrated) and then put the
> incoming data into it". VMS_VBUFFER means "the size of the buffer
> isn't a compile-time constant, so we need to fish it out of
> some other state struct field". So:
> 
>   VMSTATE_VBUFFER_UINT32: we need to migrate (a pointer to) an array
>   of uint32_t; the size of that is in some other struct field,
>   but it's a runtime constant and we can assume the memory has
>   already been allocated
> 
>   VMSTATE_VBUFFER_ALLOC_UINT32: we need to migrate an array
>   of uint32_t of variable size dependent on the inbound migration
>   data, and so the migration code must allocate it

Thanks Peter!

Do you mind if we commit your explanation as is? As:

-- >8 --
diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h
index 294d2d8486..5c6f6c5c32 100644
--- a/include/migration/vmstate.h
+++ b/include/migration/vmstate.h
@@ -573,4 +573,6 @@ extern const VMStateInfo vmstate_info_qlist;

-/* a variable length array (i.e. _type *_field) but we know the
- * length
+/**
+ * VMSTATE_STRUCT_VARRAY_POINTER_KNOWN:
+ *
+ * A variable length array (i.e. _type *_field) but we know the length.
   */
@@ -678,2 +680,10 @@ extern const VMStateInfo vmstate_info_qlist;

+/**
+ * VMSTATE_VBUFFER_UINT32:
+ *
+ * We need to migrate (a pointer to) an array of uint32_t; the size of
+ * that is in some other struct field, but it's a runtime constant and
+ * we can assume the memory has already been allocated.
+*/
+
  #define VMSTATE_VBUFFER_UINT32(_field, _state, _version, _test, 
_field_size) { \
@@ -688,2 +698,9 @@ extern const VMStateInfo vmstate_info_qlist;

+/**
+ * VMSTATE_VBUFFER_ALLOC_UINT32:
+ *
+ * We need to migrate an array of uint32_t of variable size dependent
+ * on the inbound migration data, and so the migration code must
+ * allocate it.
+*/
  #define VMSTATE_VBUFFER_ALLOC_UINT32(_field, _state, _version,       \
---

Re: [PATCH v2 2/2] hw/pflash: implement update buffer for block writes

[Peter Maydell]

On Tue, 16 Jan 2024 at 16:08, Philippe Mathieu-Daudé <philmd@linaro.org> wrote:
>
> On 12/1/24 17:54, Peter Maydell wrote:
> > On Mon, 8 Jan 2024 at 13:06, Philippe Mathieu-Daudé <philmd@linaro.org> wrote:
> >>
> >> Hi Gerd,
> >>
> >> On 8/1/24 13:53, Philippe Mathieu-Daudé wrote:
> >>> From: Gerd Hoffmann <kraxel@redhat.com>
> >>>
> >>> Add an update buffer where all block updates are staged.
> >>> Flush or discard updates properly, so we should never see
> >>> half-completed block writes in pflash storage.
> >>>
> >>> Drop a bunch of FIXME comments ;)
> >>>
> >>> Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
> >>> Message-ID: <20240105135855.268064-3-kraxel@redhat.com>
> >>> Signed-off-by: Philippe Mathieu-Daudé <philmd@linaro.org>
> >>> ---
> >>>    hw/block/pflash_cfi01.c | 106 ++++++++++++++++++++++++++++++----------
> >>>    1 file changed, 80 insertions(+), 26 deletions(-)
>
>
> >>> +static const VMStateDescription vmstate_pflash_blk_write = {
> >>> +    .name = "pflash_cfi01_blk_write",
> >>> +    .version_id = 1,
> >>> +    .minimum_version_id = 1,
> >>> +    .needed = pflash_blk_write_state_needed,
> >>> +    .fields = (const VMStateField[]) {
> >>> +        VMSTATE_VBUFFER_UINT32(blk_bytes, PFlashCFI01, 0, NULL, writeblock_size),
> >>
> >> I don't get the difference with VMSTATE_VBUFFER_ALLOC_UINT32() which
> >> sets VMS_ALLOC. In this case pflash_cfi01_realize() does the alloc so
> >> we don't need VMS_ALLOC?
> >
> > Yes, that's the idea. A VMS_ALLOC vmstate type means "this
> > block of memory is dynamically sized at runtime, so when the
> > migration code is doing inbound migration it needs to
> > allocate a buffer of the right size first (based on some
> > state struct field we've already migrated) and then put the
> > incoming data into it". VMS_VBUFFER means "the size of the buffer
> > isn't a compile-time constant, so we need to fish it out of
> > some other state struct field". So:
> >
> >   VMSTATE_VBUFFER_UINT32: we need to migrate (a pointer to) an array
> >   of uint32_t; the size of that is in some other struct field,
> >   but it's a runtime constant and we can assume the memory has
> >   already been allocated
> >
> >   VMSTATE_VBUFFER_ALLOC_UINT32: we need to migrate an array
> >   of uint32_t of variable size dependent on the inbound migration
> >   data, and so the migration code must allocate it
>
> Thanks Peter!
>
> Do you mind if we commit your explanation as is? As:
>
> -- >8 --
> diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h
> index 294d2d8486..5c6f6c5c32 100644
> --- a/include/migration/vmstate.h
> +++ b/include/migration/vmstate.h
> @@ -573,4 +573,6 @@ extern const VMStateInfo vmstate_info_qlist;
>
> -/* a variable length array (i.e. _type *_field) but we know the
> - * length
> +/**
> + * VMSTATE_STRUCT_VARRAY_POINTER_KNOWN:
> + *
> + * A variable length array (i.e. _type *_field) but we know the length.
>    */
> @@ -678,2 +680,10 @@ extern const VMStateInfo vmstate_info_qlist;
>
> +/**
> + * VMSTATE_VBUFFER_UINT32:
> + *
> + * We need to migrate (a pointer to) an array of uint32_t; the size of
> + * that is in some other struct field, but it's a runtime constant and
> + * we can assume the memory has already been allocated.
> +*/
> +
>   #define VMSTATE_VBUFFER_UINT32(_field, _state, _version, _test,
> _field_size) { \
> @@ -688,2 +698,9 @@ extern const VMStateInfo vmstate_info_qlist;
>
> +/**
> + * VMSTATE_VBUFFER_ALLOC_UINT32:
> + *
> + * We need to migrate an array of uint32_t of variable size dependent
> + * on the inbound migration data, and so the migration code must
> + * allocate it.
> +*/
>   #define VMSTATE_VBUFFER_ALLOC_UINT32(_field, _state, _version,       \
> ---

Gerd is probably a better source of comments for these macros;
there are some things I don't entirely understand about all
the inner workings, so my comments above are restricted to
only the bit that matters for this particular distinction,
and aren't really intended as documentation of the macro in
general.

-- PMM

Re: [PATCH v2 2/2] hw/pflash: implement update buffer for block writes

[Peter Xu]

On Tue, Jan 16, 2024 at 05:08:28PM +0100, Philippe Mathieu-Daudé wrote:
> On 12/1/24 17:54, Peter Maydell wrote:
> > On Mon, 8 Jan 2024 at 13:06, Philippe Mathieu-Daudé <philmd@linaro.org> wrote:
> > > 
> > > Hi Gerd,
> > > 
> > > On 8/1/24 13:53, Philippe Mathieu-Daudé wrote:
> > > > From: Gerd Hoffmann <kraxel@redhat.com>
> > > > 
> > > > Add an update buffer where all block updates are staged.
> > > > Flush or discard updates properly, so we should never see
> > > > half-completed block writes in pflash storage.
> > > > 
> > > > Drop a bunch of FIXME comments ;)
> > > > 
> > > > Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
> > > > Message-ID: <20240105135855.268064-3-kraxel@redhat.com>
> > > > Signed-off-by: Philippe Mathieu-Daudé <philmd@linaro.org>
> > > > ---
> > > >    hw/block/pflash_cfi01.c | 106 ++++++++++++++++++++++++++++++----------
> > > >    1 file changed, 80 insertions(+), 26 deletions(-)
> 
> 
> > > > +static const VMStateDescription vmstate_pflash_blk_write = {
> > > > +    .name = "pflash_cfi01_blk_write",
> > > > +    .version_id = 1,
> > > > +    .minimum_version_id = 1,
> > > > +    .needed = pflash_blk_write_state_needed,
> > > > +    .fields = (const VMStateField[]) {
> > > > +        VMSTATE_VBUFFER_UINT32(blk_bytes, PFlashCFI01, 0, NULL, writeblock_size),
> > > 
> > > I don't get the difference with VMSTATE_VBUFFER_ALLOC_UINT32() which
> > > sets VMS_ALLOC. In this case pflash_cfi01_realize() does the alloc so
> > > we don't need VMS_ALLOC?
> > 
> > Yes, that's the idea. A VMS_ALLOC vmstate type means "this
> > block of memory is dynamically sized at runtime, so when the
> > migration code is doing inbound migration it needs to
> > allocate a buffer of the right size first (based on some
> > state struct field we've already migrated) and then put the
> > incoming data into it". VMS_VBUFFER means "the size of the buffer
> > isn't a compile-time constant, so we need to fish it out of
> > some other state struct field". So:
> > 
> >   VMSTATE_VBUFFER_UINT32: we need to migrate (a pointer to) an array
> >   of uint32_t; the size of that is in some other struct field,
> >   but it's a runtime constant and we can assume the memory has
> >   already been allocated
> > 
> >   VMSTATE_VBUFFER_ALLOC_UINT32: we need to migrate an array
> >   of uint32_t of variable size dependent on the inbound migration
> >   data, and so the migration code must allocate it
> 
> Thanks Peter!
> 
> Do you mind if we commit your explanation as is? As:

Any attempt to provide more documents there for the macros would be nice if
anyone would like to post a patch.  Though some comments below for this
specific one.

> 
> -- >8 --
> diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h
> index 294d2d8486..5c6f6c5c32 100644
> --- a/include/migration/vmstate.h
> +++ b/include/migration/vmstate.h
> @@ -573,4 +573,6 @@ extern const VMStateInfo vmstate_info_qlist;
> 
> -/* a variable length array (i.e. _type *_field) but we know the
> - * length
> +/**
> + * VMSTATE_STRUCT_VARRAY_POINTER_KNOWN:
> + *
> + * A variable length array (i.e. _type *_field) but we know the length.
>   */
> @@ -678,2 +680,10 @@ extern const VMStateInfo vmstate_info_qlist;
> 
> +/**
> + * VMSTATE_VBUFFER_UINT32:
> + *
> + * We need to migrate (a pointer to) an array of uint32_t; the size of

IIUC it's not an array of uint32_t, but a buffer with dynamic size.  the
"uint32_t" is trying to describe the type of the size field, afaict.  Same
issue for below one.

For example, comparing VMSTATE_VBUFFER_UINT32 vs VMSTATE_VBUFFER, we should
see the only difference is:

    .size_offset  = vmstate_offset_value(_state, _field_size, int32_t),\

vs:

    .size_offset  = vmstate_offset_value(_state, _field_size, uint32_t),\

I think it checks the size field to match that type.

Interestingly, when I look at the code to use the size, it looks like a bug
to me, where we always parse the size field to be int32_t..

static int vmstate_size(void *opaque, const VMStateField *field)
{
    int size = field->size;

    if (field->flags & VMS_VBUFFER) {
        size = *(int32_t *)(opaque + field->size_offset);     <----------- see here..
        if (field->flags & VMS_MULTIPLY) {
            size *= field->size;
        }
    }

    return size;
}

I think it'll start to crash things when bit32 start to be used.  And I had
a feeling that this is overlooked in the commit a19cbfb346 ("spice: add qxl
device") where VMSTATE_VBUFFER_UINT32 is introduced.

I don't have a good way to trivially fix it because we don't remember that
type of size in vmsd.  However a simple solution could be that we convert
all existing VMSTATE_VBUFFER() (potentially, VMSTATE_PARTIAL_VBUFFER users;
there seems to have only 1 caller..) to always use uint32_t.  I don't think
it's wise to try using a signed for size anyway, and it should be
compatible change as we doubled the size.

I'll hold a bit to see whether there's some comment, then I can try to post
a patch.

> + * that is in some other struct field, but it's a runtime constant and
> + * we can assume the memory has already been allocated.
> +*/
> +
>  #define VMSTATE_VBUFFER_UINT32(_field, _state, _version, _test,
> _field_size) { \
> @@ -688,2 +698,9 @@ extern const VMStateInfo vmstate_info_qlist;
> 
> +/**
> + * VMSTATE_VBUFFER_ALLOC_UINT32:
> + *
> + * We need to migrate an array of uint32_t of variable size dependent
> + * on the inbound migration data, and so the migration code must
> + * allocate it.
> +*/
>  #define VMSTATE_VBUFFER_ALLOC_UINT32(_field, _state, _version,       \
> ---
> 

-- 
Peter Xu

Re: [PATCH v2 2/2] hw/pflash: implement update buffer for block writes

[Richard Henderson]

On 1/8/24 23:53, Philippe Mathieu-Daudé wrote:
> @@ -818,6 +867,9 @@ static void pflash_cfi01_realize(DeviceState *dev, Error **errp)
>       pfl->cmd = 0x00;
>       pfl->status = 0x80; /* WSM ready */
>       pflash_cfi01_fill_cfi_table(pfl);
> +
> +    pfl->blk_bytes = g_malloc(pfl->writeblock_size);

Do you need an unrealize to free?


r~