Re: [PATCH 12/12] hw/char/pl011: Implement TX FIFO

["Alex Bennée" <alex.bennee@linaro.org>]

Philippe Mathieu-Daudé <philmd@linaro.org> writes:

> If the UART back-end chardev doesn't drain data as fast as stdout
> does or blocks, buffer in the TX FIFO to try again later.
>
> This avoids having the IO-thread busy waiting on chardev back-ends,
> reported recently when testing the Trusted Reference Stack and
> using the socket backend:
> https://linaro.atlassian.net/browse/TRS-149?focusedCommentId=149574
>
> Implement registering a front-end 'watch' callback on back-end
> events, so we can resume transmitting when the back-end is writable
> again, not blocking the main loop.
>
> Similarly to the RX FIFO path, FIFO level selection is not
> implemented (interrupt is triggered when a single byte is available
> in the FIFO).
>
> Due to the addition of the TX FIFO in the instance state, increase
> the migration stream version.
>
> Reported-by: Mikko Rapeli <mikko.rapeli@linaro.org>
> Suggested-by: Alex Bennée <alex.bennee@linaro.org>
> Signed-off-by: Philippe Mathieu-Daudé <philmd@linaro.org>
> ---
> RFC because I'm pretty sure I got the migration code wrong.
>
> After writing this I noticed the hw/char/cmsdk-apb-uart.c model
> is much more complete. Instead of copy/pasting its code, I'd rather
> try to extract some generic/bstract "FIFO based chardev" QOM class;
> but this is beyond the scope of this series.
> ---
<snip>
> diff --git a/hw/char/pl011.c b/hw/char/pl011.c
> index 03c006199e..a957138405 100644
> --- a/hw/char/pl011.c
> +++ b/hw/char/pl011.c
> @@ -57,6 +57,9 @@ DeviceState *pl011_create(hwaddr addr, qemu_irq irq, Chardev *chr)
>  /* Data Register, UARTDR */
>  #define DR_BE   (1 << 10)
>  
> +/* Receive Status Register/Error Clear Register, UARTRSR/UARTECR */
> +#define RSR_OE  (1 << 3)
> +
>  /* Interrupt status bits in UARTRIS, UARTMIS, UARTIMSC */
>  #define INT_OE (1 << 10)
>  #define INT_BE (1 << 9)
> @@ -152,19 +155,94 @@ static inline void pl011_reset_tx_fifo(PL011State *s)
>      /* Reset FIFO flags */
>      s->flags &= ~PL011_FLAG_TXFF;
>      s->flags |= PL011_FLAG_TXFE;
> +
> +    fifo8_reset(&s->xmit_fifo);
> +}
> +
> +static gboolean pl011_drain_tx(PL011State *s)
> +{
> +    trace_pl011_fifo_tx_drain(fifo8_num_used(&s->xmit_fifo));
> +    pl011_reset_tx_fifo(s);
> +    s->rsr &= ~RSR_OE;
> +    return FALSE;

WHY ARE YOU SHOUTING?

> +}
> +

worth a comment the return signals something to the chardev - I guess
FEWatchFunc could do with a comment?

> +static gboolean pl011_xmit(void *do_not_use, GIOCondition cond, void *opaque)
> +{
> +    PL011State *s = opaque;
> +    int ret;
> +    const uint8_t *buf;
> +    uint32_t buflen;
> +    uint32_t count;
> +
> +    if (!qemu_chr_fe_backend_connected(&s->chr)) {
> +        /* Instant drain the fifo when there's no back-end */
> +        return pl011_drain_tx(s);
> +    }
> +
> +    count = fifo8_num_used(&s->xmit_fifo);
> +    if (!count) {
> +        return FALSE;
> +    }
> +    if  (!(s->cr & CR_UARTEN)) {
> +        /* Allow completing the current FIFO character before stopping. */
> +        count = 1;
> +    }

maybe:

  bool tx_enabled = s->cr & CR_UARTEN;
  ...
  count = tx_enabled ? fifo8_num_used(&s->xmit_fifo) : 1; /* current only */

  if (count) {

> +
> +    /* Transmit as much data as we can */
> +    buf = fifo8_peek_buf(&s->xmit_fifo, count, &buflen);
> +    ret = qemu_chr_fe_write(&s->chr, buf, buflen);
> +    if (ret >= 0) {
> +        /* Pop the data we could transmit */
> +        trace_pl011_fifo_tx_xmit(ret);
> +        fifo8_pop_buf(&s->xmit_fifo, ret, NULL);
> +        s->int_level |= INT_TX;
> +    }
> +
> +    if ((s->cr & CR_UARTEN) && !fifo8_is_empty(&s->xmit_fifo)) {

 tx_enabled && ...

> +        /* Reschedule another transmission if we couldn't transmit all */
> +        guint r = qemu_chr_fe_add_watch(&s->chr, G_IO_OUT | G_IO_HUP,
> +                                        pl011_xmit, s);
> +        if (!r) {
> +            return pl011_drain_tx(s);
> +        }
> +    }
> +
> +    pl011_update(s);

}

> +
> +    return FALSE;
>  }
>  
>  static void pl011_write_tx(PL011State *s, const uint8_t *buf, int length)
>  {
>      if (!(s->cr & (CR_UARTEN | CR_TXE))) {
> +        if (!fifo8_is_empty(&s->xmit_fifo)) {
> +            /*
> +             * If the UART is disabled in the middle of transmission
> +             * or reception, it completes the current character before
> +             * stopping.
> +             */
> +            pl011_xmit(NULL, G_IO_OUT, s);
> +        }
>          return;
>      }
>  
> -    /* XXX this blocks entire thread. Rewrite to use
> -     * qemu_chr_fe_write and background I/O callbacks */
> -    qemu_chr_fe_write_all(&s->chr, buf, 1);
> -    s->int_level |= INT_TX;
> -    pl011_update(s);
> +    if (length > fifo8_num_free(&s->xmit_fifo)) {
> +        /*
> +         * The FIFO contents remain valid because no more data is
> +         * written when the FIFO is full, only the contents of the
> +         * shift register are overwritten. The CPU must now read
> +         * the data, to empty the FIFO.
> +         */
> +        trace_pl011_fifo_tx_overrun();
> +        s->rsr |= RSR_OE;
> +        return;
> +    }
> +
> +    trace_pl011_fifo_tx_put(length);
> +    fifo8_push_all(&s->xmit_fifo, buf, length);
> +
> +    pl011_xmit(NULL, G_IO_OUT, s);
>  }
>  
>  static uint64_t pl011_read(void *opaque, hwaddr offset,
> @@ -444,12 +522,17 @@ static int pl011_post_load(void *opaque, int version_id)
>          s->read_pos = 0;
>      }
>  
> +    if (version_id >= 3 && !fifo8_is_empty(&s->xmit_fifo)) {
> +        /* Reschedule another transmission */
> +        qemu_chr_fe_add_watch(&s->chr, G_IO_OUT | G_IO_HUP, pl011_xmit, s);
> +    }
> +
>      return 0;
>  }
>  
>  static const VMStateDescription vmstate_pl011 = {
>      .name = "pl011",
> -    .version_id = 2,
> +    .version_id = 3,
>      .minimum_version_id = 2,
>      .post_load = pl011_post_load,
>      .fields = (VMStateField[]) {
> @@ -462,6 +545,7 @@ static const VMStateDescription vmstate_pl011 = {
>          VMSTATE_UINT32(int_enabled, PL011State),
>          VMSTATE_UINT32(int_level, PL011State),
>          VMSTATE_UINT32_ARRAY(read_fifo, PL011State, PL011_FIFO_DEPTH),
> +        VMSTATE_FIFO8(xmit_fifo, PL011State),

I think you want something like:

   VMSTATE_FIFO8_TEST(xmit_fifo, PL011State, pl011_is_version_3_or_better),

>          VMSTATE_UINT32(ilpr, PL011State),
>          VMSTATE_UINT32(ibrd, PL011State),
>          VMSTATE_UINT32(fbrd, PL011State),
> @@ -505,10 +589,18 @@ static void pl011_realize(DeviceState *dev, Error **errp)
>  {
>      PL011State *s = PL011(dev);
>  
> +    fifo8_create(&s->xmit_fifo, PL011_FIFO_DEPTH);
>      qemu_chr_fe_set_handlers(&s->chr, pl011_can_receive, pl011_receive,
>                               pl011_event, NULL, s, NULL, true);
>  }
>  
> +static void pl011_unrealize(DeviceState *dev)
> +{
> +    PL011State *s = PL011(dev);
> +
> +    fifo8_destroy(&s->xmit_fifo);
> +}
> +
>  static void pl011_reset(DeviceState *dev)
>  {
>      PL011State *s = PL011(dev);
> @@ -534,6 +626,7 @@ static void pl011_class_init(ObjectClass *oc, void *data)
>      DeviceClass *dc = DEVICE_CLASS(oc);
>  
>      dc->realize = pl011_realize;
> +    dc->unrealize = pl011_unrealize;
>      dc->reset = pl011_reset;
>      dc->vmsd = &vmstate_pl011;
>      device_class_set_props(dc, pl011_properties);
> diff --git a/hw/char/trace-events b/hw/char/trace-events
> index 9fd40e3aae..4c25564066 100644
> --- a/hw/char/trace-events
> +++ b/hw/char/trace-events
> @@ -60,6 +60,10 @@ pl011_write(uint32_t addr, uint32_t value, const char *regname) "addr 0x%03x val
>  pl011_can_receive(uint32_t lcr, int read_count, int r) "LCR 0x%08x read_count %d returning %d"
>  pl011_fifo_rx_put(uint32_t c, int read_count) "new char 0x%02x read_count now %d"
>  pl011_fifo_rx_full(void) "RX FIFO now full, RXFF set"
> +pl011_fifo_tx_put(int count) "TX FIFO push %d"
> +pl011_fifo_tx_xmit(int count) "TX FIFO pop %d"
> +pl011_fifo_tx_overrun(void) "TX FIFO overrun"
> +pl011_fifo_tx_drain(unsigned drained) "TX FIFO draining %u"
>  pl011_baudrate_change(unsigned int baudrate, uint64_t clock, uint32_t ibrd, uint32_t fbrd) "new baudrate %u (clk: %" PRIu64 "hz, ibrd: %" PRIu32 ", fbrd: %" PRIu32 ")"
>  
>  # cmsdk-apb-uart.c


-- 
Alex Bennée
Virtualisation Tech Lead @ Linaro