[Qemu-devel] Re: [PATCH 21/26] Implement TCE translation for sPAPR VIO

[Alexander Graf <agraf@suse.de>]

On 03/16/2011 05:56 AM, David Gibson wrote:
> From: Ben Herrenschmidt<benh@kernel.crashing.org>
>
> This patch implements the necessary infrastructure and hypercalls for
> sPAPR's TCE (Translation Control Entry) IOMMU mechanism.  This is necessary
> for all virtual IO devices which do DMA (i.e. nearly all of them).
>
> Signed-off-by: Ben Herrenschmidt<benh@kernel.crashing.org>
> Signed-off-by: David Gibson<dwg@au1.ibm.com>
> ---
>   hw/spapr.c     |    3 +-
>   hw/spapr_vio.c |  232 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
>   hw/spapr_vio.h |   32 ++++++++
>   3 files changed, 266 insertions(+), 1 deletions(-)
>
> diff --git a/hw/spapr.c b/hw/spapr.c
> index e7f8864..a362889 100644
> --- a/hw/spapr.c
> +++ b/hw/spapr.c
> @@ -62,7 +62,8 @@ static void *spapr_create_fdt(int *fdt_size, ram_addr_t ramsize,
>       uint32_t start_prop = cpu_to_be32(initrd_base);
>       uint32_t end_prop = cpu_to_be32(initrd_base + initrd_size);
>       uint32_t pft_size_prop[] = {0, cpu_to_be32(hash_shift)};
> -    char hypertas_prop[] = "hcall-pft\0hcall-term\0hcall-dabr\0hcall-interrupt";
> +    char hypertas_prop[] = "hcall-pft\0hcall-term\0hcall-dabr\0hcall-interrupt"
> +        "\0hcall-tce";
>       uint32_t interrupt_server_ranges_prop[] = {0, cpu_to_be32(smp_cpus)};
>       int i;
>       char *modelname;
> diff --git a/hw/spapr_vio.c b/hw/spapr_vio.c
> index 45edd94..37cf51e 100644
> --- a/hw/spapr_vio.c
> +++ b/hw/spapr_vio.c
> @@ -37,6 +37,7 @@
>   #endif /* CONFIG_FDT */
>
>   /* #define DEBUG_SPAPR */
> +/* #define DEBUG_TCE */
>
>   #ifdef DEBUG_SPAPR
>   #define dprintf(fmt, ...) \
> @@ -114,6 +115,28 @@ static int vio_make_devnode(VIOsPAPRDevice *dev,
>               return ret;
>       }
>
> +    if (dev->rtce_window_size) {
> +        uint32_t dma_prop[] = {cpu_to_be32(dev->reg),
> +                               0, 0,
> +                               0, cpu_to_be32(dev->rtce_window_size)};
> +
> +        ret = fdt_setprop_cell(fdt, node_off, "ibm,#dma-address-cells", 2);
> +        if (ret<  0) {
> +            return ret;
> +        }
> +
> +        ret = fdt_setprop_cell(fdt, node_off, "ibm,#dma-size-cells", 2);
> +        if (ret<  0) {
> +            return ret;
> +        }
> +
> +        ret = fdt_setprop(fdt, node_off, "ibm,my-dma-window", dma_prop,
> +                          sizeof(dma_prop));
> +        if (ret<  0) {
> +            return ret;
> +        }
> +    }
> +
>       if (info->devnode) {
>           ret = (info->devnode)(dev, fdt, node_off);
>           if (ret<  0) {
> @@ -125,6 +148,210 @@ static int vio_make_devnode(VIOsPAPRDevice *dev,
>   }
>   #endif /* CONFIG_FDT */
>
> +/*
> + * RTCE handling
> + */
> +
> +static void rtce_init(VIOsPAPRDevice *dev)
> +{
> +    size_t size = (dev->rtce_window_size>>  SPAPR_VIO_TCE_PAGE_SHIFT)
> +        * sizeof(VIOsPAPR_RTCE);
> +
> +    if (size) {
> +        dev->rtce_table = qemu_mallocz(size);
> +    }
> +}
> +
> +static target_ulong h_put_tce(CPUState *env, sPAPREnvironment *spapr,
> +                              target_ulong opcode, target_ulong *args)
> +{
> +    target_ulong liobn = args[0];
> +    target_ulong ioba = args[1];
> +    target_ulong tce = args[2];
> +    VIOsPAPRDevice *dev = spapr_vio_find_by_reg(spapr->vio_bus, liobn);
> +    VIOsPAPR_RTCE *rtce;
> +
> +    if (!dev) {
> +        fprintf(stderr, "spapr_vio_put_tce on non-existent LIOBN "
> +                TARGET_FMT_lx "\n",
> +                liobn);
> +        return H_PARAMETER;
> +    }
> +
> +    ioba&= ~(SPAPR_VIO_TCE_PAGE_SIZE - 1);
> +
> +#ifdef DEBUG_TCE
> +    fprintf(stderr, "spapr_vio_put_tce on %s  ioba 0x" TARGET_FMT_lx
> +            "  TCE 0x" TARGET_FMT_lx "\n", dev->qdev.id, ioba, tce);
> +#endif
> +
> +    if (ioba>= dev->rtce_window_size) {
> +        fprintf(stderr, "spapr_vio_put_tce on out-of-boards IOBA 0x" TARGET_FMT_lx "\n",
> +                ioba);
> +        return H_PARAMETER;
> +    }
> +
> +    rtce = dev->rtce_table + (ioba>>  SPAPR_VIO_TCE_PAGE_SHIFT);
> +    rtce->tce = tce;
> +
> +    return H_SUCCESS;
> +}
> +
> +int spapr_vio_check_tces(VIOsPAPRDevice *dev, target_ulong ioba,
> +                         target_ulong len, enum VIOsPAPR_TCEAccess access)
> +{
> +    int start, end, i;
> +
> +    start = ioba>>  SPAPR_VIO_TCE_PAGE_SHIFT;
> +    end = (ioba + len - 1)>>  SPAPR_VIO_TCE_PAGE_SHIFT;
> +
> +    for (i = start; i<= end; i++) {
> +        if ((dev->rtce_table[i].tce&  access) != access) {
> +            fprintf(stderr, "FAIL on %d\n", i);
> +            return -1;
> +        }
> +    }
> +
> +    return 0;
> +}
> +
> +/* XX Might want to special case KVM for speed ? */

XXX

> +int spapr_tce_dma_write(VIOsPAPRDevice *dev, uint64_t taddr, const void *buf,
> +                        uint32_t size)
> +{
> +#ifdef DEBUG_TCE
> +    fprintf(stderr, "spapr_tce_dma_write taddr=0x%llx size=0x%x\n",
> +            (unsigned long long)taddr, size);
> +#endif
> +
> +    while(size) {
> +        uint64_t tce;
> +        uint32_t lsize;
> +        uint64_t txaddr;
> +
> +        /* Check if we are in bound */
> +        if (taddr>= dev->rtce_window_size) {
> +            fprintf(stderr, "spapr_tce_dma_write out of bounds\n");
> +            return -H_DEST_PARM;
> +        }
> +        tce = dev->rtce_table[taddr>>  SPAPR_VIO_TCE_PAGE_SHIFT].tce;
> +
> +        /* How much til end of page ? */
> +        lsize = MIN(size, ((~taddr)&  SPAPR_VIO_TCE_PAGE_MASK) + 1);
> +
> +        /* Check TCE */
> +        if (!(tce&  2))

Braces

> +            return -H_DEST_PARM;
> +
> +        /* Translate */
> +        txaddr = (tce&  ~SPAPR_VIO_TCE_PAGE_MASK) | (taddr&  SPAPR_VIO_TCE_PAGE_MASK);
> +
> +#ifdef DEBUG_TCE
> +        fprintf(stderr, " ->  write to txaddr=0x%llx, size=0x%x\n",
> +                (unsigned long long)txaddr, lsize);
> +#endif
> +
> +        /* Do it */
> +        cpu_physical_memory_write(txaddr, buf, lsize);
> +        buf += lsize;
> +        taddr += lsize;
> +        size -= lsize;
> +    }
> +    return 0;
> +}
> +
> +/* XX Might want to special case KVM for speed ? */

XXX

> +int spapr_tce_dma_zero(VIOsPAPRDevice *dev, uint64_t taddr, uint32_t size)
> +{
> +    uint8_t *zeroes;
> +
> +#ifdef DEBUG_TCE
> +    fprintf(stderr, "spapr_tce_dma_zero taddr=0x%llx size=0x%x\n",
> +            (unsigned long long)taddr, size);
> +#endif
> +
> +    /* FIXME: do this better... */
> +    zeroes = alloca(size);
> +    memset(zeroes, 0, size);

You sure that zeroes is still alive during the call? If I were a 
compiler, I'd probably optimize the return away so that it'd end up 
being a simple branch to spapr_tce_dma_write - coincidentally 
invalidating the stack that zeroes is on.


Alex