Re: [Qemu-devel] [PATCH 22/26] Implement sPAPR Virtual LAN (ibmveth)

[Anthony Liguori <anthony@codemonkey.ws>]

On 03/15/2011 11:56 PM, David Gibson wrote:
> This patch implements the PAPR specified Inter Virtual Machine Logical
> LAN; that is the virtual hardware used by the Linux ibmveth driver.
>
> Signed-off-by: Paul Mackerras<paulus@samba.org>
> Signed-off-by: David Gibson<dwg@au1.ibm.com>
> ---
>   Makefile.target |    2 +-
>   hw/spapr.c      |   21 +++-
>   hw/spapr_llan.c |  476 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
>   hw/spapr_vio.h  |    9 +-
>   4 files changed, 503 insertions(+), 5 deletions(-)
>   create mode 100644 hw/spapr_llan.c
>
> diff --git a/Makefile.target b/Makefile.target
> index 2b0588e..ef86d43 100644
> --- a/Makefile.target
> +++ b/Makefile.target
> @@ -233,7 +233,7 @@ obj-ppc-y += ppc_oldworld.o
>   obj-ppc-y += ppc_newworld.o
>   # IBM pSeries (sPAPR)
>   obj-ppc-y += spapr.o spapr_hcall.o spapr_rtas.o spapr_vio.o
> -obj-ppc-y += xics.o spapr_vty.o
> +obj-ppc-y += xics.o spapr_vty.o spapr_llan.o
>   # PowerPC 4xx boards
>   obj-ppc-y += ppc4xx_devs.o ppc4xx_pci.o ppc405_uc.o ppc405_boards.o
>   obj-ppc-y += ppc440.o ppc440_bamboo.o
> diff --git a/hw/spapr.c b/hw/spapr.c
> index a362889..44cf3cc 100644
> --- a/hw/spapr.c
> +++ b/hw/spapr.c
> @@ -27,6 +27,7 @@
>   #include "sysemu.h"
>   #include "hw.h"
>   #include "elf.h"
> +#include "net.h"
>
>   #include "hw/boards.h"
>   #include "hw/ppc.h"
> @@ -315,7 +316,7 @@ static void ppc_spapr_init(ram_addr_t ram_size,
>       qemu_free(filename);
>
>       /* Set up Interrupt Controller */
> -    spapr->icp = xics_system_init(smp_cpus,&env, MAX_SERIAL_PORTS);
> +    spapr->icp = xics_system_init(smp_cpus, envs, MAX_SERIAL_PORTS + nb_nics);
>
>       /* Set up VIO bus */
>       spapr->vio_bus = spapr_vio_bus_init();
> @@ -327,6 +328,24 @@ static void ppc_spapr_init(ram_addr_t ram_size,
>           }
>       }
>
> +    for (i = 0; i<  nb_nics; i++, irq++) {
> +        NICInfo *nd =&nd_table[i];
> +
> +        if (!nd->model) {
> +            nd->model = qemu_strdup("ibmveth");
> +        }
> +
> +        if (strcmp(nd->model, "ibmveth") == 0) {
> +            spapr_vlan_create(spapr->vio_bus, 0x1000 + i, nd,
> +                              xics_find_qirq(spapr->icp, irq), irq);
> +        } else {
> +            fprintf(stderr, "pSeries (sPAPR) platform does not support "
> +                    "NIC model '%s' (only ibmveth is supported)\n",
> +                    nd->model);
> +            exit(1);
> +        }
> +    }
> +
>       if (kernel_filename) {
>           uint64_t lowaddr = 0;
>
> diff --git a/hw/spapr_llan.c b/hw/spapr_llan.c
> new file mode 100644
> index 0000000..da0562d
> --- /dev/null
> +++ b/hw/spapr_llan.c
> @@ -0,0 +1,476 @@
> +#include "hw.h"
> +#include "net.h"
> +#include "hw/qdev.h"
> +#include "hw/spapr.h"
> +#include "hw/spapr_vio.h"
> +
> +#include<libfdt.h>
> +
> +#define ETH_ALEN        6
> +
> +//#define DEBUG
> +
> +#ifdef DEBUG
> +#define dprintf(fmt...) do { fprintf(stderr, fmt); } while(0)
> +#else
> +#define dprintf(fmt...)
> +#endif
> +
> +/*
> + * Virtual LAN device
> + */
> +
> +typedef uint64_t vlan_bd_t;
> +
> +#define VLAN_BD_VALID        0x8000000000000000ULL
> +#define VLAN_BD_TOGGLE       0x4000000000000000ULL
> +#define VLAN_BD_NO_CSUM      0x0200000000000000ULL
> +#define VLAN_BD_CSUM_GOOD    0x0100000000000000ULL
> +#define VLAN_BD_LEN_MASK     0x00ffffff00000000ULL
> +#define VLAN_BD_LEN(bd)      (((bd)&  VLAN_BD_LEN_MASK)>>  32)
> +#define VLAN_BD_ADDR_MASK    0x00000000ffffffffULL
> +#define VLAN_BD_ADDR(bd)     ((bd)&  VLAN_BD_ADDR_MASK)
> +
> +#define VLAN_VALID_BD(addr, len) (VLAN_BD_VALID | \
> +                                  (((len)<<  32)&  VLAN_BD_LEN_MASK) |  \
> +                                  (addr&  VLAN_BD_ADDR_MASK))
> +
> +#define VLAN_RXQC_TOGGLE     0x80
> +#define VLAN_RXQC_VALID      0x40
> +#define VLAN_RXQC_NO_CSUM    0x02
> +#define VLAN_RXQC_CSUM_GOOD  0x01
> +
> +#define VLAN_RQ_ALIGNMENT    16
> +#define VLAN_RXQ_BD_OFF      0
> +#define VLAN_FILTER_BD_OFF   8
> +#define VLAN_RX_BDS_OFF      16
> +#define VLAN_MAX_BUFS        ((SPAPR_VIO_TCE_PAGE_SIZE - VLAN_RX_BDS_OFF) / 8)
> +
> +typedef struct VIOsPAPRVLANDevice {
> +    VIOsPAPRDevice sdev;
> +    NICConf nicconf;
> +    NICState *nic;
> +    int isopen;
> +    target_ulong buf_list;
> +    int add_buf_ptr, use_buf_ptr, rx_bufs;
> +    target_ulong rxq_ptr;
> +} VIOsPAPRVLANDevice;
> +
> +static int spapr_vlan_can_receive(VLANClientState *nc)
> +{
> +    VIOsPAPRVLANDevice *dev = DO_UPCAST(NICState, nc, nc)->opaque;
> +
> +    return (dev->isopen&&  dev->rx_bufs>  0);
> +}
> +
> +static ssize_t spapr_vlan_receive(VLANClientState *nc, const uint8_t *buf,
> +                                  size_t size)
> +{
> +    VIOsPAPRDevice *sdev = DO_UPCAST(NICState, nc, nc)->opaque;
> +    VIOsPAPRVLANDevice *dev = (VIOsPAPRVLANDevice *)sdev;
> +    vlan_bd_t rxq_bd = ldq_tce(sdev, dev->buf_list + VLAN_RXQ_BD_OFF);
> +    vlan_bd_t bd;
> +    int buf_ptr = dev->use_buf_ptr;
> +    uint64_t handle;
> +    uint8_t control;
> +
> +    dprintf("spapr_vlan_receive() [%s] rx_bufs=%d\n", sdev->qdev.id,
> +            dev->rx_bufs);
> +
> +    if (!dev->isopen) {
> +        return -1;
> +    }
> +
> +    if (!dev->rx_bufs) {
> +        return -1;
> +    }
> +
> +    do {
> +        buf_ptr += 8;
> +        if (buf_ptr>= SPAPR_VIO_TCE_PAGE_SIZE) {
> +            buf_ptr = VLAN_RX_BDS_OFF;
> +        }
> +
> +        bd = ldq_tce(sdev, dev->buf_list + buf_ptr);
> +        dprintf("use_buf_ptr=%d bd=0x%016llx\n",
> +                buf_ptr, (unsigned long long)bd);
> +    } while ((!(bd&  VLAN_BD_VALID) || (VLAN_BD_LEN(bd)<  (size + 8)))
> +&&  (buf_ptr != dev->use_buf_ptr));
> +
> +    if (!(bd&  VLAN_BD_VALID) || (VLAN_BD_LEN(bd)<  (size + 8))) {
> +        /* Failed to find a suitable buffer */
> +        return -1;
> +    }
> +
> +    /* Remove the buffer from the pool */
> +    dev->rx_bufs--;
> +    dev->use_buf_ptr = buf_ptr;
> +    stq_tce(sdev, dev->buf_list + dev->use_buf_ptr, 0);
> +
> +    dprintf("Found buffer: ptr=%d num=%d\n", dev->use_buf_ptr, dev->rx_bufs);
> +
> +    /* Transfer the packet data */
> +    if (spapr_tce_dma_write(sdev, VLAN_BD_ADDR(bd) + 8, buf, size)<  0) {
> +        return -1;
> +    }
> +
> +    dprintf("spapr_vlan_receive: DMA write completed\n");
> +
> +    /* Update the receive queue */
> +    control = VLAN_RXQC_TOGGLE | VLAN_RXQC_VALID;
> +    if (rxq_bd&  VLAN_BD_TOGGLE) {
> +        control ^= VLAN_RXQC_TOGGLE;
> +    }
> +
> +    handle = ldq_tce(sdev, VLAN_BD_ADDR(bd));
> +    stq_tce(sdev, VLAN_BD_ADDR(rxq_bd) + dev->rxq_ptr + 8, handle);
> +    stw_tce(sdev, VLAN_BD_ADDR(rxq_bd) + dev->rxq_ptr + 4, size);
> +    sth_tce(sdev, VLAN_BD_ADDR(rxq_bd) + dev->rxq_ptr + 2, 8);
> +    stb_tce(sdev, VLAN_BD_ADDR(rxq_bd) + dev->rxq_ptr, control);
> +
> +    dprintf("wrote rxq entry (ptr=0x%llx): 0x%016llx 0x%016llx\n",
> +            (unsigned long long)dev->rxq_ptr,
> +            (unsigned long long)ldq_tce(sdev, VLAN_BD_ADDR(rxq_bd) +
> +                                        dev->rxq_ptr),
> +            (unsigned long long)ldq_tce(sdev, VLAN_BD_ADDR(rxq_bd) +
> +                                        dev->rxq_ptr + 8));
> +
> +    dev->rxq_ptr += 16;
> +    if (dev->rxq_ptr>= VLAN_BD_LEN(rxq_bd)) {
> +        dev->rxq_ptr = 0;
> +        stq_tce(sdev, dev->buf_list + VLAN_RXQ_BD_OFF, rxq_bd ^ VLAN_BD_TOGGLE);
> +    }
> +
> +    if (sdev->signal_state&  1) {
> +        qemu_irq_pulse(sdev->qirq);
> +    }
> +
> +    return size;
> +}
> +
> +static NetClientInfo net_spapr_vlan_info = {
> +    .type = NET_CLIENT_TYPE_NIC,
> +    .size = sizeof(NICState),
> +    .can_receive = spapr_vlan_can_receive,
> +    .receive = spapr_vlan_receive,
> +};
> +
> +static int spapr_vlan_init(VIOsPAPRDevice *sdev)
> +{
> +    VIOsPAPRVLANDevice *dev = (VIOsPAPRVLANDevice *)sdev;
> +    VIOsPAPRBus *bus;
> +
> +    bus = DO_UPCAST(VIOsPAPRBus, bus, sdev->qdev.parent_bus);
> +
> +    qemu_macaddr_default_if_unset(&dev->nicconf.macaddr);
> +
> +    dev->nic = qemu_new_nic(&net_spapr_vlan_info,&dev->nicconf,
> +                            sdev->qdev.info->name, sdev->qdev.id, dev);
> +    qemu_format_nic_info_str(&dev->nic->nc, dev->nicconf.macaddr.a);
> +
> +    return 0;
> +}
> +
> +void spapr_vlan_create(VIOsPAPRBus *bus, uint32_t reg, NICInfo *nd,
> +                       qemu_irq qirq, uint32_t vio_irq_num)
> +{
> +    DeviceState *dev;
> +    VIOsPAPRDevice *sdev;
> +
> +    dev = qdev_create(&bus->bus, "spapr-vlan");
> +    qdev_prop_set_uint32(dev, "reg", reg);
> +
> +    qdev_set_nic_properties(dev, nd);
> +
> +    qdev_init_nofail(dev);
> +    sdev = (VIOsPAPRDevice *)dev;
> +    sdev->qirq = qirq;
> +    sdev->vio_irq_num = vio_irq_num;
> +}
> +
> +static int spapr_vlan_devnode(VIOsPAPRDevice *dev, void *fdt, int node_off)
> +{
> +    VIOsPAPRVLANDevice *vdev = (VIOsPAPRVLANDevice *)dev;
> +    int ret;
> +
> +    ret = fdt_setprop(fdt, node_off, "local-mac-address",
> +&vdev->nicconf.macaddr, ETH_ALEN);
> +    if (ret<  0) {
> +        return ret;
> +    }
> +
> +    ret = fdt_setprop_cell(fdt, node_off, "ibm,mac-address-filters", 0);
> +    if (ret<  0) {
> +        return ret;
> +    }
> +
> +    return 0;
> +}
> +
> +static int check_bd(VIOsPAPRVLANDevice *dev, vlan_bd_t bd, target_ulong alignment)
> +{
> +    if ((VLAN_BD_ADDR(bd) % alignment)
> +        || (VLAN_BD_LEN(bd) % alignment)) {
> +        return -1;
> +    }
> +
> +    if (spapr_vio_check_tces(&dev->sdev, VLAN_BD_ADDR(bd),
> +                             VLAN_BD_LEN(bd), SPAPR_TCE_RW) != 0) {
> +        return -1;
> +    }
> +
> +    return 0;
> +}
> +
> +static target_ulong h_register_logical_lan(CPUState *env, sPAPREnvironment *spapr,
> +                                           target_ulong opcode,
> +                                           target_ulong *args)
> +{
> +    target_ulong reg = args[0];
> +    target_ulong buf_list = args[1];
> +    target_ulong rec_queue = args[2];
> +    target_ulong filter_list = args[3];
> +//    target_ulong mac_address = args[4];
> +    VIOsPAPRDevice *sdev = spapr_vio_find_by_reg(spapr->vio_bus, reg);
> +    VIOsPAPRVLANDevice *dev = (VIOsPAPRVLANDevice *)sdev;
> +    vlan_bd_t filter_list_bd;
> +#ifdef DEBUG
> +    target_ulong mac_address = args[4];
> +#endif
> +
> +    if (!dev) {
> +        return H_PARAMETER;
> +    }
> +
> +    if (dev->isopen) {
> +        fprintf(stderr, "H_REGISTER_LOGICAL_LAN called twice without "
> +                "H_FREE_LOGICAL_LAN\n");
> +        return H_RESOURCE;
> +    }
> +
> +    if (check_bd(dev, VLAN_VALID_BD(buf_list, SPAPR_VIO_TCE_PAGE_SIZE),
> +                 SPAPR_VIO_TCE_PAGE_SIZE)<  0) {
> +        fprintf(stderr, "Bad buf_list 0x" TARGET_FMT_lx
> +                " for H_REGISTER_LOGICAL_LAN\n", buf_list);
> +        return H_PARAMETER;
> +    }
> +
> +    filter_list_bd = VLAN_VALID_BD(filter_list, SPAPR_VIO_TCE_PAGE_SIZE);
> +    if (check_bd(dev, filter_list_bd, SPAPR_VIO_TCE_PAGE_SIZE)<  0) {
> +        fprintf(stderr, "Bad filter_list 0x" TARGET_FMT_lx
> +                " for H_REGISTER_LOGICAL_LAN\n", filter_list);
> +        return H_PARAMETER;
> +    }
> +
> +    if (!(rec_queue&  VLAN_BD_VALID)
> +        || (check_bd(dev, rec_queue, VLAN_RQ_ALIGNMENT)<  0)) {
> +        fprintf(stderr, "Bad receive queue for H_REGISTER_LOGICAL_LAN\n");
> +        return H_PARAMETER;
> +    }
> +
> +    dev->buf_list = buf_list;
> +    sdev->signal_state = 0;
> +
> +    rec_queue&= ~VLAN_BD_TOGGLE;
> +
> +    /* Initialize the buffer list */
> +    stq_tce(sdev, buf_list, rec_queue);
> +    stq_tce(sdev, buf_list + 8, filter_list_bd);
> +    spapr_tce_dma_zero(sdev, buf_list + VLAN_RX_BDS_OFF,
> +                       SPAPR_VIO_TCE_PAGE_SIZE - VLAN_RX_BDS_OFF);
> +    dev->add_buf_ptr = VLAN_RX_BDS_OFF - 8;
> +    dev->use_buf_ptr = VLAN_RX_BDS_OFF - 8;
> +    dev->rx_bufs = 0;
> +    dev->rxq_ptr = 0;
> +
> +    /* Initialize the receive queue */
> +    spapr_tce_dma_zero(sdev, VLAN_BD_ADDR(rec_queue), VLAN_BD_LEN(rec_queue));
> +
> +    dev->isopen = 1;
> +    return H_SUCCESS;
> +}
> +
> +
> +static target_ulong h_free_logical_lan(CPUState *env, sPAPREnvironment *spapr,
> +                                       target_ulong opcode, target_ulong *args)
> +{
> +    target_ulong reg = args[0];
> +    VIOsPAPRDevice *sdev = spapr_vio_find_by_reg(spapr->vio_bus, reg);
> +    VIOsPAPRVLANDevice *dev = (VIOsPAPRVLANDevice *)sdev;
> +
> +    if (!dev) {
> +        return H_PARAMETER;
> +    }
> +
> +    if (!dev->isopen) {
> +        fprintf(stderr, "H_FREE_LOGICAL_LAN called without "
> +                "H_REGISTER_LOGICAL_LAN\n");
> +        return H_RESOURCE;
> +    }
> +
> +    dev->buf_list = 0;
> +    dev->rx_bufs = 0;
> +    dev->isopen = 0;
> +    return H_SUCCESS;
> +}
> +
> +static target_ulong h_add_logical_lan_buffer(CPUState *env, sPAPREnvironment *spapr,
> +                                             target_ulong opcode, target_ulong *args)
> +{
> +    target_ulong reg = args[0];
> +    target_ulong buf = args[1];
> +    VIOsPAPRDevice *sdev = spapr_vio_find_by_reg(spapr->vio_bus, reg);
> +    VIOsPAPRVLANDevice *dev = (VIOsPAPRVLANDevice *)sdev;
> +    vlan_bd_t bd;
> +
> +    dprintf("H_ADD_LOGICAL_LAN_BUFFER(0x" TARGET_FMT_lx
> +            ", 0x" TARGET_FMT_lx ")\n", reg, buf);
> +
> +    if (!sdev) {
> +        fprintf(stderr, "Wrong device in h_add_logical_lan_buffer\n");
> +        return H_PARAMETER;
> +    }
> +
> +    if ((check_bd(dev, buf, 4)<  0)
> +        || (VLAN_BD_LEN(buf)<  16)) {
> +        fprintf(stderr, "Bad buffer enqueued in h_add_logical_lan_buffer\n");
> +        return H_PARAMETER;
> +    }
> +
> +    if (!dev->isopen || dev->rx_bufs>= VLAN_MAX_BUFS) {
> +        return H_RESOURCE;
> +    }
> +
> +    do {
> +        dev->add_buf_ptr += 8;
> +        if (dev->add_buf_ptr>= SPAPR_VIO_TCE_PAGE_SIZE) {
> +            dev->add_buf_ptr = VLAN_RX_BDS_OFF;
> +        }
> +
> +        bd = ldq_tce(sdev, dev->buf_list + dev->add_buf_ptr);
> +    } while (bd&  VLAN_BD_VALID);
> +
> +    stq_tce(sdev, dev->buf_list + dev->add_buf_ptr, buf);
> +
> +    dev->rx_bufs++;
> +
> +    dprintf("h_add_logical_lan_buffer():  Added buf  ptr=%d  rx_bufs=%d"
> +            " bd=0x%016llx\n", dev->add_buf_ptr, dev->rx_bufs,
> +            (unsigned long long)buf);
> +
> +    return H_SUCCESS;
> +}
> +
> +static target_ulong h_send_logical_lan(CPUState *env, sPAPREnvironment *spapr,
> +                                       target_ulong opcode, target_ulong *args)
> +{
> +    target_ulong reg = args[0];
> +    target_ulong *bufs = args + 1;
> +    target_ulong continue_token = args[7];
> +    VIOsPAPRDevice *sdev = spapr_vio_find_by_reg(spapr->vio_bus, reg);
> +    VIOsPAPRVLANDevice *dev = (VIOsPAPRVLANDevice *)sdev;
> +    unsigned total_len;
> +    uint8_t *lbuf, *p;
> +    int i, nbufs;
> +    int ret = H_SUCCESS;
> +
> +    dprintf("H_SEND_LOGICAL_LAN(0x" TARGET_FMT_lx ",<bufs>, 0x"
> +            TARGET_FMT_lx ")\n", reg, continue_token);
> +
> +    if (!sdev) {
> +        return H_PARAMETER;
> +    }
> +
> +    dprintf("rxbufs = %d\n", dev->rx_bufs);
> +
> +    if (!dev->isopen) {
> +        return H_DROPPED;
> +    }
> +
> +    if (continue_token) {
> +        return H_HARDWARE; /* FIXME actually handle this */
> +    }
> +
> +    total_len = 0;
> +    for (i = 0; i<  6; i++) {
> +        dprintf("   buf desc: 0x" TARGET_FMT_lx "\n", bufs[i]);
> +        if (!(bufs[i]&  VLAN_BD_VALID)) {
> +            break;
> +        }
> +        total_len += VLAN_BD_LEN(bufs[i]);
> +    }
> +
> +    nbufs = i;
> +    dprintf("h_send_logical_lan() %d buffers, total length 0x%x\n",
> +            nbufs, total_len);
> +
> +    if (total_len == 0) {
> +        return ret;
> +    }
> +
> +    lbuf = qemu_mallocz(total_len);
> +    p = lbuf;
> +    for (i = 0; i<  nbufs; i++) {
> +        ret = spapr_tce_dma_read(sdev, VLAN_BD_ADDR(bufs[i]),
> +                                 p, VLAN_BD_LEN(bufs[i]));
> +        if (ret<  0) {
> +            goto out;
> +        }
> +
> +        p += VLAN_BD_LEN(bufs[i]);
> +    }

I don't like the idea that there's a guest driven allocation that can 
reach 100mb here.  I'd suggest that we at least limit total_len to 64k 
to be on the safe side since a packet can't be larger than that anyway.

Regards,

Anthony Liguori