Re: [Qemu-devel] [RFC PATCH RDMA support v3: 05/10] RDMA connection establishment (migration-rdma.c).

["Michael R. Hines" <mrhines@linux.vnet.ibm.com>]

Paolo, I just pulled your changes from master.

Will get started on merging with all the new comments....

- Michael

On 03/11/2013 09:41 AM, Paolo Bonzini wrote:
> Il 11/03/2013 05:33, Michael.R.Hines.mrhines@linux.vnet.ibm.com ha scritto:
>> From: "Michael R. Hines" <mrhines@us.ibm.com>
>>
>> Use 'migrate rdma:host:port' to begin the migration.
>>
>> The TCP control channel has finally been eliminated
>> when RDMA is used. Documentation of the use of SEND message
>> for transferring device state is covered in docs/rdma.txt
>>
>> Signed-off-by: Michael R. Hines <mrhines@us.ibm.com>
>> ---
>>   migration-rdma.c |  198 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
>>   1 file changed, 198 insertions(+)
>>   create mode 100644 migration-rdma.c
>>
>> diff --git a/migration-rdma.c b/migration-rdma.c
>> new file mode 100644
>> index 0000000..822b17a
>> --- /dev/null
>> +++ b/migration-rdma.c
>> @@ -0,0 +1,198 @@
>> +/*
>> + *  Copyright (C) 2013 Michael R. Hines <mrhines@us.ibm.com>
>> + *  Copyright (C) 2013 Jiuxing Liu <jl@us.ibm.com>
>> + *
>> + *  This program is free software; you can redistribute it and/or modify
>> + *  it under the terms of the GNU General Public License as published by
>> + *  the Free Software Foundation; under version 2 of the License.
>> + *
>> + *  This program is distributed in the hope that it will be useful,
>> + *  but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> + *  GNU General Public License for more details.
>> + *
>> + *  You should have received a copy of the GNU General Public License
>> + *  along with this program; if not, see <http://www.gnu.org/licenses/>.
>> + */
>> +#include "migration/rdma.h"
>> +#include "qemu-common.h"
>> +#include "migration/migration.h"
>> +#include "migration/qemu-file.h"
>> +#include <stdio.h>
>> +#include <sys/types.h>
>> +#include <sys/socket.h>
>> +#include <netdb.h>
>> +#include <arpa/inet.h>
>> +#include <string.h>
>> +
>> +//#define DEBUG_MIGRATION_RDMA
>> +
>> +#ifdef DEBUG_MIGRATION_RDMA
>> +#define DPRINTF(fmt, ...) \
>> +    do { printf("migration-rdma: " fmt, ## __VA_ARGS__); } while (0)
>> +#else
>> +#define DPRINTF(fmt, ...) \
>> +    do { } while (0)
>> +#endif
>> +
>> +static int rdma_accept_incoming_migration(RDMAData *rdma, Error **errp)
>> +{
>> +    int ret;
>> +
>> +    ret = qemu_rdma_migrate_listen(rdma, rdma->host, rdma->port);
>> +    if (ret) {
>> +        qemu_rdma_print("rdma migration: error listening!");
>> +        goto err_rdma_server_wait;
>> +    }
>> +
>> +    ret = qemu_rdma_alloc_qp(&rdma->rdma_ctx);
>> +    if (ret) {
>> +        qemu_rdma_print("rdma migration: error allocating qp!");
>> +        goto err_rdma_server_wait;
>> +    }
>> +
>> +    ret = qemu_rdma_migrate_accept(&rdma->rdma_ctx, NULL, NULL, NULL, 0);
>> +    if (ret) {
>> +        qemu_rdma_print("rdma migration: error accepting connection!");
>> +        goto err_rdma_server_wait;
>> +    }
>> +
>> +    ret = qemu_rdma_post_recv_qemu_file(rdma);
>> +    if (ret) {
>> +        qemu_rdma_print("rdma migration: error posting second qemu file recv!");
>> +        goto err_rdma_server_wait;
>> +    }
>> +
>> +    ret = qemu_rdma_post_send_remote_info(rdma);
>> +    if (ret) {
>> +        qemu_rdma_print("rdma migration: error sending remote info!");
>> +        goto err_rdma_server_wait;
>> +    }
>> +
>> +    ret = qemu_rdma_wait_for_wrid(rdma, RDMA_WRID_SEND_REMOTE_INFO);
>> +    if (ret < 0) {
>> +        qemu_rdma_print("rdma migration: polling remote info error!");
>> +        goto err_rdma_server_wait;
>> +    }
>> +
>> +    rdma->total_bytes = 0;
>> +    rdma->enabled = 1;
>> +    qemu_rdma_dump_gid("server_connect", rdma->rdma_ctx.cm_id);
>> +    return 0;
>> +
>> +err_rdma_server_wait:
>> +    qemu_rdma_cleanup(rdma);
>> +    return -1;
>> +
>> +}
>> +
>> +int rdma_start_incoming_migration(const char * host_port, Error **errp)
>> +{
>> +    QEMUFile *f;
>> +    int ret;
>> +    RDMAData rdma;
>> +
>> +    if ((ret = qemu_rdma_data_init(&rdma, host_port, errp)) < 0)
>> +        return ret;
>> +
>> +    ret = qemu_rdma_server_init(&rdma, NULL);
>> +
>> +    DPRINTF("Starting RDMA-based incoming migration\n");
>> +
>> +    if (!ret) {
>> +        DPRINTF("qemu_rdma_server_init success\n");
>> +        ret = qemu_rdma_server_prepare(&rdma, NULL);
>> +
>> +        if (!ret) {
>> +            DPRINTF("qemu_rdma_server_prepare success\n");
>> +
>> +            ret = rdma_accept_incoming_migration(&rdma, NULL);
>> +            if(!ret)
>> +                DPRINTF("qemu_rdma_accept_incoming_migration success\n");
>> +            f = qemu_fopen_rdma(&rdma);
>> +            if (f == NULL) {
>> +                fprintf(stderr, "could not qemu_fopen RDMA\n");
>> +                ret = -EIO;
>> +            }
>> +
>> +            process_incoming_migration(f);
>> +        }
>> +    }
>> +
>> +    return ret;
>> +}
>> +
>> +/*
>> + * Not sure what this is for yet...
>> + */
>> +static int qemu_rdma_errno(MigrationState *s)
>> +{
>> +    return 0;
>> +}
>> +
>> +/*
>> + * SEND messages for device state only.
>> + * pc.ram is handled elsewhere...
>> + */
>> +static int qemu_rdma_send(MigrationState *s, const void * buf, size_t size)
>> +{
>> +    size_t len, remaining = size;
>> +    uint8_t * data = (void *) buf;
>> +
>> +    if (qemu_rdma_write_flush(&s->rdma) < 0) {
>> +        qemu_file_set_error(s->file, -EIO);
>> +        return -EIO;
>> +    }
>> +
>> +    while(remaining) {
>> +        len = MIN(remaining, RDMA_SEND_INCREMENT);
>> +        remaining -= len;
>> +        DPRINTF("iter Sending %" PRId64 "-byte SEND\n", len);
>> +
>> +        if(qemu_rdma_exchange_send(&s->rdma, data, len) < 0)
>> +                return -EINVAL;
>> +
>> +        data += len;
>> +    }
>> +
>> +    return size;
>> +}
>> +
>> +static int qemu_rdma_close(MigrationState *s)
>> +{
>> +    DPRINTF("qemu_rdma_close\n");
>> +    qemu_rdma_cleanup(&s->rdma);
>> +    return 0;
>> +}
>> +
>> +void rdma_start_outgoing_migration(MigrationState *s, const char *host_port, Error **errp)
>> +{
>> +    int ret;
>> +
>> +#ifndef CONFIG_RDMA
>> +    error_set(errp, QERR_FEATURE_DISABLED, "rdma migration");
>> +    return;
>> +#endif
>> +
>> +    if (qemu_rdma_data_init(&s->rdma, host_port, errp) < 0)
>> +        return;
>> +
>> +    ret = qemu_rdma_client_init(&s->rdma, NULL);
>> +    if(!ret) {
>> +        DPRINTF("qemu_rdma_client_init success\n");
>> +        ret = qemu_rdma_client_connect(&s->rdma, NULL);
>> +
>> +        if(!ret) {
>> +            s->get_error = qemu_rdma_errno;
>> +            s->write = qemu_rdma_send;
>> +            s->close = qemu_rdma_close;
>> +            s->fd = -2;
>> +            DPRINTF("qemu_rdma_client_connect success\n");
>> +            migrate_fd_connect(s);
>> +            return;
>> +        }
>> +    }
>> +
>> +    s->fd = -1;
>> +    migrate_fd_error(s);
>> +}
>>
> Note that as soon as the pending migration patches hit upstream, almost
> all of this code will move to QEMUFileRDMA (in particular qemu_rdma_send
> and qemu_rdma_close).  It should become simpler.
>
> Paolo
>