From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([208.118.235.92]:39036) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1UF9D8-0005O3-Up for qemu-devel@nongnu.org; Mon, 11 Mar 2013 16:20:47 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1UF9D3-0004Rc-Jg for qemu-devel@nongnu.org; Mon, 11 Mar 2013 16:20:42 -0400 Received: from e39.co.us.ibm.com ([32.97.110.160]:42875) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1UF9D3-0004RT-9w for qemu-devel@nongnu.org; Mon, 11 Mar 2013 16:20:37 -0400 Received: from /spool/local by e39.co.us.ibm.com with IBM ESMTP SMTP Gateway: Authorized Use Only! Violators will be prosecuted for from ; Mon, 11 Mar 2013 14:20:35 -0600 Received: from d01relay07.pok.ibm.com (d01relay07.pok.ibm.com [9.56.227.147]) by d01dlp03.pok.ibm.com (Postfix) with ESMTP id 67028C90058 for ; Mon, 11 Mar 2013 16:20:30 -0400 (EDT) Received: from d01av03.pok.ibm.com (d01av03.pok.ibm.com [9.56.224.217]) by d01relay07.pok.ibm.com (8.13.8/8.13.8/NCO v10.0) with ESMTP id r2BKK6IT65077306 for ; Mon, 11 Mar 2013 16:20:07 -0400 Received: from d01av03.pok.ibm.com (loopback [127.0.0.1]) by d01av03.pok.ibm.com (8.14.4/8.13.1/NCO v10.0 AVout) with ESMTP id r2BKK6Xh030970 for ; Mon, 11 Mar 2013 17:20:06 -0300 Message-ID: <513E3C74.8090806@linux.vnet.ibm.com> Date: Mon, 11 Mar 2013 16:20:04 -0400 From: "Michael R. Hines" MIME-Version: 1.0 References: <1362976414-21396-1-git-send-email-mrhines@us.ibm.com> <1362976414-21396-6-git-send-email-mrhines@us.ibm.com> <513DDF01.2050400@redhat.com> In-Reply-To: <513DDF01.2050400@redhat.com> Content-Type: text/plain; charset=ISO-8859-15; format=flowed Content-Transfer-Encoding: 7bit Subject: Re: [Qemu-devel] [RFC PATCH RDMA support v3: 05/10] RDMA connection establishment (migration-rdma.c). List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Paolo Bonzini Cc: aliguori@us.ibm.com, mst@redhat.com, michael.r.hines.mrhines@linux.vnet.ibm.com, qemu-devel@nongnu.org, owasserm@redhat.com, abali@us.ibm.com, mrhines@us.ibm.com, gokul@us.ibm.com Paolo, I just pulled your changes from master. Will get started on merging with all the new comments.... - Michael On 03/11/2013 09:41 AM, Paolo Bonzini wrote: > Il 11/03/2013 05:33, Michael.R.Hines.mrhines@linux.vnet.ibm.com ha scritto: >> From: "Michael R. Hines" >> >> Use 'migrate rdma:host:port' to begin the migration. >> >> The TCP control channel has finally been eliminated >> when RDMA is used. Documentation of the use of SEND message >> for transferring device state is covered in docs/rdma.txt >> >> Signed-off-by: Michael R. Hines >> --- >> migration-rdma.c | 198 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ >> 1 file changed, 198 insertions(+) >> create mode 100644 migration-rdma.c >> >> diff --git a/migration-rdma.c b/migration-rdma.c >> new file mode 100644 >> index 0000000..822b17a >> --- /dev/null >> +++ b/migration-rdma.c >> @@ -0,0 +1,198 @@ >> +/* >> + * Copyright (C) 2013 Michael R. Hines >> + * Copyright (C) 2013 Jiuxing Liu >> + * >> + * This program is free software; you can redistribute it and/or modify >> + * it under the terms of the GNU General Public License as published by >> + * the Free Software Foundation; under version 2 of the License. >> + * >> + * This program is distributed in the hope that it will be useful, >> + * but WITHOUT ANY WARRANTY; without even the implied warranty of >> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >> + * GNU General Public License for more details. >> + * >> + * You should have received a copy of the GNU General Public License >> + * along with this program; if not, see . >> + */ >> +#include "migration/rdma.h" >> +#include "qemu-common.h" >> +#include "migration/migration.h" >> +#include "migration/qemu-file.h" >> +#include >> +#include >> +#include >> +#include >> +#include >> +#include >> + >> +//#define DEBUG_MIGRATION_RDMA >> + >> +#ifdef DEBUG_MIGRATION_RDMA >> +#define DPRINTF(fmt, ...) \ >> + do { printf("migration-rdma: " fmt, ## __VA_ARGS__); } while (0) >> +#else >> +#define DPRINTF(fmt, ...) \ >> + do { } while (0) >> +#endif >> + >> +static int rdma_accept_incoming_migration(RDMAData *rdma, Error **errp) >> +{ >> + int ret; >> + >> + ret = qemu_rdma_migrate_listen(rdma, rdma->host, rdma->port); >> + if (ret) { >> + qemu_rdma_print("rdma migration: error listening!"); >> + goto err_rdma_server_wait; >> + } >> + >> + ret = qemu_rdma_alloc_qp(&rdma->rdma_ctx); >> + if (ret) { >> + qemu_rdma_print("rdma migration: error allocating qp!"); >> + goto err_rdma_server_wait; >> + } >> + >> + ret = qemu_rdma_migrate_accept(&rdma->rdma_ctx, NULL, NULL, NULL, 0); >> + if (ret) { >> + qemu_rdma_print("rdma migration: error accepting connection!"); >> + goto err_rdma_server_wait; >> + } >> + >> + ret = qemu_rdma_post_recv_qemu_file(rdma); >> + if (ret) { >> + qemu_rdma_print("rdma migration: error posting second qemu file recv!"); >> + goto err_rdma_server_wait; >> + } >> + >> + ret = qemu_rdma_post_send_remote_info(rdma); >> + if (ret) { >> + qemu_rdma_print("rdma migration: error sending remote info!"); >> + goto err_rdma_server_wait; >> + } >> + >> + ret = qemu_rdma_wait_for_wrid(rdma, RDMA_WRID_SEND_REMOTE_INFO); >> + if (ret < 0) { >> + qemu_rdma_print("rdma migration: polling remote info error!"); >> + goto err_rdma_server_wait; >> + } >> + >> + rdma->total_bytes = 0; >> + rdma->enabled = 1; >> + qemu_rdma_dump_gid("server_connect", rdma->rdma_ctx.cm_id); >> + return 0; >> + >> +err_rdma_server_wait: >> + qemu_rdma_cleanup(rdma); >> + return -1; >> + >> +} >> + >> +int rdma_start_incoming_migration(const char * host_port, Error **errp) >> +{ >> + QEMUFile *f; >> + int ret; >> + RDMAData rdma; >> + >> + if ((ret = qemu_rdma_data_init(&rdma, host_port, errp)) < 0) >> + return ret; >> + >> + ret = qemu_rdma_server_init(&rdma, NULL); >> + >> + DPRINTF("Starting RDMA-based incoming migration\n"); >> + >> + if (!ret) { >> + DPRINTF("qemu_rdma_server_init success\n"); >> + ret = qemu_rdma_server_prepare(&rdma, NULL); >> + >> + if (!ret) { >> + DPRINTF("qemu_rdma_server_prepare success\n"); >> + >> + ret = rdma_accept_incoming_migration(&rdma, NULL); >> + if(!ret) >> + DPRINTF("qemu_rdma_accept_incoming_migration success\n"); >> + f = qemu_fopen_rdma(&rdma); >> + if (f == NULL) { >> + fprintf(stderr, "could not qemu_fopen RDMA\n"); >> + ret = -EIO; >> + } >> + >> + process_incoming_migration(f); >> + } >> + } >> + >> + return ret; >> +} >> + >> +/* >> + * Not sure what this is for yet... >> + */ >> +static int qemu_rdma_errno(MigrationState *s) >> +{ >> + return 0; >> +} >> + >> +/* >> + * SEND messages for device state only. >> + * pc.ram is handled elsewhere... >> + */ >> +static int qemu_rdma_send(MigrationState *s, const void * buf, size_t size) >> +{ >> + size_t len, remaining = size; >> + uint8_t * data = (void *) buf; >> + >> + if (qemu_rdma_write_flush(&s->rdma) < 0) { >> + qemu_file_set_error(s->file, -EIO); >> + return -EIO; >> + } >> + >> + while(remaining) { >> + len = MIN(remaining, RDMA_SEND_INCREMENT); >> + remaining -= len; >> + DPRINTF("iter Sending %" PRId64 "-byte SEND\n", len); >> + >> + if(qemu_rdma_exchange_send(&s->rdma, data, len) < 0) >> + return -EINVAL; >> + >> + data += len; >> + } >> + >> + return size; >> +} >> + >> +static int qemu_rdma_close(MigrationState *s) >> +{ >> + DPRINTF("qemu_rdma_close\n"); >> + qemu_rdma_cleanup(&s->rdma); >> + return 0; >> +} >> + >> +void rdma_start_outgoing_migration(MigrationState *s, const char *host_port, Error **errp) >> +{ >> + int ret; >> + >> +#ifndef CONFIG_RDMA >> + error_set(errp, QERR_FEATURE_DISABLED, "rdma migration"); >> + return; >> +#endif >> + >> + if (qemu_rdma_data_init(&s->rdma, host_port, errp) < 0) >> + return; >> + >> + ret = qemu_rdma_client_init(&s->rdma, NULL); >> + if(!ret) { >> + DPRINTF("qemu_rdma_client_init success\n"); >> + ret = qemu_rdma_client_connect(&s->rdma, NULL); >> + >> + if(!ret) { >> + s->get_error = qemu_rdma_errno; >> + s->write = qemu_rdma_send; >> + s->close = qemu_rdma_close; >> + s->fd = -2; >> + DPRINTF("qemu_rdma_client_connect success\n"); >> + migrate_fd_connect(s); >> + return; >> + } >> + } >> + >> + s->fd = -1; >> + migrate_fd_error(s); >> +} >> > Note that as soon as the pending migration patches hit upstream, almost > all of this code will move to QEMUFileRDMA (in particular qemu_rdma_send > and qemu_rdma_close). It should become simpler. > > Paolo >