From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([2001:4830:134:3::10]:52360) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1WGyM8-00037w-B7 for qemu-devel@nongnu.org; Fri, 21 Feb 2014 17:14:09 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1WGyM3-0000hN-12 for qemu-devel@nongnu.org; Fri, 21 Feb 2014 17:14:04 -0500 Received: from mx1.redhat.com ([209.132.183.28]:56395) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1WGyM2-0000hA-M6 for qemu-devel@nongnu.org; Fri, 21 Feb 2014 17:13:58 -0500 Received: from int-mx12.intmail.prod.int.phx2.redhat.com (int-mx12.intmail.prod.int.phx2.redhat.com [10.5.11.25]) by mx1.redhat.com (8.14.4/8.14.4) with ESMTP id s1LMDvRD004739 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=OK) for ; Fri, 21 Feb 2014 17:13:58 -0500 From: Kevin Wolf Date: Fri, 21 Feb 2014 23:12:43 +0100 Message-Id: <1393020771-32712-47-git-send-email-kwolf@redhat.com> In-Reply-To: <1393020771-32712-1-git-send-email-kwolf@redhat.com> References: <1393020771-32712-1-git-send-email-kwolf@redhat.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: quoted-printable Subject: [Qemu-devel] [PULL 46/54] quorum: Add quorum mechanism. List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: qemu-devel@nongnu.org Cc: kwolf@redhat.com From: Beno=C3=AEt Canet This patchset enables the core of the quorum mechanism. The num_children reads are compared to get the majority version and if th= is version exists more than threshold times the guest won't see the error at= all. If a block is corrupted or if an error occurs during an IO or if the quor= um cannot be established QMP events are used to report to the management. Use gnutls's SHA-256 to compare versions. --enable-quorum must be used to enable the feature. Signed-off-by: Benoit Canet Reviewed-by: Max Reitz Signed-off-by: Kevin Wolf --- block/Makefile.objs | 2 +- block/quorum.c | 391 ++++++++++++++++++++++++++++++++++++++++= +++++- configure | 36 +++++ docs/qmp/qmp-events.txt | 36 +++++ include/monitor/monitor.h | 2 + monitor.c | 2 + 6 files changed, 467 insertions(+), 2 deletions(-) diff --git a/block/Makefile.objs b/block/Makefile.objs index 716556f..425d7fb 100644 --- a/block/Makefile.objs +++ b/block/Makefile.objs @@ -3,7 +3,7 @@ block-obj-y +=3D qcow2.o qcow2-refcount.o qcow2-cluster.o= qcow2-snapshot.o qcow2-c block-obj-y +=3D qed.o qed-gencb.o qed-l2-cache.o qed-table.o qed-cluste= r.o block-obj-y +=3D qed-check.o block-obj-$(CONFIG_VHDX) +=3D vhdx.o vhdx-endian.o vhdx-log.o -block-obj-y +=3D quorum.o +block-obj-$(CONFIG_QUORUM) +=3D quorum.o block-obj-y +=3D parallels.o blkdebug.o blkverify.o block-obj-y +=3D snapshot.o qapi.o block-obj-$(CONFIG_WIN32) +=3D raw-win32.o win32-aio.o diff --git a/block/quorum.c b/block/quorum.c index fd19662..4beee96 100644 --- a/block/quorum.c +++ b/block/quorum.c @@ -13,7 +13,43 @@ * See the COPYING file in the top-level directory. */ =20 +#include +#include #include "block/block_int.h" +#include "qapi/qmp/qjson.h" + +#define HASH_LENGTH 32 + +/* This union holds a vote hash value */ +typedef union QuorumVoteValue { + char h[HASH_LENGTH]; /* SHA-256 hash */ + int64_t l; /* simpler 64 bits hash */ +} QuorumVoteValue; + +/* A vote item */ +typedef struct QuorumVoteItem { + int index; + QLIST_ENTRY(QuorumVoteItem) next; +} QuorumVoteItem; + +/* this structure is a vote version. A version is the set of votes shari= ng the + * same vote value. + * The set of votes will be tracked with the items field and its cardina= lity is + * vote_count. + */ +typedef struct QuorumVoteVersion { + QuorumVoteValue value; + int index; + int vote_count; + QLIST_HEAD(, QuorumVoteItem) items; + QLIST_ENTRY(QuorumVoteVersion) next; +} QuorumVoteVersion; + +/* this structure holds a group of vote versions together */ +typedef struct QuorumVotes { + QLIST_HEAD(, QuorumVoteVersion) vote_list; + bool (*compare)(QuorumVoteValue *a, QuorumVoteValue *b); +} QuorumVotes; =20 /* the following structure holds the state of one quorum instance */ typedef struct BDRVQuorumState { @@ -65,10 +101,14 @@ struct QuorumAIOCB { int count; /* number of completed AIOCB */ int success_count; /* number of successfully completed AIOC= B */ =20 + QuorumVotes votes; + bool is_read; int vote_ret; }; =20 +static void quorum_vote(QuorumAIOCB *acb); + static void quorum_aio_cancel(BlockDriverAIOCB *blockacb) { QuorumAIOCB *acb =3D container_of(blockacb, QuorumAIOCB, common); @@ -94,6 +134,10 @@ static void quorum_aio_finalize(QuorumAIOCB *acb) BDRVQuorumState *s =3D acb->common.bs->opaque; int i, ret =3D 0; =20 + if (acb->vote_ret) { + ret =3D acb->vote_ret; + } + acb->common.cb(acb->common.opaque, ret); =20 if (acb->is_read) { @@ -107,6 +151,16 @@ static void quorum_aio_finalize(QuorumAIOCB *acb) qemu_aio_release(acb); } =20 +static bool quorum_sha256_compare(QuorumVoteValue *a, QuorumVoteValue *b= ) +{ + return !memcmp(a->h, b->h, HASH_LENGTH); +} + +static bool quorum_64bits_compare(QuorumVoteValue *a, QuorumVoteValue *b= ) +{ + return a->l =3D=3D b->l; +} + static QuorumAIOCB *quorum_aio_get(BDRVQuorumState *s, BlockDriverState *bs, QEMUIOVector *qiov, @@ -125,6 +179,8 @@ static QuorumAIOCB *quorum_aio_get(BDRVQuorumState *s= , acb->qcrs =3D g_new0(QuorumChildRequest, s->num_children); acb->count =3D 0; acb->success_count =3D 0; + acb->votes.compare =3D quorum_sha256_compare; + QLIST_INIT(&acb->votes.vote_list); acb->is_read =3D false; acb->vote_ret =3D 0; =20 @@ -137,6 +193,48 @@ static QuorumAIOCB *quorum_aio_get(BDRVQuorumState *= s, return acb; } =20 +static void quorum_report_bad(QuorumAIOCB *acb, char *node_name, int ret= ) +{ + QObject *data; + assert(node_name); + data =3D qobject_from_jsonf("{ 'ret': %d" + ", 'node-name': %s" + ", 'sector-num': %" PRId64 + ", 'sectors-count': %d }", + ret, node_name, acb->sector_num, acb->nb_s= ectors); + monitor_protocol_event(QEVENT_QUORUM_REPORT_BAD, data); + qobject_decref(data); +} + +static void quorum_report_failure(QuorumAIOCB *acb) +{ + QObject *data; + const char *reference =3D acb->common.bs->device_name[0] ? + acb->common.bs->device_name : + acb->common.bs->node_name; + data =3D qobject_from_jsonf("{ 'reference': %s" + ", 'sector-num': %" PRId64 + ", 'sectors-count': %d }", + reference, acb->sector_num, acb->nb_sector= s); + monitor_protocol_event(QEVENT_QUORUM_FAILURE, data); + qobject_decref(data); +} + +static int quorum_vote_error(QuorumAIOCB *acb); + +static bool quorum_has_too_much_io_failed(QuorumAIOCB *acb) +{ + BDRVQuorumState *s =3D acb->common.bs->opaque; + + if (acb->success_count < s->threshold) { + acb->vote_ret =3D quorum_vote_error(acb); + quorum_report_failure(acb); + return true; + } + + return false; +} + static void quorum_aio_cb(void *opaque, int ret) { QuorumChildRequest *sacb =3D opaque; @@ -147,6 +245,8 @@ static void quorum_aio_cb(void *opaque, int ret) acb->count++; if (ret =3D=3D 0) { acb->success_count++; + } else { + quorum_report_bad(acb, sacb->aiocb->bs->node_name, ret); } assert(acb->count <=3D s->num_children); assert(acb->success_count <=3D s->num_children); @@ -154,9 +254,298 @@ static void quorum_aio_cb(void *opaque, int ret) return; } =20 + /* Do the vote on read */ + if (acb->is_read) { + quorum_vote(acb); + } else { + quorum_has_too_much_io_failed(acb); + } + quorum_aio_finalize(acb); } =20 +static void quorum_report_bad_versions(BDRVQuorumState *s, + QuorumAIOCB *acb, + QuorumVoteValue *value) +{ + QuorumVoteVersion *version; + QuorumVoteItem *item; + + QLIST_FOREACH(version, &acb->votes.vote_list, next) { + if (acb->votes.compare(&version->value, value)) { + continue; + } + QLIST_FOREACH(item, &version->items, next) { + quorum_report_bad(acb, s->bs[item->index]->node_name, 0); + } + } +} + +static void quorum_copy_qiov(QEMUIOVector *dest, QEMUIOVector *source) +{ + int i; + assert(dest->niov =3D=3D source->niov); + assert(dest->size =3D=3D source->size); + for (i =3D 0; i < source->niov; i++) { + assert(dest->iov[i].iov_len =3D=3D source->iov[i].iov_len); + memcpy(dest->iov[i].iov_base, + source->iov[i].iov_base, + source->iov[i].iov_len); + } +} + +static void quorum_count_vote(QuorumVotes *votes, + QuorumVoteValue *value, + int index) +{ + QuorumVoteVersion *v =3D NULL, *version =3D NULL; + QuorumVoteItem *item; + + /* look if we have something with this hash */ + QLIST_FOREACH(v, &votes->vote_list, next) { + if (votes->compare(&v->value, value)) { + version =3D v; + break; + } + } + + /* It's a version not yet in the list add it */ + if (!version) { + version =3D g_new0(QuorumVoteVersion, 1); + QLIST_INIT(&version->items); + memcpy(&version->value, value, sizeof(version->value)); + version->index =3D index; + version->vote_count =3D 0; + QLIST_INSERT_HEAD(&votes->vote_list, version, next); + } + + version->vote_count++; + + item =3D g_new0(QuorumVoteItem, 1); + item->index =3D index; + QLIST_INSERT_HEAD(&version->items, item, next); +} + +static void quorum_free_vote_list(QuorumVotes *votes) +{ + QuorumVoteVersion *version, *next_version; + QuorumVoteItem *item, *next_item; + + QLIST_FOREACH_SAFE(version, &votes->vote_list, next, next_version) { + QLIST_REMOVE(version, next); + QLIST_FOREACH_SAFE(item, &version->items, next, next_item) { + QLIST_REMOVE(item, next); + g_free(item); + } + g_free(version); + } +} + +static int quorum_compute_hash(QuorumAIOCB *acb, int i, QuorumVoteValue = *hash) +{ + int j, ret; + gnutls_hash_hd_t dig; + QEMUIOVector *qiov =3D &acb->qcrs[i].qiov; + + ret =3D gnutls_hash_init(&dig, GNUTLS_DIG_SHA256); + + if (ret < 0) { + return ret; + } + + for (j =3D 0; j < qiov->niov; j++) { + ret =3D gnutls_hash(dig, qiov->iov[j].iov_base, qiov->iov[j].iov= _len); + if (ret < 0) { + break; + } + } + + gnutls_hash_deinit(dig, (void *) hash); + return ret; +} + +static QuorumVoteVersion *quorum_get_vote_winner(QuorumVotes *votes) +{ + int max =3D 0; + QuorumVoteVersion *candidate, *winner =3D NULL; + + QLIST_FOREACH(candidate, &votes->vote_list, next) { + if (candidate->vote_count > max) { + max =3D candidate->vote_count; + winner =3D candidate; + } + } + + return winner; +} + +/* qemu_iovec_compare is handy for blkverify mode because it returns the= first + * differing byte location. Yet it is handcoded to compare vectors one b= yte + * after another so it does not benefit from the libc SIMD optimizations. + * quorum_iovec_compare is written for speed and should be used in the n= on + * blkverify mode of quorum. + */ +static bool quorum_iovec_compare(QEMUIOVector *a, QEMUIOVector *b) +{ + int i; + int result; + + assert(a->niov =3D=3D b->niov); + for (i =3D 0; i < a->niov; i++) { + assert(a->iov[i].iov_len =3D=3D b->iov[i].iov_len); + result =3D memcmp(a->iov[i].iov_base, + b->iov[i].iov_base, + a->iov[i].iov_len); + if (result) { + return false; + } + } + + return true; +} + +static void GCC_FMT_ATTR(2, 3) quorum_err(QuorumAIOCB *acb, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fprintf(stderr, "quorum: sector_num=3D%" PRId64 " nb_sectors=3D%d ", + acb->sector_num, acb->nb_sectors); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); + va_end(ap); + exit(1); +} + +static bool quorum_compare(QuorumAIOCB *acb, + QEMUIOVector *a, + QEMUIOVector *b) +{ + BDRVQuorumState *s =3D acb->common.bs->opaque; + ssize_t offset; + + /* This driver will replace blkverify in this particular case */ + if (s->is_blkverify) { + offset =3D qemu_iovec_compare(a, b); + if (offset !=3D -1) { + quorum_err(acb, "contents mismatch in sector %" PRId64, + acb->sector_num + + (uint64_t)(offset / BDRV_SECTOR_SIZE)); + } + return true; + } + + return quorum_iovec_compare(a, b); +} + +/* Do a vote to get the error code */ +static int quorum_vote_error(QuorumAIOCB *acb) +{ + BDRVQuorumState *s =3D acb->common.bs->opaque; + QuorumVoteVersion *winner =3D NULL; + QuorumVotes error_votes; + QuorumVoteValue result_value; + int i, ret =3D 0; + bool error =3D false; + + QLIST_INIT(&error_votes.vote_list); + error_votes.compare =3D quorum_64bits_compare; + + for (i =3D 0; i < s->num_children; i++) { + ret =3D acb->qcrs[i].ret; + if (ret) { + error =3D true; + result_value.l =3D ret; + quorum_count_vote(&error_votes, &result_value, i); + } + } + + if (error) { + winner =3D quorum_get_vote_winner(&error_votes); + ret =3D winner->value.l; + } + + quorum_free_vote_list(&error_votes); + + return ret; +} + +static void quorum_vote(QuorumAIOCB *acb) +{ + bool quorum =3D true; + int i, j, ret; + QuorumVoteValue hash; + BDRVQuorumState *s =3D acb->common.bs->opaque; + QuorumVoteVersion *winner; + + if (quorum_has_too_much_io_failed(acb)) { + return; + } + + /* get the index of the first successful read */ + for (i =3D 0; i < s->num_children; i++) { + if (!acb->qcrs[i].ret) { + break; + } + } + + assert(i < s->num_children); + + /* compare this read with all other successful reads stopping at quo= rum + * failure + */ + for (j =3D i + 1; j < s->num_children; j++) { + if (acb->qcrs[j].ret) { + continue; + } + quorum =3D quorum_compare(acb, &acb->qcrs[i].qiov, &acb->qcrs[j]= .qiov); + if (!quorum) { + break; + } + } + + /* Every successful read agrees */ + if (quorum) { + quorum_copy_qiov(acb->qiov, &acb->qcrs[i].qiov); + return; + } + + /* compute hashes for each successful read, also store indexes */ + for (i =3D 0; i < s->num_children; i++) { + if (acb->qcrs[i].ret) { + continue; + } + ret =3D quorum_compute_hash(acb, i, &hash); + /* if ever the hash computation failed */ + if (ret < 0) { + acb->vote_ret =3D ret; + goto free_exit; + } + quorum_count_vote(&acb->votes, &hash, i); + } + + /* vote to select the most represented version */ + winner =3D quorum_get_vote_winner(&acb->votes); + + /* if the winner count is smaller than threshold the read fails */ + if (winner->vote_count < s->threshold) { + quorum_report_failure(acb); + acb->vote_ret =3D -EIO; + goto free_exit; + } + + /* we have a winner: copy it */ + quorum_copy_qiov(acb->qiov, &acb->qcrs[winner->index].qiov); + + /* some versions are bad print them */ + quorum_report_bad_versions(s, acb, &winner->value); + +free_exit: + /* free lists */ + quorum_free_vote_list(&acb->votes); +} + static BlockDriverAIOCB *quorum_aio_readv(BlockDriverState *bs, int64_t sector_num, QEMUIOVector *qiov, @@ -178,7 +567,7 @@ static BlockDriverAIOCB *quorum_aio_readv(BlockDriver= State *bs, } =20 for (i =3D 0; i < s->num_children; i++) { - bdrv_aio_readv(s->bs[i], sector_num, qiov, nb_sectors, + bdrv_aio_readv(s->bs[i], sector_num, &acb->qcrs[i].qiov, nb_sect= ors, quorum_aio_cb, &acb->qcrs[i]); } =20 diff --git a/configure b/configure index 39f2a1a..6143acb 100755 --- a/configure +++ b/configure @@ -264,6 +264,7 @@ gtkabi=3D"2.0" tpm=3D"no" libssh2=3D"" vhdx=3D"" +quorum=3D"no" =20 # parse CC options first for opt do @@ -1005,6 +1006,10 @@ for opt do ;; --disable-vhdx) vhdx=3D"no" ;; + --disable-quorum) quorum=3D"no" + ;; + --enable-quorum) quorum=3D"yes" + ;; *) echo "ERROR: unknown option $opt"; show_help=3D"yes" ;; esac @@ -1261,6 +1266,8 @@ Advanced options (experts only): --enable-libssh2 enable ssh block device support --disable-vhdx disables support for the Microsoft VHDX image= format --enable-vhdx enable support for the Microsoft VHDX image f= ormat + --disable-quorum disable quorum block filter support + --enable-quorum enable quorum block filter support =20 NOTE: The object files are built at the place where configure is launche= d EOF @@ -1937,6 +1944,30 @@ EOF fi =20 ########################################## +# Quorum probe (check for gnutls) +if test "$quorum" !=3D "no" ; then +cat > $TMPC < +#include +int main(void) {char data[4096], digest[32]; +gnutls_hash_fast(GNUTLS_DIG_SHA256, data, 4096, digest); +return 0; +} +EOF +quorum_tls_cflags=3D`$pkg_config --cflags gnutls 2> /dev/null` +quorum_tls_libs=3D`$pkg_config --libs gnutls 2> /dev/null` +if compile_prog "$quorum_tls_cflags" "$quorum_tls_libs" ; then + qcow_tls=3Dyes + libs_softmmu=3D"$quorum_tls_libs $libs_softmmu" + libs_tools=3D"$quorum_tls_libs $libs_softmmu" + QEMU_CFLAGS=3D"$QEMU_CFLAGS $quorum_tls_cflags" +else + echo "gnutls > 2.10.0 required to compile Quorum" + exit 1 +fi +fi + +########################################## # VNC SASL detection if test "$vnc" =3D "yes" -a "$vnc_sasl" !=3D "no" ; then cat > $TMPC < Your SDL version is too old - please upgrade to have SDL suppor= t" @@ -4295,6 +4327,10 @@ if test "$libssh2" =3D "yes" ; then echo "CONFIG_LIBSSH2=3Dy" >> $config_host_mak fi =20 +if test "$quorum" =3D "yes" ; then + echo "CONFIG_QUORUM=3Dy" >> $config_host_mak +fi + if test "$virtio_blk_data_plane" =3D "yes" ; then echo 'CONFIG_VIRTIO_BLK_DATA_PLANE=3D$(CONFIG_VIRTIO)' >> $config_host= _mak fi diff --git a/docs/qmp/qmp-events.txt b/docs/qmp/qmp-events.txt index a378c87..00f9515 100644 --- a/docs/qmp/qmp-events.txt +++ b/docs/qmp/qmp-events.txt @@ -500,3 +500,39 @@ Example: =20 Note: If action is "reset", "shutdown", or "pause" the WATCHDOG event is followed respectively by the RESET, SHUTDOWN, or STOP events. + +QUORUM_FAILURE +-------------- + +Emitted by the Quorum block driver if it fails to establish a quorum. + +Data: + +- "reference": device name if defined else node name. +- "sector-num": Number of the first sector of the failed read operatio= n. +- "sector-count": Failed read operation sector count. + +Example: + +{ "event": "QUORUM_FAILURE", + "data": { "reference": "usr1", "sector-num": 345435, "sector-count"= : 5 }, + "timestamp": { "seconds": 1344522075, "microseconds": 745528 } } + +QUORUM_REPORT_BAD +----------------- + +Emitted to report a corruption of a Quorum file. + +Data: + +- "ret": The IO return code. +- "node-name": The graph node name of the block driver state. +- "sector-num": Number of the first sector of the failed read operatio= n. +- "sector-count": Failed read operation sector count. + +Example: + +{ "event": "QUORUM_REPORT_BAD", + "data": { "ret": 0, "node-name": "1.raw", "sector-num": 345435, + "sector-count": 5 }, + "timestamp": { "seconds": 1344522075, "microseconds": 745528 } } diff --git a/include/monitor/monitor.h b/include/monitor/monitor.h index 7e5f752..a49ea11 100644 --- a/include/monitor/monitor.h +++ b/include/monitor/monitor.h @@ -49,6 +49,8 @@ typedef enum MonitorEvent { QEVENT_SPICE_MIGRATE_COMPLETED, QEVENT_GUEST_PANICKED, QEVENT_BLOCK_IMAGE_CORRUPTED, + QEVENT_QUORUM_FAILURE, + QEVENT_QUORUM_REPORT_BAD, =20 /* Add to 'monitor_event_names' array in monitor.c when * defining new events here */ diff --git a/monitor.c b/monitor.c index de90fba..8ae095f 100644 --- a/monitor.c +++ b/monitor.c @@ -508,6 +508,8 @@ static const char *monitor_event_names[] =3D { [QEVENT_SPICE_MIGRATE_COMPLETED] =3D "SPICE_MIGRATE_COMPLETED", [QEVENT_GUEST_PANICKED] =3D "GUEST_PANICKED", [QEVENT_BLOCK_IMAGE_CORRUPTED] =3D "BLOCK_IMAGE_CORRUPTED", + [QEVENT_QUORUM_FAILURE] =3D "QUORUM_FAILURE", + [QEVENT_QUORUM_REPORT_BAD] =3D "QUORUM_REPORT_BAD", }; QEMU_BUILD_BUG_ON(ARRAY_SIZE(monitor_event_names) !=3D QEVENT_MAX) =20 --=20 1.8.1.4