From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mailman by lists.gnu.org with tmda-scanned (Exim 4.43) id 1Ls5yp-0001f3-GK for qemu-devel@nongnu.org; Thu, 09 Apr 2009 21:56:31 -0400 Received: from exim by lists.gnu.org with spam-scanned (Exim 4.43) id 1Ls5yk-0001Zp-WA for qemu-devel@nongnu.org; Thu, 09 Apr 2009 21:56:30 -0400 Received: from [199.232.76.173] (port=35198 helo=monty-python.gnu.org) by lists.gnu.org with esmtp (Exim 4.43) id 1Ls5yk-0001Zi-Pt for qemu-devel@nongnu.org; Thu, 09 Apr 2009 21:56:26 -0400 Received: from phong.sigbus.net ([65.49.35.42]:60090) by monty-python.gnu.org with esmtps (TLS-1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.60) (envelope-from ) id 1Ls5yk-0003fn-1Q for qemu-devel@nongnu.org; Thu, 09 Apr 2009 21:56:26 -0400 Received: from voxel (c-71-202-202-194.hsd1.ca.comcast.net [71.202.202.194]) by phong.sigbus.net (Postfix) with ESMTPSA id 3F56995C0A1 for ; Thu, 9 Apr 2009 18:56:23 -0700 (PDT) Received: from nolan by voxel with local (Exim 4.69) (envelope-from ) id 1Ls5yg-0001i5-DW for qemu-devel@nongnu.org; Thu, 09 Apr 2009 18:56:22 -0700 From: Nolan Content-Type: text/plain Content-Transfer-Encoding: 7bit Date: Thu, 09 Apr 2009 18:56:21 -0700 Message-Id: <1239328581.15350.688.camel@voxel> Mime-Version: 1.0 Subject: [Qemu-devel] [PATCH] An alternative http protocol Reply-To: qemu-devel@nongnu.org List-Id: qemu-devel.nongnu.org List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: "qemu-devel@nongnu.org" Alexander Graf's posting of his HTTP protocol for block devices prompted me to finally cleanup and submit my implementation of the same thing, so that both may be considered and compared. The most obvious difference is that this implementation uses libneon instead of libcurl. It supports HTTPS, proxies, and HTTP basic auth, though be aware that if you pass a URL with credentials on the command line, anyone who can run "ps" on the host can see your credentials! It also supports rounding range request sizes up and a simple 1 entry cache, defaulting to 256KB. This _greatly_ reduces the load on the server when using a HTTP cdrom .iso, as well as when using an HTTP image as a source for "qemu-img convert". Without this, booting off an HTTP livecd iso results in tens of thousands to hundreds of thousands of 1500 byte HTTP range requests. It has been heavily tested in production patched into kvm-84, and lightly tested on qemu trunk. It does, however, use bdrv_read, so Anthony's complaint about AIO applies to this version as well. Signed-off-by: Nolan Leake sigbus.net> Makefile | 7 Makefile.target | 5 block-http.c | 463 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ block.c | 3 block.h | 1 configure | 24 ++ 6 files changed, 503 insertions(+) Index: Makefile.target =================================================================== --- Makefile.target (revision 7060) +++ Makefile.target (working copy) @@ -557,6 +557,11 @@ LIBS += $(CONFIG_BLUEZ_LIBS) endif +ifdef CONFIG_HTTP_BLOCK +LIBS += $(CONFIG_HTTP_BLOCK_LIBS) +CPPFLAGS += $(CONFIG_HTTP_BLOCK_CFLAGS) +endif + # SCSI layer OBJS+= lsi53c895a.o esp.o Index: Makefile =================================================================== --- Makefile (revision 7060) +++ Makefile (working copy) @@ -54,6 +54,10 @@ BLOCK_OBJS+=block-dmg.o block-bochs.o block-vpc.o block-vvfat.o BLOCK_OBJS+=block-qcow2.o block-parallels.o block-nbd.o BLOCK_OBJS+=nbd.o block.o aio.o +ifdef CONFIG_HTTP_BLOCK +BLOCK_OBJS+=block-http.o +LIBS+=$(CONFIG_HTTP_BLOCK_LIBS) +endif ifdef CONFIG_WIN32 BLOCK_OBJS += block-raw-win32.o @@ -192,6 +196,9 @@ bt-host.o: CFLAGS += $(CONFIG_BLUEZ_CFLAGS) +block-http.o: block-http.c + $(CC) $(CFLAGS) $(CPPFLAGS) $(CONFIG_HTTP_BLOCK_CFLAGS) -c -o $@ $< + libqemu_common.a: $(OBJS) ####################################################################### Index: block.c =================================================================== --- block.c (revision 7060) +++ block.c (working copy) @@ -1488,6 +1488,9 @@ bdrv_register(&bdrv_qcow2); bdrv_register(&bdrv_parallels); bdrv_register(&bdrv_nbd); +#ifdef CONFIG_HTTP_BLOCK + bdrv_register(&bdrv_http); +#endif } void aio_pool_init(AIOPool *pool, int aiocb_size, Index: block.h =================================================================== --- block.h (revision 7060) +++ block.h (working copy) @@ -20,6 +20,7 @@ extern BlockDriver bdrv_qcow2; extern BlockDriver bdrv_parallels; extern BlockDriver bdrv_nbd; +extern BlockDriver bdrv_http; typedef struct BlockDriverInfo { /* in bytes, 0 if irrelevant */ Index: configure =================================================================== --- configure (revision 7060) +++ configure (working copy) @@ -191,6 +191,7 @@ fdt="yes" sdl_x11="no" pkgversion="" +http_block="yes" # OS specific if check_define __linux__ ; then @@ -473,6 +474,8 @@ ;; --with-pkgversion=*) pkgversion=" ($optarg)" ;; + --disable-http-block) http_block="no" + ;; *) echo "ERROR: unknown option $opt"; show_help="yes" ;; esac @@ -597,6 +600,7 @@ echo " --disable-aio disable AIO support" echo " --disable-blobs disable installing provided firmware blobs" echo " --kerneldir=PATH look for kernel includes in PATH" +echo " --disable-http-block disable support for http block devices" echo "" echo "NOTE: The object files are built at the place where configure is launched" exit 1 @@ -1137,6 +1141,19 @@ fi fi +########################################## +# libneon probe +if test "$http_block" = "yes" ; then + cat > $TMPC << EOF +#include +int main(void) { return ne_sock_init(); } +EOF + if ! $cc $ARCH_CFLAGS `neon-config --cflags` `neon-config --libs` \ + -o $TMPE $TMPC 2> /dev/null ; then + http_block=no + fi +fi + # Check if tools are available to build documentation. if [ -x "`which texi2html 2>/dev/null`" ] && \ [ -x "`which pod2man 2>/dev/null`" ]; then @@ -1240,6 +1257,7 @@ echo "KVM support $kvm" echo "fdt support $fdt" echo "preadv support $preadv" +echo "HTTP block $http_block" if test $sdl_too_old = "yes"; then echo "-> Your SDL version is too old - please upgrade to have SDL support" @@ -1550,6 +1568,12 @@ echo "#define HAVE_FDT 1" >> $config_h echo "FDT_LIBS=-lfdt" >> $config_mak fi +if test "$http_block" = "yes" ; then + echo "#define CONFIG_HTTP_BLOCK 1" >> $config_h + echo "CONFIG_HTTP_BLOCK=yes" >> $config_mak + echo "CONFIG_HTTP_BLOCK_CFLAGS=`neon-config --cflags`" >> $config_mak + echo "CONFIG_HTTP_BLOCK_LIBS=`neon-config --libs`" >> $config_mak +fi # XXX: suppress that if [ "$bsd" = "yes" ] ; then Index: block-http.c =================================================================== --- block-http.c (revision 0) +++ block-http.c (revision 0) @@ -0,0 +1,463 @@ +/* + * QEMU Block driver for HTTP (read-only) + * + * Copyright (C) 2008 Nolan Leake + * + * Partially derived from block-nbd.c: + * Copyright (C) 2008 Bull S.A.S. + * Author: Laurent Vivier + * Some parts: + * Copyright (C) 2007 Anthony Liguori + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "qemu-common.h" +#include "block_int.h" +#include "assert.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#define HTTP_DEBUG 0 + +#define DEFAULT_CACHE_BLOCK_SIZE 262144 + +typedef struct BDRVHTTPState { + ne_session *session; + ne_uri proxy_uri; + ne_uri uri; + char *path; + int64_t size; + + uint8_t *cache; + uint64_t cache_offset; + int cache_len; + + int cache_block_size; +} BDRVHTTPState; + +static int use_count = 0; + +static int parse_uri(BDRVHTTPState *s, const char *uri) +{ + if (s->path) { + ne_free(s->path); + s->path = NULL; + } + ne_uri_free(&s->uri); + + if (ne_uri_parse(uri, &s->uri) != 0) { + fprintf(stderr, "HTTP: bad URL %s\n", uri); + return -EINVAL; + } + if (strcmp(s->uri.scheme, "http") != 0) { + fprintf(stderr, "HTTP: only http is supported\n"); + return -EINVAL; + } + if (s->uri.port == 0) { + s->uri.port = ne_uri_defaultport(s->uri.scheme); + } + + if (s->path) { + ne_free(s->path); + } + s->path = ne_path_escape(s->uri.path); + if (!s->path) { + fprintf(stderr, "HTTP: couldn't escape path\n"); + return -EINVAL; + } + + return 0; +} + +static int auth_creds(void *userdata, const char *realm, + int attempt, char *username, char *password) +{ + const char *userinfo = (const char *)userdata; + char *auth; + char *colon; + + if (!userinfo) { + return 1; + } + + auth = strdup(userinfo); + colon = index(auth, ':'); + if (colon) { + strncpy(password, colon + 1, NE_ABUFSIZ); + password[NE_ABUFSIZ - 1] = '\0'; + *colon = '\0'; + } else { + password[0] = '\0'; + } + strncpy(username, auth, NE_ABUFSIZ); + username[NE_ABUFSIZ - 1] = '\0'; + free(auth); + + return attempt; /* Only try once. */ +} + +static int create_session(BDRVHTTPState *s, const char *uri) +{ + const char *proxy; + + if (s->session) { + ne_session_destroy(s->session); + s->session = NULL; + } + + if (parse_uri(s, uri) != 0) { + fprintf(stderr, "HTTP: bad URI: %s\n", uri); + return -EINVAL; + } + + s->session = ne_session_create(s->uri.scheme, + s->uri.host, + s->uri.port); + if (!s->session) { + fprintf(stderr, "HTTP: couldn't create session.\n"); + return -EINVAL; + } + + ne_set_useragent(s->session, "qemu http-block"); + + ne_set_read_timeout(s->session, 10); + /* + * The connect timeout is essential to workaround a neon bug. Without + * it, neon calls blocking connect, and utterly fails to handle EINTR. + * Adding a timeout forces it onto the nonblocking connect path, using + * poll (and correctly handling EINTRs from poll). + * + * Incidentally, qemu is an excellent EINTR trial-by-fire for libraries. + */ + ne_set_connect_timeout(s->session, 10); + + proxy = getenv("HTTP_PROXY"); + if (!proxy) { + proxy = getenv("http_proxy"); + } + if (proxy) { + ne_uri_free(&s->proxy_uri); + if (ne_uri_parse(proxy, &s->proxy_uri) != 0) { + fprintf(stderr, "HTTP: bad proxy URL %s\n", proxy); + return -EINVAL; + } + ne_session_proxy(s->session, s->proxy_uri.host, s->proxy_uri.port); + if (s->proxy_uri.userinfo) { + ne_set_proxy_auth(s->session, auth_creds, s->proxy_uri.userinfo); + } + } + + if (s->uri.userinfo) { + ne_set_server_auth(s->session, auth_creds, s->uri.userinfo); + } + + return 0; +} + +static int get_size(BDRVHTTPState *s) +{ + int ret = 0; + ne_request *req = NULL; + const ne_status *st; + const char *hdr; + int err; + + req = ne_request_create(s->session, "HEAD", s->path); + if (!req) { + fprintf(stderr, "HTTP: couldn't create HEAD request\n"); + goto fail; + } + + err = ne_request_dispatch(req); + if (err != NE_OK) { + fprintf(stderr, "HTTP: couldn't dispatch HEAD request %d %s\n", + err, ne_get_error(s->session)); + goto fail; + } + + st = ne_get_status(req); + switch (st->klass) { + case 3: + hdr = ne_get_response_header(req, "Location"); + if (!hdr) { + fprintf(stderr, "HTTP: HEAD returned %d %s, but no Location\n", + st->code, st->reason_phrase); + goto fail; + } + + if (create_session(s, hdr) < 0) { + fprintf(stderr, "HTTP HEAD couldn't create session\n"); + goto fail; + } + + ret = get_size(s); + goto out; + + case 2: + hdr = ne_get_response_header(req, "Content-Length"); + if (!hdr) { + fprintf(stderr, "HTTP: HEAD had no Content-Length\n"); + goto fail; + } + + s->size = atoll(hdr); + if (s->size <= 0) { + fprintf(stderr, "HTTP: HEAD had invalid Content-Length %s\n", hdr); + goto fail; + } + + goto out; + + default: + fprintf(stderr, "HTTP: server returned code %d %s for HEAD\n", + st->code, st->reason_phrase); + goto fail; + } + +out: + if (req) { + ne_request_destroy(req); + } + return ret; + +fail: + ret = -EIO; + goto out; +} + +static void http_close(BlockDriverState *bs) +{ + BDRVHTTPState *s = bs->opaque; + + if (s->cache) { + qemu_free(s->cache); + } + + if (s->session) { + ne_session_destroy(s->session); + s->session = NULL; + } + + ne_uri_free(&s->proxy_uri); + + ne_uri_free(&s->uri); + if (s->path) { + ne_free(s->path); + s->path = NULL; + } +} + +static int http_open(BlockDriverState *bs, const char* filename, int flags) +{ + BDRVHTTPState *s = (BDRVHTTPState *)bs->opaque; + char *block_size; + + if ((flags & BDRV_O_CREAT)) { + goto fail; + } + + if (use_count == 0) { + use_count++; + if (ne_sock_init() != 0) { + fprintf(stderr, "HTTP: libneon init failed\n"); + goto fail; + } +#if HTTP_DEBUG != 0 + ne_debug_init(stderr, ~0); +#endif + } + + if (create_session(s, filename) < 0) { + goto fail; + } + + if (get_size(s) < 0) { + goto fail; + } + + block_size = getenv("HTTP_CACHE_BLOCK_SIZE"); + if (block_size) { + s->cache_block_size = atoi(block_size); + } else { + s->cache_block_size = DEFAULT_CACHE_BLOCK_SIZE; + } + if (s->cache_block_size > 0) { + if ((s->cache_block_size & 0x1ff) != 0) { + fprintf(stderr, "HTTP_CACHE_BLOCK_SIZE %d not a multiple of 512\n", + s->cache_block_size); + goto fail; + } + s->cache = qemu_malloc(s->cache_block_size); + } + + return 0; + +fail: + http_close(bs); + return -EINVAL; +} + +static int http_real_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors) +{ + BDRVHTTPState *s = bs->opaque; + + int64_t offset = sector_num * 512; + int len = nb_sectors * 512; + + ne_request *req = NULL; + const ne_status *st; + int ret = 0; + int err; + + assert(len > 0 && buf); + + req = ne_request_create(s->session, "GET", s->path); + if (!req) { + fprintf(stderr, "HTTP: couldn't create GET request\n"); + ret = -EIO; + goto out; + } + + ne_print_request_header(req, "Range", "bytes=%ld-%ld", + offset, offset + len - 1); + ne_add_request_header(req, "Accept-Ranges", "bytes"); + + err = ne_begin_request(req); + if (err != NE_OK) { + fprintf(stderr, "HTTP: couldn't begin GET request %d %s\n", + err, ne_get_error(s->session)); + ret = -EIO; + goto out; + } + + st = ne_get_status(req); + if (st->klass != 2) { + fprintf(stderr, "HTTP: GET failed: %d %s\n", + st->code, st->reason_phrase); + ret = -EIO; + goto out; + } + if (st->code == 200 && + strstr(ne_get_response_header(req, "Accept-Ranges"), "bytes")) { + fprintf(stderr, "HTTP: File too small\n"); + ret = -EIO; + goto out; + } + if (st->code != 206) { + fprintf(stderr, "HTTP: Server doesn't support Range requests\n"); + ret = -EIO; + goto out; + } + + if (ne_get_response_header(req, "Content-Range") == NULL) { + fprintf(stderr, "HTTP: GET response had no Content-Range header\n"); + ret = -EIO; + goto out; + } + + while ((err = ne_read_response_block(req, buf, len)) > 0) { + len -= err; + buf += err; + assert(len >= 0); + } + if (err != NE_OK) { + fprintf(stderr, "HTTP: couldn't read GET response %d %s\n", + err, ne_get_error(s->session)); + ret = -EIO; + goto out; + } + + err = ne_end_request(req); + if (err != NE_OK) { + fprintf(stderr, "HTTP: couldn't finish GET request %d %s\n", + err, ne_get_error(s->session)); + ret = -EIO; + goto out; + } + +out: + if (req) { + ne_request_destroy(req); + } + return ret; +} + +static int http_read(BlockDriverState *bs, int64_t sector_num, + uint8_t *buf, int nb_sectors) +{ + BDRVHTTPState *s = bs->opaque; + + if (s->cache_block_size == 0 || + nb_sectors * 512 > s->cache_block_size) { + return http_real_read(bs, sector_num, buf, nb_sectors); + } else { + uint64_t off = sector_num * 512; + int len = nb_sectors * 512; + + if (off < s->cache_offset || + off + len > s->cache_offset + s->cache_len) { + int ret; + + s->cache_offset = off; + s->cache_len = off + s->cache_block_size > s->size ? + s->size - off : s->cache_block_size; + + ret = http_real_read(bs, sector_num, s->cache, s->cache_len / 512); + if (ret != 0) { + s->cache_len = 0; + return ret; + } + } + + memcpy(buf, s->cache + (off - s->cache_offset), len); + + return 0; + } +} + +static int64_t http_getlength(BlockDriverState *bs) +{ + BDRVHTTPState *s = bs->opaque; + + return s->size; +} + +BlockDriver bdrv_http = { + "http", + sizeof (BDRVHTTPState), + NULL, /* no probe for protocols */ + http_open, + http_read, + NULL, /* write */ + http_close, + .bdrv_getlength = http_getlength, + .protocol_name = "http", +};