[Qemu-devel] [PATCH] Add HTTP protocol using curl

[Qemu-devel] [PATCH] Add HTTP protocol using curl

[Alexander Graf]

Currently Qemu can read from posix I/O and NBD. This patch adds a
third protocol to the game: HTTP.

In certain situations it can be useful to access HTTP data directly,
for example if you want to try out an http provided OS image, but
don't know if you want to download it yet.

Using this patch you can now try it on on the fly. Just use it like:

qemu -cdrom http://host/path/my.iso

In order to not reinvent the wheel, this patch uses libcurl.

Warning:

With stock qemu I do get segmentation faults accessing http backed
files, with kvm-userspace's IO threads I don't. I haven't tracked
down why exactly that happens though.

Signed-off-by: Alexander Graf <agraf@suse.de>
---
 Makefile        |    6 ++
 Makefile.target |    2 +-
 block-http.c    |  140 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 block.c         |    3 +
 block.h         |    1 +
 configure       |   26 ++++++++++-
 6 files changed, 176 insertions(+), 2 deletions(-)
 create mode 100644 block-http.c

diff --git a/Makefile b/Makefile
index 633774e..9481c1a 100644
--- a/Makefile
+++ b/Makefile
@@ -64,6 +64,10 @@ endif
 BLOCK_OBJS += block-raw-posix.o
 endif
 
+ifdef CONFIG_CURL
+BLOCK_OBJS += block-http.o
+endif
+
 ######################################################################
 # libqemu_common.a: Target independent part of system emulation. The
 # long term path is to suppress *all* target specific code in case of
@@ -166,6 +170,8 @@ endif
 
 LIBS+=$(VDE_LIBS)
 
+LIBS+=$(CURL_LIBS)
+
 cocoa.o: cocoa.m
 
 keymaps.o: keymaps.c keymaps.h
diff --git a/Makefile.target b/Makefile.target
index b32d1af..6062efb 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -726,7 +726,7 @@ endif
 
 vl.o: qemu-options.h
 
-$(QEMU_PROG): LIBS += $(SDL_LIBS) $(COCOA_LIBS) $(CURSES_LIBS) $(BRLAPI_LIBS) $(VDE_LIBS)
+$(QEMU_PROG): LIBS += $(SDL_LIBS) $(COCOA_LIBS) $(CURSES_LIBS) $(BRLAPI_LIBS) $(VDE_LIBS) $(CURL_LIBS)
 
 $(QEMU_PROG): $(OBJS) ../libqemu_common.a libqemu.a
 	$(LINK)
diff --git a/block-http.c b/block-http.c
new file mode 100644
index 0000000..7820f0b
--- /dev/null
+++ b/block-http.c
@@ -0,0 +1,140 @@
+/*
+ * QEMU Block driver for HTTP images
+ *
+ * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu-common.h"
+#include "block_int.h"
+#include <curl/curl.h>
+
+// #define DEBUG
+
+#ifdef DEBUG
+#define printd printf
+#else
+#define printd(x, ...)
+#endif
+
+#define SECTOR_SIZE 512
+
+typedef struct BDRVHTTPState {
+    CURL *curl;
+    uint8_t *buf;
+    int64_t len;
+    char range[128];
+} BDRVHTTPState;
+
+static size_t http_read_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
+{
+    BDRVHTTPState *s = ((BlockDriverState*)opaque)->opaque;
+    size_t realsize = size * nmemb;
+
+    memcpy(s->buf, ptr, realsize);
+    s->buf += realsize;
+
+    printd("CURL: Just read %d bytes\n", realsize);
+    return realsize;
+}
+
+static int http_open(BlockDriverState *bs, const char *filename, int flags)
+{
+    BDRVHTTPState *s = bs->opaque;
+    double d;
+    static int inited = 0;
+
+    if (!inited) {
+        curl_global_init(CURL_GLOBAL_ALL);
+        inited = 1;
+    }
+
+    printd("CURL: Opening %s\n", filename);
+    s->curl = curl_easy_init();
+    if (!s->curl)
+        goto out_noclean;
+    curl_easy_setopt(s->curl, CURLOPT_URL, filename);
+    curl_easy_setopt(s->curl, CURLOPT_TIMEOUT, 5);
+    curl_easy_setopt(s->curl, CURLOPT_WRITEFUNCTION, http_read_cb);
+    curl_easy_setopt(s->curl, CURLOPT_WRITEDATA, (void *)bs);
+
+    // Get file size
+
+    curl_easy_setopt(s->curl, CURLOPT_NOBODY, 1);
+    if (curl_easy_perform(s->curl))
+        goto out;
+    curl_easy_getinfo(s->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &d);
+    curl_easy_setopt(s->curl, CURLOPT_NOBODY, 0);
+    if (!d)
+        goto out;
+    printd("CURL: Size = %lld\n", (long long)d);
+    s->len = (int64_t)d;
+
+    return 0;
+
+out:
+    printd("CURL: Error opening file\n");
+    curl_easy_cleanup(s->curl);
+out_noclean:
+    return -EINVAL;
+}
+
+static int http_read(BlockDriverState *bs, int64_t sector_num,
+                    uint8_t *buf, int nb_sectors)
+{
+    BDRVHTTPState *s = bs->opaque;
+    CURLcode res;
+    long long start = sector_num * SECTOR_SIZE;
+    long long end = start + (nb_sectors * SECTOR_SIZE) - 1;
+
+    s->buf = buf;
+
+    // seek(sector_num * SECTOR_SIZE);
+    sprintf(s->range, "%lld-%lld", start, end);
+    printd("CURL: Reading %lld at %lld (%s)\n", (nb_sectors * SECTOR_SIZE), start, s->range);
+    curl_easy_setopt(s->curl, CURLOPT_RANGE, s->range);
+
+    // read(nb_sectors * SECTOR_SIZE); -> buf
+    res = curl_easy_perform(s->curl);
+
+    return 0;
+}
+
+static void http_close(BlockDriverState *bs)
+{
+    BDRVHTTPState *s = bs->opaque;
+    printd("CURL: Close\n");
+    curl_easy_cleanup(s->curl);
+}
+
+static int64_t http_getlength(BlockDriverState *bs)
+{
+    BDRVHTTPState *s = bs->opaque;
+    return s->len;
+}
+
+BlockDriver bdrv_http = {
+    .format_name	= "http",
+    .protocol_name	= "http",
+    .instance_size	= sizeof(BDRVHTTPState),
+    .bdrv_open		= http_open,
+    .bdrv_read		= http_read,
+    .bdrv_close		= http_close,
+    .bdrv_getlength	= http_getlength,
+};
diff --git a/block.c b/block.c
index dd40b56..ca44e37 100644
--- a/block.c
+++ b/block.c
@@ -1539,6 +1539,9 @@ void bdrv_init(void)
     bdrv_register(&bdrv_qcow2);
     bdrv_register(&bdrv_parallels);
     bdrv_register(&bdrv_nbd);
+#ifdef CONFIG_CURL
+    bdrv_register(&bdrv_http);
+#endif
 }
 
 void aio_pool_init(AIOPool *pool, int aiocb_size,
diff --git a/block.h b/block.h
index f1919b6..6cadead 100644
--- a/block.h
+++ b/block.h
@@ -20,6 +20,7 @@ extern BlockDriver bdrv_vvfat;
 extern BlockDriver bdrv_qcow2;
 extern BlockDriver bdrv_parallels;
 extern BlockDriver bdrv_nbd;
+extern BlockDriver bdrv_http;
 
 typedef struct BlockDriverInfo {
     /* in bytes, 0 if irrelevant */
diff --git a/configure b/configure
index 15b7856..bc4d31b 100755
--- a/configure
+++ b/configure
@@ -180,6 +180,7 @@ bsd_user="no"
 build_docs="no"
 uname_release=""
 curses="yes"
+curl="yes"
 aio="yes"
 nptl="yes"
 mixemu="no"
@@ -460,6 +461,8 @@ for opt do
   ;;
   --disable-curses) curses="no"
   ;;
+  --disable-curl) curl="no"
+  ;;
   --disable-nptl) nptl="no"
   ;;
   --enable-mixemu) mixemu="yes"
@@ -571,7 +574,8 @@ echo "  --enable-mixemu          enable mixer emulation"
 echo "  --disable-brlapi         disable BrlAPI"
 echo "  --disable-vnc-tls        disable TLS encryption for VNC server"
 echo "  --disable-vnc-sasl       disable SASL encryption for VNC server"
 echo "  --disable-curses         disable curses output"
+echo "  --disable-curl           disable curl connectivity"
 echo "  --disable-bluez          disable bluez stack connectivity"
 echo "  --disable-kvm            disable KVM acceleration support"
 echo "  --disable-nptl           disable usermode NPTL support"
@@ -1010,6 +1013,21 @@ EOF
 fi # test "$curses"
 
 ##########################################
+# curl probe
+
+if test "$curl" = "yes" ; then
+  curl=no
+  cat > $TMPC << EOF
+#include <curl/curl.h>
+int main(void) { return curl_easy_init(); }
+EOF
+  curl_libs=`curl-config --libs`
+  if $cc $ARCH_CFLAGS $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then
+    curl=yes
+  fi
+fi # test "$curl"
+
+##########################################
 # bluez support probe
 if test "$bluez" = "yes" ; then
   `pkg-config bluez` || bluez="no"
@@ -1193,6 +1211,7 @@ if test "$sdl" != "no" ; then
     echo "SDL static link   $sdl_static"
 fi
 echo "curses support    $curses"
+echo "curl support      $curl"
 echo "mingw32 support   $mingw32"
 echo "Audio drivers     $audio_drv_list"
 echo "Extra audio cards $audio_card_list"
@@ -1501,6 +1520,11 @@ if test "$curses" = "yes" ; then
   echo "CONFIG_CURSES=yes" >> $config_mak
   echo "CURSES_LIBS=-lcurses" >> $config_mak
 fi
+if test "$curl" = "yes" ; then
+  echo "CONFIG_CURL=yes" >> $config_mak
+  echo "CURL_LIBS=$curl_libs" >> $config_mak
+  echo "#define CONFIG_CURL 1" >> $config_h
+fi
 if test "$brlapi" = "yes" ; then
   echo "CONFIG_BRLAPI=yes" >> $config_mak
   echo "#define CONFIG_BRLAPI 1" >> $config_h
-- 
1.6.0.2

[Qemu-devel] Re: [PATCH] Add HTTP protocol using curl

[Anthony Liguori]

Alexander Graf wrote:
> Currently Qemu can read from posix I/O and NBD. This patch adds a
> third protocol to the game: HTTP.
>
> In certain situations it can be useful to access HTTP data directly,
> for example if you want to try out an http provided OS image, but
> don't know if you want to download it yet.
>
> Using this patch you can now try it on on the fly. Just use it like:
>
> qemu -cdrom http://host/path/my.iso
>
> In order to not reinvent the wheel, this patch uses libcurl.
>   

I have one of these too that reinvents the wheel.  The main reason was 
to enable bdrv_aio_* implementations.

I'm happy to merge a version that uses libcurl but I don't want to merge 
it unless it's implementing bdrv_aio_.  As far as I'm concerned, 
backends implementing bdrv_read/bdrv_write is deprecated.

Regards,

Anthony Liguori

[Qemu-devel] Re: [PATCH] Add HTTP protocol using curl

[Alexander Graf]

On 09.04.2009, at 23:37, Anthony Liguori wrote:

> Alexander Graf wrote:
>> Currently Qemu can read from posix I/O and NBD. This patch adds a
>> third protocol to the game: HTTP.
>>
>> In certain situations it can be useful to access HTTP data directly,
>> for example if you want to try out an http provided OS image, but
>> don't know if you want to download it yet.
>>
>> Using this patch you can now try it on on the fly. Just use it like:
>>
>> qemu -cdrom http://host/path/my.iso
>>
>> In order to not reinvent the wheel, this patch uses libcurl.
>>
>
> I have one of these too that reinvents the wheel.  The main reason  
> was to enable bdrv_aio_* implementations.
>
> I'm happy to merge a version that uses libcurl but I don't want to  
> merge it unless it's implementing bdrv_aio_.  As far as I'm  
> concerned, backends implementing bdrv_read/bdrv_write is deprecated.

I agree. Libcurl provides means to download asynchronously as well, so  
AIO is possible with it. I didn't implement it because I needed  
something quick before I leave for 3 weeks of vacation (today ;-)).

So in case you don't merge it, that's fine. I wanted to improve it  
next month anyways.

Alex

Re: [Qemu-devel] [PATCH] Add HTTP protocol using curl

[Blue Swirl]

On 4/9/09, Alexander Graf <agraf@suse.de> wrote:
> Currently Qemu can read from posix I/O and NBD. This patch adds a
>  third protocol to the game: HTTP.
>
>  In certain situations it can be useful to access HTTP data directly,
>  for example if you want to try out an http provided OS image, but
>  don't know if you want to download it yet.
>
>  Using this patch you can now try it on on the fly. Just use it like:
>
>  qemu -cdrom http://host/path/my.iso
>
>  In order to not reinvent the wheel, this patch uses libcurl.
>
>  Warning:
>
>  With stock qemu I do get segmentation faults accessing http backed
>  files, with kvm-userspace's IO threads I don't. I haven't tracked
>  down why exactly that happens though.
>
>  Signed-off-by: Alexander Graf <agraf@suse.de>

The docs should be updated as well.

>  +    sprintf(s->range, "%lld-%lld", start, end);

Please use snprintf.