[Qemu-devel] [PATCH] Patches from PyQemu project

[Qemu-devel] [PATCH] Patches from PyQemu project

[Maria Zabolotnaya]

[-- Attachment #1.1: Type: text/plain, Size: 1069 bytes --]

Please see previous message for general PyQemu project description. Here are
the patches developed during the project:

1-qemu-override-mtype.patch
Add -mtype command line option to let override ARM MTYPE passed to the
kernel (useful for initial testing, prototyping, and debugging of new
machine).

2-qemu-mplugin.patch
Add -mplugin switch to allow loading of shared library and registering a
machine declared in it.

3-qemu-build-so.patch
Build QEMU as a shared library.

4-qemu-no-statics.patch
Remove static declaration from some QEMU symbols, so they were exported from
shared library.

5-qemu-gccxml-friendly.patch
This is auxiliary patch to make QEMY header C++ friendly, which is required
by gccxml, which in turn is required by ctypes utility h2xml to
automatically generate Python interface files from C headers.

6-qemu-extra-sdstate-accessors.patch
Few extra accessors for SDState structure (as was required to develop
emulation of ASIC3 SD controller). Alternative approach would be to make the
structure itself public.



Best regards,
Maria Zabolotnaya.

[-- Attachment #1.2: Type: text/html, Size: 1146 bytes --]

[-- Attachment #2: 1-qemu-override-mtype.patch --]
[-- Type: text/x-diff, Size: 2839 bytes --]

Index: vl.c
===================================================================
RCS file: /sources/qemu/qemu/vl.c,v
retrieving revision 1.323
diff -u -r1.323 vl.c
--- vl.c	29 Jul 2007 17:57:25 -0000	1.323
+++ vl.c	19 Aug 2007 01:31:31 -0000
@@ -196,6 +197,7 @@
 const char *option_rom[MAX_OPTION_ROMS];
 int nb_option_roms;
 int semihosting_enabled = 0;
+int override_mtype = 0;
 int autostart = 1;
 #ifdef TARGET_ARM
 int old_param = 0;
@@ -6590,6 +6592,9 @@
            "\n"
            "Standard options:\n"
            "-M machine      select emulated machine (-M ? for list)\n"
+#ifdef TARGET_ARM
+           "-mtype machid   set ARM machine type for generic machines\n"
+#endif
            "-cpu cpu        select CPU (-cpu ? for list)\n"
            "-fda/-fdb file  use 'file' as floppy disk 0/1 image\n"
            "-hda/-hdb file  use 'file' as IDE hard disk 0/1 image\n"
@@ -6805,6 +6811,7 @@
     QEMU_OPTION_name,
     QEMU_OPTION_prom_env,
     QEMU_OPTION_old_param,
+    QEMU_OPTION_mtype,
 };
 
 typedef struct QEMUOption {
@@ -6901,6 +6909,7 @@
     { "option-rom", HAS_ARG, QEMU_OPTION_option_rom },
 #if defined(TARGET_ARM) || defined(TARGET_M68K)
     { "semihosting", 0, QEMU_OPTION_semihosting },
+    { "mtype", HAS_ARG, QEMU_OPTION_mtype },
 #endif
     { "name", HAS_ARG, QEMU_OPTION_name },
 #if defined(TARGET_SPARC)
@@ -7684,6 +7694,12 @@
                 nb_prom_envs++;
                 break;
 #endif
+            case QEMU_OPTION_mtype:
+                {
+                    const char *p = optarg;
+                    override_mtype = strtol(p, (char **)&p, 0);
+                }
+                break;
 #ifdef TARGET_ARM
             case QEMU_OPTION_old_param:
                 old_param = 1;
Index: vl.h
===================================================================
RCS file: /sources/qemu/qemu/vl.h,v
retrieving revision 1.260
diff -u -r1.260 vl.h
--- vl.h	16 Aug 2007 19:56:27 -0000	1.260
+++ vl.h	19 Aug 2007 01:31:31 -0000
@@ -171,6 +171,8 @@
 extern const char *option_rom[MAX_OPTION_ROMS];
 extern int nb_option_roms;
 
+extern int override_mtype;
+
 #ifdef TARGET_SPARC
 #define MAX_PROM_ENVS 128
 extern const char *prom_envs[MAX_PROM_ENVS];
Index: hw/arm_boot.c
===================================================================
RCS file: /sources/qemu/qemu/hw/arm_boot.c,v
retrieving revision 1.8
diff -u -r1.8 arm_boot.c
--- hw/arm_boot.c	27 Jul 2007 22:08:46 -0000	1.8
+++ hw/arm_boot.c	19 Aug 2007 01:31:31 -0000
@@ -169,6 +169,8 @@
         env->kernel_filename = kernel_filename;
         env->kernel_cmdline = kernel_cmdline;
         env->initrd_filename = initrd_filename;
+        if (override_mtype)
+            board_id = override_mtype;
         env->board_id = board_id;
         env->loader_start = loader_start;
         qemu_register_reset(main_cpu_reset, env);

[-- Attachment #3: 2-qemu-mplugin.patch --]
[-- Type: text/x-diff, Size: 2825 bytes --]

Index: osdep.h
===================================================================
RCS file: /sources/qemu/qemu/osdep.h,v
retrieving revision 1.10
diff -u -r1.10 osdep.h
--- osdep.h	7 Jun 2007 23:09:47 -0000	1.10
+++ osdep.h	19 Aug 2007 01:31:30 -0000
@@ -28,4 +28,14 @@
 #define qemu_gettimeofday(tp) gettimeofday(tp, NULL);
 #endif /* !_WIN32 */
 
+#ifdef _WIN32
+#define qemu_dlopen(name, flags) LoadLibrary(name)
+#define qemu_dlsym(handle, name) ((void*)GetProcAddress(handle, name))
+#define qemu_dlerror() "DLL load error"
+#else
+#define qemu_dlopen(name, flags) dlopen(name, flags)
+#define qemu_dlsym(handle, name) dlsym(handle, name)
+#define qemu_dlerror() dlerror()
+#endif /* !_WIN32 */
+
 #endif
Index: vl.c
===================================================================
RCS file: /sources/qemu/qemu/vl.c,v
retrieving revision 1.323
diff -u -r1.323 vl.c
--- vl.c	29 Jul 2007 17:57:25 -0000	1.323
+++ vl.c	19 Aug 2007 01:31:31 -0000
@@ -42,6 +42,7 @@
 #include <netinet/in.h>
 #include <dirent.h>
 #include <netdb.h>
+#include <dlfcn.h>
 #ifdef _BSD
 #include <sys/stat.h>
 #ifndef __APPLE__
@@ -6712,6 +6713,7 @@
 #ifdef TARGET_SPARC
            "-prom-env variable=value  set OpenBIOS nvram variables\n"
 #endif
+           "-mplugin machine.so  load machine plugin\n"
            "\n"
            "During emulation, the following keys are useful:\n"
            "ctrl-alt-f      toggle full screen\n"
@@ -6810,6 +6812,7 @@
     QEMU_OPTION_prom_env,
     QEMU_OPTION_old_param,
     QEMU_OPTION_mtype,
+    QEMU_OPTION_machine_plugin,
 };
 
 typedef struct QEMUOption {
@@ -6915,6 +6918,7 @@
 #if defined(TARGET_ARM)
     { "old-param", 0, QEMU_OPTION_old_param },
 #endif
+    { "mplugin", HAS_ARG, QEMU_OPTION_machine_plugin },
     { NULL },
 };
 
@@ -7690,6 +7694,27 @@
                 nb_prom_envs++;
                 break;
 #endif
+            case QEMU_OPTION_machine_plugin:
+                {
+                    void *handle;
+                    QEMUMachine *machine;
+ 
+                    handle = qemu_dlopen(optarg, RTLD_NOW | RTLD_GLOBAL);
+                    if (!handle) {
+                        fprintf(stderr, "Cannot load plugin: %s\n", qemu_dlerror());
+                        exit(1);
+                    }
+                    machine = qemu_dlsym(handle, "machine");
+                    if (!machine) {
+                        fprintf(stderr, "Cannot find machine in plugin\n");
+                        exit(1);
+                    }
+                    if (qemu_register_machine(machine)) {
+                        fprintf(stderr, "Cannot register machine\n");
+                        exit(1);
+                    }
+                }
+                break;
             case QEMU_OPTION_mtype:
                 {
                     const char *p = optarg;

[-- Attachment #4: 3-qemu-build-so.patch --]
[-- Type: text/x-diff, Size: 1045 bytes --]

Index: Makefile.target
===================================================================
RCS file: /sources/qemu/qemu/Makefile.target,v
retrieving revision 1.191
diff -u -r1.191 Makefile.target
--- Makefile.target	31 Jul 2007 23:44:21 -0000	1.191
+++ Makefile.target	19 Aug 2007 01:31:30 -0000
@@ -519,6 +519,12 @@
 VL_LDFLAGS+=-p
 endif
 
+ifdef CONFIG_WIN32
+VL_LIBS+=-Wl,--out-implib,libqemu-system-$(TARGET_ARCH2).dll.a
+else
+VL_LIBS+=-ldl
+endif
+
 ifeq ($(ARCH),ia64)
 VL_LDFLAGS+=-Wl,-G0 -Wl,-T,$(SRC_PATH)/ia64.ld
 endif
@@ -535,7 +541,10 @@
 endif
 
 $(QEMU_SYSTEM): $(VL_OBJS) libqemu.a
-	$(CC) $(VL_LDFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS) $(SDL_LIBS) $(COCOA_LIBS) $(VL_LIBS)
+	$(CC) -Wl,--export-dynamic $(VL_LDFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS) $(SDL_LIBS) $(COCOA_LIBS) $(VL_LIBS)
+
+qemu-system-$(TARGET_ARCH2).so: $(VL_OBJS) libqemu.a
+	$(CC) --shared -Wl,--export-dynamic $(VL_LDFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS) $(SDL_LIBS) $(COCOA_LIBS) $(VL_LIBS)
 
 cocoa.o: cocoa.m
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(BASE_CFLAGS) -c -o $@ $<

[-- Attachment #5: 4-qemu-no-statics.patch --]
[-- Type: text/x-diff, Size: 1752 bytes --]

Index: vl.c
===================================================================
RCS file: /sources/qemu/qemu/vl.c,v
retrieving revision 1.323
diff -u -r1.323 vl.c
--- vl.c	29 Jul 2007 17:57:25 -0000	1.323
+++ vl.c	19 Aug 2007 01:31:31 -0000
@@ -146,7 +147,7 @@
 /* point to the block driver where the snapshots are managed */
 BlockDriverState *bs_snapshots;
 int vga_ram_size;
-static DisplayState display_state;
+DisplayState display_state;
 int nographic;
 const char* keyboard_layout = NULL;
 int64_t ticks_per_sec;
@@ -910,7 +912,7 @@
     }
 }
 
-static void init_timers(void)
+void init_timers(void)
 {
     init_get_clock();
     ticks_per_sec = QEMU_TIMER_BASE;
@@ -943,7 +945,7 @@
     }
 }
 
-static void timer_save(QEMUFile *f, void *opaque)
+void timer_save(QEMUFile *f, void *opaque)
 {
     if (cpu_ticks_enabled) {
         hw_error("cannot save state if virtual timers are running");
@@ -953,7 +955,7 @@
     qemu_put_be64s(f, &cpu_clock_offset);
 }
 
-static int timer_load(QEMUFile *f, void *opaque, int version_id)
+int timer_load(QEMUFile *f, void *opaque, int version_id)
 {
     if (version_id != 1 && version_id != 2)
         return -EINVAL;
@@ -1063,7 +1065,7 @@
 
 #endif /* !defined(_WIN32) */
 
-static void init_timer_alarm(void)
+void init_timer_alarm(void)
 {
 #ifdef _WIN32
     {
@@ -6016,7 +6018,7 @@
     inflateEnd(&s->zstream);
 }
 
-static void ram_save(QEMUFile *f, void *opaque)
+void ram_save(QEMUFile *f, void *opaque)
 {
     int i;
     RamCompressState s1, *s = &s1;
@@ -6060,7 +6062,7 @@
     ram_compress_close(s);
 }
 
-static int ram_load(QEMUFile *f, void *opaque, int version_id)
+int ram_load(QEMUFile *f, void *opaque, int version_id)
 {
     RamDecompressState s1, *s = &s1;
     uint8_t buf[10];

[-- Attachment #6: 5-qemu-gccxml-friendly.patch --]
[-- Type: text/x-diff, Size: 1397 bytes --]

Index: cpu-all.h
===================================================================
RCS file: /sources/qemu/qemu/cpu-all.h,v
retrieving revision 1.74
diff -u -r1.74 cpu-all.h
--- cpu-all.h	29 Jul 2007 17:57:24 -0000	1.74
+++ cpu-all.h	19 Aug 2007 01:31:30 -0000
@@ -416,7 +416,7 @@
 {
     uint32_t a,b;
     a = ldl_be_p(ptr);
-    b = ldl_be_p(ptr+4);
+    b = ldl_be_p((char*)ptr+4);
     return (((uint64_t)a<<32)|b);
 }
 
@@ -453,7 +453,7 @@
 static inline void stq_be_p(void *ptr, uint64_t v)
 {
     stl_be_p(ptr, v >> 32);
-    stl_be_p(ptr + 4, v);
+    stl_be_p((char*)ptr + 4, v);
 }
 
 /* float access */
@@ -482,7 +482,7 @@
 {
     CPU_DoubleU u;
     u.l.upper = ldl_be_p(ptr);
-    u.l.lower = ldl_be_p(ptr + 4);
+    u.l.lower = ldl_be_p((char*)ptr + 4);
     return u.d;
 }
 
@@ -491,7 +491,7 @@
     CPU_DoubleU u;
     u.d = v;
     stl_be_p(ptr, u.l.upper);
-    stl_be_p(ptr + 4, u.l.lower);
+    stl_be_p((char*)ptr + 4, u.l.lower);
 }
 
 #else
Index: audio/audio.h
===================================================================
RCS file: /sources/qemu/qemu/audio/audio.h,v
retrieving revision 1.9
diff -u -r1.9 audio.h
--- audio/audio.h	17 Feb 2007 22:19:29 -0000	1.9
+++ audio/audio.h	19 Aug 2007 01:31:31 -0000
@@ -144,7 +144,7 @@
 
 static inline void *advance (void *p, int incr)
 {
-    uint8_t *d = p;
+    uint8_t *d = (uint8_t*)p;
     return (d + incr);
 }
 

[-- Attachment #7: 6-qemu-extra-sdstate-accessors.patch --]
[-- Type: text/x-diff, Size: 964 bytes --]

Index: hw/sd.c
===================================================================
RCS file: /sources/qemu/qemu/hw/sd.c,v
retrieving revision 1.2
diff -u -r1.2 sd.c
--- hw/sd.c	30 Jul 2007 23:54:51 -0000	1.2
+++ hw/sd.c	19 Aug 2007 01:31:31 -0000
@@ -1507,3 +1507,13 @@
 {
     return sd->state == sd_sendingdata_state;
 }
+
+int sd_get_card_status(SDState *sd)
+{
+    return sd->card_status;
+}
+        
+int sd_get_state(SDState *sd)
+{
+    return sd->state;
+}
Index: hw/sd.h
===================================================================
RCS file: /sources/qemu/qemu/hw/sd.h,v
retrieving revision 1.2
diff -u -r1.2 sd.h
--- hw/sd.h	30 Jul 2007 23:54:51 -0000	1.2
+++ hw/sd.h	19 Aug 2007 01:31:31 -0000
@@ -78,5 +78,7 @@
                void (*readonly_cb)(void *, int),
                void (*inserted_cb)(void *, int));
 int sd_data_ready(SDState *sd);
+int sd_get_card_status(SDState *sd); 
+int sd_get_state(SDState *sd); 
 
 #endif	/* __hw_sd_h */

Re: [Qemu-devel] [PATCH] Patches from PyQemu project

[Blue Swirl]

On 9/2/07, Maria Zabolotnaya <maria.zabolotnaya@gmail.com> wrote:
> 2-qemu-mplugin.patch
> Add -mplugin switch to allow loading of shared library and registering a
> machine declared in it.

Sorry to ruin your GSoC project, but the plugin system was discussed
last year, please see this thread:
http://thread.gmane.org/gmane.comp.emulators.qemu/14341/focus=14473

> 4-qemu-no-statics.patch
> Remove static declaration from some QEMU symbols, so they were exported from shared
> library.

I don't think this API is worth supporting in the future.

Re: [Qemu-devel] [PATCH] Patches from PyQemu project

[Anthony Liguori]

On Mon, 2007-09-03 at 18:41 +0300, Blue Swirl wrote:
> On 9/2/07, Maria Zabolotnaya <maria.zabolotnaya@gmail.com> wrote:
> > 2-qemu-mplugin.patch
> > Add -mplugin switch to allow loading of shared library and registering a
> > machine declared in it.
> 
> Sorry to ruin your GSoC project, but the plugin system was discussed
> last year, please see this thread:
> http://thread.gmane.org/gmane.comp.emulators.qemu/14341/focus=14473

I've always agreed that allowing plugins was not a good idea.  However,
I had a different thought recently.

While I don't think there's much of a reason to allow plugins for QEMU,
it would be interesting to make some of QEMU's device emulation into
more a of a library that could be used by other programs.

With things like KVM making it relatively simple to do CPU emulation, if
QEMU's device emulation was available as a library (even a GPL library),
it would be pretty easy to do interesting things without forking QEMU
which is what everyone seems to be doing these days.

My initial thought is to make the libraries at the individual device
level.

Regards,

Anthony Liguori

> > 4-qemu-no-statics.patch
> > Remove static declaration from some QEMU symbols, so they were exported from shared
> > library.
> 
> I don't think this API is worth supporting in the future.
> 
> 

[Qemu-devel] Re: qemu device emulation libraries (was [PATCH] Patches from the PyQemu project)

[Hollis Blanchard]

On Mon, 03 Sep 2007 15:44:13 -0500, Anthony Liguori wrote:

> On Mon, 2007-09-03 at 18:41 +0300, Blue Swirl wrote:
>> On 9/2/07, Maria Zabolotnaya <maria.zabolotnaya@gmail.com> wrote:
>> > 2-qemu-mplugin.patch
>> > Add -mplugin switch to allow loading of shared library and registering a
>> > machine declared in it.
>> 
>> Sorry to ruin your GSoC project, but the plugin system was discussed
>> last year, please see this thread:
>> http://thread.gmane.org/gmane.comp.emulators.qemu/14341/focus=14473
> 
> I've always agreed that allowing plugins was not a good idea.  However,
> I had a different thought recently.
> 
> While I don't think there's much of a reason to allow plugins for QEMU,
> it would be interesting to make some of QEMU's device emulation into
> more a of a library that could be used by other programs.
> 
> With things like KVM making it relatively simple to do CPU emulation, if
> QEMU's device emulation was available as a library (even a GPL library),
> it would be pretty easy to do interesting things without forking QEMU
> which is what everyone seems to be doing these days.
> 
> My initial thought is to make the libraries at the individual device
> level.

This could be very valuable when thinking about running qemu *on* embedded
systems with constrained memory and processing power, which is exactly what
the KVM for embedded PowerPC project is considering. In that scenario,
being able to strip out all unnecessary functionality (especially
including devices known to be irrelevant) becomes very important.

Strictly speaking, creating a library isn't required to meet that goal,
but it will enforce the modularity we need, and IMHO that is just good
coding practice.

(I'm not even talking about Anthony's points about avoiding qemu forks,
which I also agree with.)

-- 
Hollis Blanchard
IBM Linux Technology Center

[Qemu-devel] Re: [PATCH] Patches from PyQemu project

[Brian Johnson]

Anthony Liguori wrote:
> On Mon, 2007-09-03 at 18:41 +0300, Blue Swirl wrote:
>> Sorry to ruin your GSoC project, but the plugin system was discussed
>> last year, please see this thread:
>> http://thread.gmane.org/gmane.comp.emulators.qemu/14341/focus=14473
> 
> I've always agreed that allowing plugins was not a good idea.  However,
> I had a different thought recently.
> 
> While I don't think there's much of a reason to allow plugins for QEMU,
> it would be interesting to make some of QEMU's device emulation into
> more a of a library that could be used by other programs.
> 
> With things like KVM making it relatively simple to do CPU emulation, if
> QEMU's device emulation was available as a library (even a GPL library),
> it would be pretty easy to do interesting things without forking QEMU
> which is what everyone seems to be doing these days.

Yes -- it would be valuable to have QEMU's devices available as a 
separate library(ies), with a simple, well-defined API.  Some of us use 
other CPU emulators (yes, yes, I know we should emulate all 
architectures under qemu, but that's not always easy :) and would love 
to use qemu's device emulations.

> My initial thought is to make the libraries at the individual device
> level.

It would be good to have a general mechanism for bus providers, 
interrupts, APICs, chipsets, etc. as well, so we could emulate fancier 
architectures than a simple PC (or simple Sparc/MIPS/ARM/etc. box.)  For 
instance, I'd like to emulate multiple PCIe host bridges, each with an 
APIC and multiple cards, which might contain PCI-to-PCI bridges.  And 
I'd like to emulate NUMA systems with many memory controllers and a 
complex memory map, with multiple sets of chipset registers.  I don't 
expect qemu to do this off the shelf, but I'd like to avoid hardcoding 
PC assumptions into the device libraries, so I can code the fancy 
machines myself and use the I/O as-is.

Any more thoughts on how to structure the libraries?

Brian J. Johnson

Re: [Qemu-devel] Re: qemu device emulation libraries (was [PATCH] Patches from the PyQemu project)

[Paul Brook]

> This could be very valuable when thinking about running qemu *on* embedded
> systems with constrained memory and processing power, which is exactly what
> the KVM for embedded PowerPC project is considering. In that scenario,
> being able to strip out all unnecessary functionality (especially
> including devices known to be irrelevant) becomes very important.

If you care about memory overhead the last thing you want is to be loading 
loads of bitty little shared libraries. You want to build a single binary 
with just the features you need.

Paul

Re: [Qemu-devel] Re: qemu device emulation libraries (was [PATCH] Patches from the PyQemu project)

[Hollis Blanchard]

On Tue, 2007-09-04 at 21:04 +0100, Paul Brook wrote:
> > This could be very valuable when thinking about running qemu *on* embedded
> > systems with constrained memory and processing power, which is exactly what
> > the KVM for embedded PowerPC project is considering. In that scenario,
> > being able to strip out all unnecessary functionality (especially
> > including devices known to be irrelevant) becomes very important.
> 
> If you care about memory overhead the last thing you want is to be loading 
> loads of bitty little shared libraries. You want to build a single binary 
> with just the features you need.

Hmm, that's a good point. Is that something you think can reasonably be
accomplished with qemu today?

-- 
Hollis Blanchard
IBM Linux Technology Center

Re: [Qemu-devel] Re: qemu device emulation libraries (was [PATCH] Patches from the PyQemu project)

[Paul Brook]

On Tuesday 04 September 2007, Hollis Blanchard wrote:
> On Tue, 2007-09-04 at 21:04 +0100, Paul Brook wrote:
> > > This could be very valuable when thinking about running qemu *on*
> > > embedded systems with constrained memory and processing power, which is
> > > exactly what the KVM for embedded PowerPC project is considering. In
> > > that scenario, being able to strip out all unnecessary functionality
> > > (especially including devices known to be irrelevant) becomes very
> > > important.
> >
> > If you care about memory overhead the last thing you want is to be
> > loading loads of bitty little shared libraries. You want to build a
> > single binary with just the features you need.
>
> Hmm, that's a good point. Is that something you think can reasonably be
> accomplished with qemu today?

It should be fairly easy.

Paul

Re: [Qemu-devel] Re: qemu device emulation libraries (was [PATCH] Patches from the PyQemu project)

[Anthony Liguori]

On Tue, 2007-09-04 at 21:04 +0100, Paul Brook wrote:
> > This could be very valuable when thinking about running qemu *on* embedded
> > systems with constrained memory and processing power, which is exactly what
> > the KVM for embedded PowerPC project is considering. In that scenario,
> > being able to strip out all unnecessary functionality (especially
> > including devices known to be irrelevant) becomes very important.
> 
> If you care about memory overhead the last thing you want is to be loading 
> loads of bitty little shared libraries. You want to build a single binary 
> with just the features you need.

Presumably, like most other packages, the .a's would be installed too
such that a static binary could be built.  If you were looking to build
a single-purposed process that didn't have all the features of QEMU,
then this would certainly help.

Regards,

Anthony Liguori

> 
> Paul
> 
> 

Re: [Qemu-devel] Re: [PATCH] Patches from PyQemu project

[Anthony Liguori]

On Tue, 2007-09-04 at 14:57 -0500, Brian Johnson wrote:
> Anthony Liguori wrote:
> > With things like KVM making it relatively simple to do CPU emulation, if
> > QEMU's device emulation was available as a library (even a GPL library),
> > it would be pretty easy to do interesting things without forking QEMU
> > which is what everyone seems to be doing these days.
> 
> Yes -- it would be valuable to have QEMU's devices available as a 
> separate library(ies), with a simple, well-defined API.  Some of us use 
> other CPU emulators (yes, yes, I know we should emulate all 
> architectures under qemu, but that's not always easy :) and would love 
> to use qemu's device emulations.
> 
> > My initial thought is to make the libraries at the individual device
> > level.
> 
> It would be good to have a general mechanism for bus providers, 

Well, I'm not much for over generalizing.  My current approach would be
incremental, start by focusing on a target like i386, and make the
devices into libraries.  The libraries should not be independently
loadable by another process.

I think two types of abstractions will become apparently.  The first is
how the devices communicate with the VM and covers things like
register_ioport_{read,write}, generating interrupts, etc.  QEMU already
has some nice abstractions here so this is pretty straight forward.  My
thinking is that we'll be able to have one set of APIs to cover this.

The second type of abstraction is the functionality the individual
devices provide.  This is covered by things like the BlockDriverState
interface.  Again, QEMU does a pretty good job at abstracting this
today.

Regards,

Anthony Liguori

> interrupts, APICs, chipsets, etc. as well, so we could emulate fancier 
> architectures than a simple PC (or simple Sparc/MIPS/ARM/etc. box.)  For 
> instance, I'd like to emulate multiple PCIe host bridges, each with an 
> APIC and multiple cards, which might contain PCI-to-PCI bridges.  And 
> I'd like to emulate NUMA systems with many memory controllers and a 
> complex memory map, with multiple sets of chipset registers.  I don't 
> expect qemu to do this off the shelf, but I'd like to avoid hardcoding 
> PC assumptions into the device libraries, so I can code the fancy 
> machines myself and use the I/O as-is.
> 
> Any more thoughts on how to structure the libraries?
> 
> Brian J. Johnson
> 
> 

Re: [Qemu-devel] [PATCH] Patches from PyQemu project

[Paul Sokolovsky]

Hello Anthony,

Monday, September 3, 2007, 11:44:13 PM, you wrote:

> On Mon, 2007-09-03 at 18:41 +0300, Blue Swirl wrote:
>> On 9/2/07, Maria Zabolotnaya <maria.zabolotnaya@gmail.com> wrote:
>> > 2-qemu-mplugin.patch
>> > Add -mplugin switch to allow loading of shared library and registering a
>> > machine declared in it.
>> 
>> Sorry to ruin your GSoC project, but the plugin system was discussed
>> last year, please see this thread:
>> http://thread.gmane.org/gmane.comp.emulators.qemu/14341/focus=14473


   As Maria's GSoC mentor for the project, I'm glad to add the following
comments:

> I've always agreed that allowing plugins was not a good idea.  However,
> I had a different thought recently.

  Great! Some believed this is bound to happen sooner or later.

> While I don't think there's much of a reason to allow plugins for QEMU,

  Oh, come on! ;-)

> it would be interesting to make some of QEMU's device emulation into
> more a of a library that could be used by other programs.

> With things like KVM making it relatively simple to do CPU emulation, if
> QEMU's device emulation was available as a library (even a GPL library),
> it would be pretty easy to do interesting things without forking QEMU
> which is what everyone seems to be doing these days.

  You noticed that too? Back in that 2006 thread, this was given as one
(among many) of reasons why plugins are usuful. Back then I gave one
example of fork in the domain of my interest (ARM-based real-world
products): gumstix emulation. Now in the same area I know about
the following forks: gumstix, Neo1973 (OpenMoko), hackndev.com fork for
Palm machines emulation, now this recently announced Nokia770/800
effort. Possibly, good enough count already to think what's good for
qemu and what's not.

> My initial thought is to make the libraries at the individual device
> level.

  Roundabout way works too, why not. I can understand your idea to
ignore end users' needs and instead treat QEMU as "research" platform.
Well, after all our project was structured exactly this way - instead of
crafting patches which won't be accepted upstream anyway, it was
decided to treat the whole QEMU as a library and control it using
high-level language. That plugin-support patch was produced as a proof
of concept on the initial project stage, just to show how it is easy
(well, who'd think otherwise).

  Of course, you, as a primary QEMU contributor could do it
differently, but indeed, why make users' life easier than they worth?
Let's offload functionality of loading all those "libraries at the
individual device level" (which are not plugins at all btw) to some
3rd-party "launcher". After all, we can prophet that again, sooner or
later that functionality still will become part of QEMU, right?

> Regards,

> Anthony Liguori

>> > 4-qemu-no-statics.patch
>> > Remove static declaration from some QEMU symbols, so they were exported from shared
>> > library.
>> 
>> I don't think this API is worth supporting in the future.
>> 
>> 





-- 
Best regards,
 Paul                            mailto:pmiscml@gmail.com

Re: [Qemu-devel] Re: [PATCH] Patches from PyQemu project

[Thiemo Seufer]

Brian Johnson wrote:
[snip]
>> My initial thought is to make the libraries at the individual device
>> level.
>
> It would be good to have a general mechanism for bus providers, interrupts, 
> APICs, chipsets, etc. as well, so we could emulate fancier architectures 
> than a simple PC (or simple Sparc/MIPS/ARM/etc. box.)  For instance, I'd 
> like to emulate multiple PCIe host bridges, each with an APIC and multiple 
> cards, which might contain PCI-to-PCI bridges.  And I'd like to emulate 
> NUMA systems with many memory controllers and a complex memory map, with 
> multiple sets of chipset registers.  I don't expect qemu to do this off the 
> shelf,

Why not? I would like to see better abstracted and more capable device
emulations in Qemu.

> but I'd like to avoid hardcoding PC assumptions into the device 
> libraries, so I can code the fancy machines myself and use the I/O as-is.

Then, what does a librar-ized Qemu device with its hardcoded PC
assumptions help you?

One reason why I don't like the idea is that it introduces a external
interface which is hard to maintain.


Thiemo

Re: [Qemu-devel] Re: [PATCH] Patches from PyQemu project

[Anthony Liguori]

On Tue, 2007-09-04 at 23:27 +0100, Thiemo Seufer wrote:
> Brian Johnson wrote:
> [snip]
> >> My initial thought is to make the libraries at the individual device
> >> level.
> >
> > It would be good to have a general mechanism for bus providers, interrupts, 
> > APICs, chipsets, etc. as well, so we could emulate fancier architectures 
> > than a simple PC (or simple Sparc/MIPS/ARM/etc. box.)  For instance, I'd 
> > like to emulate multiple PCIe host bridges, each with an APIC and multiple 
> > cards, which might contain PCI-to-PCI bridges.  And I'd like to emulate 
> > NUMA systems with many memory controllers and a complex memory map, with 
> > multiple sets of chipset registers.  I don't expect qemu to do this off the 
> > shelf,
> 
> Why not? I would like to see better abstracted and more capable device
> emulations in Qemu.
> 
> > but I'd like to avoid hardcoding PC assumptions into the device 
> > libraries, so I can code the fancy machines myself and use the I/O as-is.
> 
> Then, what does a librar-ized Qemu device with its hardcoded PC
> assumptions help you?
> 
> One reason why I don't like the idea is that it introduces a external
> interface which is hard to maintain.

It just depends on how you support the interface.  There's really
nothing wrong with having a per-device version #define and making no
guarantees that the interface stays consistent across versions.  The
goal isn't to have a third party tool be able to use *any* version of
QEMU's device model but rather to allow it to use *some* version of it
instead of forking it into it's own code base.

However, I don't think that the interface would change that much anyway.

Regards,

Anthony Liguori

> 
> Thiemo
> 
> 

Re: [kvm-devel] [Qemu-devel] Re: qemu device emulation libraries (was [PATCH] Patches from the PyQemu project)

[Jimi Xenidis]

On Sep 4, 2007, at 4:38 PM, Paul Brook wrote:

> On Tuesday 04 September 2007, Hollis Blanchard wrote:
>> On Tue, 2007-09-04 at 21:04 +0100, Paul Brook wrote:
>>>> This could be very valuable when thinking about running qemu *on*
>>>> embedded systems with constrained memory and processing power,  
>>>> which is
>>>> exactly what the KVM for embedded PowerPC project is  
>>>> considering. In
>>>> that scenario, being able to strip out all unnecessary  
>>>> functionality
>>>> (especially including devices known to be irrelevant) becomes very
>>>> important.
>>>
>>> If you care about memory overhead the last thing you want is to be
>>> loading loads of bitty little shared libraries. You want to build a
>>> single binary with just the features you need.
>>
>> Hmm, that's a good point. Is that something you think can  
>> reasonably be
>> accomplished with qemu today?
>
> It should be fairly easy.

Why not both?
It could be like the linux kernel, =m means build an SO and =y means  
link it statically.
The key is for it to be modular and configurable.
-JX

Re: [Qemu-devel] Re: [PATCH] Patches from PyQemu project

[Anthony Liguori]

On Tue, 2007-09-04 at 23:27 +0100, Thiemo Seufer wrote:
> Brian Johnson wrote:
> [snip]
> >> My initial thought is to make the libraries at the individual device
> >> level.
> >
> > It would be good to have a general mechanism for bus providers, interrupts, 
> > APICs, chipsets, etc. as well, so we could emulate fancier architectures 
> > than a simple PC (or simple Sparc/MIPS/ARM/etc. box.)  For instance, I'd 
> > like to emulate multiple PCIe host bridges, each with an APIC and multiple 
> > cards, which might contain PCI-to-PCI bridges.  And I'd like to emulate 
> > NUMA systems with many memory controllers and a complex memory map, with 
> > multiple sets of chipset registers.  I don't expect qemu to do this off the 
> > shelf,
> 
> Why not? I would like to see better abstracted and more capable device
> emulations in Qemu.
> 
> > but I'd like to avoid hardcoding PC assumptions into the device 
> > libraries, so I can code the fancy machines myself and use the I/O as-is.
> 
> Then, what does a librar-ized Qemu device with its hardcoded PC
> assumptions help you?

Let me give a very pragmatic answer.  Right now, KVM has it's own main
loop which uses a thread per-vcpu.  Device IO is handled in the context
of the VCPU that originated the IO.  I think there's some desire to move
to an N+1 threading model where the extra thread does things like VGA
scanning, VNC, etc.

Xen, on the other hand, uses a single thread for everything and doesn't
attempt to model multiple VCPUs within the QEMU process.  All IO is
delivered via the same channel.  QEMU is strictly used for devices.

Merging either of these with QEMU's current model would be challenging,
merging both would be extremely challenging b/c the two main loop models
are mutually exclusive.

However, there's no real trouble merging any of the device emulation
back.  KVM doesn't maintain any changes to things like device emulation
(other than some KVM-specific APIC stuff).  Xen, on the other hand,
mostly has fixes for some of the devices that haven't been submitted
back to QEMU.  Other devices are completely unmodified but are severely
lagging behind the QEMU versions.

VirtualBox is way different than QEMU of course.  A bunch of their
device emulation is pretty close to QEMU though.

I think device emulation is an independent enough problem that there's a
good chance everyone can agree on a single implementation.  While it's a
little more work in QEMU to keep these things as libraries and maintain
an interface, I suspect the benefits outweigh the extra cost in that
it's much more likely we'll see more patches.

There are useful bits of QEMU that could be widely consumed.  Their
nature is such that their interfaces are unlikely to change dramatically
in the future.  To me, this just screams "library" :-)

Regards,

Anthony Liguoi

> One reason why I don't like the idea is that it introduces a external
> interface which is hard to maintain.
> 
> 
> Thiemo
> 
> 

Re: [Qemu-devel] Re: [PATCH] Patches from PyQemu project

[Fabrice Bellard]

Anthony Liguori wrote:
> On Tue, 2007-09-04 at 23:27 +0100, Thiemo Seufer wrote:
> 
>>Brian Johnson wrote:
>>[snip]
>>
>>>>My initial thought is to make the libraries at the individual device
>>>>level.
>>>
>>>It would be good to have a general mechanism for bus providers, interrupts, 
>>>APICs, chipsets, etc. as well, so we could emulate fancier architectures 
>>>than a simple PC (or simple Sparc/MIPS/ARM/etc. box.)  For instance, I'd 
>>>like to emulate multiple PCIe host bridges, each with an APIC and multiple 
>>>cards, which might contain PCI-to-PCI bridges.  And I'd like to emulate 
>>>NUMA systems with many memory controllers and a complex memory map, with 
>>>multiple sets of chipset registers.  I don't expect qemu to do this off the 
>>>shelf,
>>
>>Why not? I would like to see better abstracted and more capable device
>>emulations in Qemu.
>>
>>
>>>but I'd like to avoid hardcoding PC assumptions into the device 
>>>libraries, so I can code the fancy machines myself and use the I/O as-is.
>>
>>Then, what does a librar-ized Qemu device with its hardcoded PC
>>assumptions help you?
> 
> 
> Let me give a very pragmatic answer.  Right now, KVM has it's own main
> loop which uses a thread per-vcpu.  Device IO is handled in the context
> of the VCPU that originated the IO.  I think there's some desire to move
> to an N+1 threading model where the extra thread does things like VGA
> scanning, VNC, etc.
> 
> Xen, on the other hand, uses a single thread for everything and doesn't
> attempt to model multiple VCPUs within the QEMU process.  All IO is
> delivered via the same channel.  QEMU is strictly used for devices.
> 
> Merging either of these with QEMU's current model would be challenging,
> merging both would be extremely challenging b/c the two main loop models
> are mutually exclusive.
> 
> However, there's no real trouble merging any of the device emulation
> back.  KVM doesn't maintain any changes to things like device emulation
> (other than some KVM-specific APIC stuff).  Xen, on the other hand,
> mostly has fixes for some of the devices that haven't been submitted
> back to QEMU.  Other devices are completely unmodified but are severely
> lagging behind the QEMU versions.
> 
> VirtualBox is way different than QEMU of course.  A bunch of their
> device emulation is pretty close to QEMU though.
> 
> I think device emulation is an independent enough problem that there's a
> good chance everyone can agree on a single implementation.  While it's a
> little more work in QEMU to keep these things as libraries and maintain
> an interface, I suspect the benefits outweigh the extra cost in that
> it's much more likely we'll see more patches.
> 
> There are useful bits of QEMU that could be widely consumed.  Their
> nature is such that their interfaces are unlikely to change dramatically
> in the future.  To me, this just screams "library" :-)

I agree with the point that QEMU should be improved so that it is easier 
to reuse the device model. But I consider that the priority is not to 
export a library but to improve the existing QEMU C APIs. The goal 
should be that it is possible to emulate several different CPUs at once 
with several unrelated buses. Moreover the model should support 
deterministic simulation. Currently the PCI bus is the closest to what 
is needed, but even in it there are still some conceptual problems (see 
the DMA problems which were discussed before).

Regarding the other projects, I see no technical reason why the KVM code 
cannot be merged. QEMU already contains the necessary support code for 
KQEMU and I see no conceptual difference with KVM. Adding a new main 
loop for KVM or a new KVM specific CPU is not something I am against.

Regards,

Fabrice.