[Qemu-devel] [PATCH 1/3] QDict: Introduce qdict_iter()

[Qemu-devel] [PATCH 1/3] QDict: Introduce qdict_iter()

[Anthony Liguori]

From: Luiz Capitulino <lcapitulino@redhat.com>

This adds iterator support to QDict, it will be used by the
(to be introduced) QError module.

Signed-off-by: Luiz Capitulino <lcapitulino@redhat.com>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
---
 qdict.c |   19 +++++++++++++++++++
 qdict.h |    3 +++
 2 files changed, 22 insertions(+), 0 deletions(-)

diff --git a/qdict.c b/qdict.c
index a302f4c..0e04cb1 100644
--- a/qdict.c
+++ b/qdict.c
@@ -242,6 +242,25 @@ const char *qdict_get_try_str(const QDict *qdict, const char *key)
 }
 
 /**
+ * qdict_iter(): Iterate over all the dictionary's stored values.
+ *
+ * This function allows the user to provide an iterator, which will be
+ * called for each stored value in the dictionary.
+ */
+void qdict_iter(const QDict *qdict,
+                void (*iter)(const char *key, QObject *obj, void *opaque),
+                void *opaque)
+{
+    int i;
+    QDictEntry *entry;
+
+    for (i = 0; i < QDICT_HASH_SIZE; i++) {
+        QLIST_FOREACH(entry, &qdict->table[i], next)
+            iter(entry->key, entry->value, opaque);
+    }
+}
+
+/**
  * qentry_destroy(): Free all the memory allocated by a QDictEntry
  */
 static void qentry_destroy(QDictEntry *e)
diff --git a/qdict.h b/qdict.h
index 3102ca2..14b2633 100644
--- a/qdict.h
+++ b/qdict.h
@@ -27,6 +27,9 @@ void qdict_del(QDict *qdict, const char *key);
 int qdict_haskey(const QDict *qdict, const char *key);
 QObject *qdict_get(const QDict *qdict, const char *key);
 QDict *qobject_to_qdict(const QObject *obj);
+void qdict_iter(const QDict *qdict,
+                void (*iter)(const char *key, QObject *obj, void *opaque),
+                void *opaque);
 
 /* Helper to qdict_put_obj(), accepts any object */
 #define qdict_put(qdict, key, obj) \
-- 
1.6.2.5

[Qemu-devel] [PATCH 2/3] Provide marshalling mechanism for json

[Anthony Liguori]

This introduces qobject_to_json which will convert a QObject to a JSON string
representation.

Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
---
 qjson.c |  178 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 qjson.h |    3 +
 2 files changed, 181 insertions(+), 0 deletions(-)

diff --git a/qjson.c b/qjson.c
index 45207f2..7270909 100644
--- a/qjson.c
+++ b/qjson.c
@@ -15,6 +15,11 @@
 #include "json-parser.h"
 #include "json-streamer.h"
 #include "qjson.h"
+#include "qint.h"
+#include "qlist.h"
+#include "qbool.h"
+#include "qfloat.h"
+#include "qdict.h"
 
 typedef struct JSONParsingState
 {
@@ -58,3 +63,176 @@ QObject *qobject_from_jsonf(const char *string, ...)
 
     return state.result;
 }
+
+typedef struct ToJsonIterState
+{
+    int count;
+    QString *str;
+} ToJsonIterState;
+
+static void to_json(const QObject *obj, QString *str);
+
+static void to_json_dict_iter(const char *key, QObject *obj, void *opaque)
+{
+    ToJsonIterState *s = opaque;
+    QString *qkey;
+
+    if (s->count) {
+        qstring_append(s->str, ", ");
+    }
+
+    qkey = qstring_from_str(key);
+    to_json(QOBJECT(qkey), s->str);
+    QDECREF(qkey);
+
+    qstring_append(s->str, ": ");
+    to_json(obj, s->str);
+    s->count++;
+}
+
+static void to_json_list_iter(QObject *obj, void *opaque)
+{
+    ToJsonIterState *s = opaque;
+
+    if (s->count) {
+        qstring_append(s->str, ", ");
+    }
+
+    to_json(obj, s->str);
+    s->count++;
+}
+
+static void to_json(const QObject *obj, QString *str)
+{
+    switch (qobject_type(obj)) {
+    case QTYPE_QINT: {
+        QInt *val = qobject_to_qint(obj);
+        char buffer[1024];
+
+        snprintf(buffer, sizeof(buffer), "%" PRId64, qint_get_int(val));
+        qstring_append(str, buffer);
+        break;
+    }
+    case QTYPE_QSTRING: {
+        QString *val = qobject_to_qstring(obj);
+        const char *ptr;
+
+        ptr = qstring_get_str(val);
+        qstring_append(str, "\"");
+        while (*ptr) {
+            if ((ptr[0] & 0xE0) == 0xE0 &&
+                (ptr[1] & 0x80) && (ptr[2] & 0x80)) {
+                uint16_t wchar;
+                char escape[7];
+
+                wchar  = (ptr[0] & 0x0F) << 12;
+                wchar |= (ptr[1] & 0x3F) << 6;
+                wchar |= (ptr[2] & 0x3F);
+                ptr += 2;
+
+                snprintf(escape, sizeof(escape), "\\u%04X", wchar);
+                qstring_append(str, escape);
+            } else if ((ptr[0] & 0xE0) == 0xC0 && (ptr[1] & 0x80)) {
+                uint16_t wchar;
+                char escape[7];
+
+                wchar  = (ptr[0] & 0x1F) << 6;
+                wchar |= (ptr[1] & 0x3F);
+                ptr++;
+
+                snprintf(escape, sizeof(escape), "\\u%04X", wchar);
+                qstring_append(str, escape);
+            } else switch (ptr[0]) {
+                case '\"':
+                    qstring_append(str, "\\\"");
+                    break;
+                case '\\':
+                    qstring_append(str, "\\\\");
+                    break;
+                case '\b':
+                    qstring_append(str, "\\b");
+                    break;
+                case '\n':
+                    qstring_append(str, "\\n");
+                    break;
+                case '\r':
+                    qstring_append(str, "\\r");
+                    break;
+                case '\t':
+                    qstring_append(str, "\\t");
+                    break;
+                default: {
+                    char buf[2] = { ptr[0], 0 };
+                    qstring_append(str, buf);
+                    break;
+                }
+                }
+            ptr++;
+        }
+        qstring_append(str, "\"");
+        break;
+    }
+    case QTYPE_QDICT: {
+        ToJsonIterState s;
+        QDict *val = qobject_to_qdict(obj);
+
+        s.count = 0;
+        s.str = str;
+        qstring_append(str, "{");
+        qdict_iter(val, to_json_dict_iter, &s);
+        qstring_append(str, "}");
+        break;
+    }
+    case QTYPE_QLIST: {
+        ToJsonIterState s;
+        QList *val = qobject_to_qlist(obj);
+
+        s.count = 0;
+        s.str = str;
+        qstring_append(str, "[");
+        qlist_iter(val, (void *)to_json_list_iter, &s);
+        qstring_append(str, "]");
+        break;
+    }
+    case QTYPE_QFLOAT: {
+        QFloat *val = qobject_to_qfloat(obj);
+        char buffer[1024];
+        int len;
+
+        len = snprintf(buffer, sizeof(buffer), "%f", qfloat_get_double(val));
+        while (len > 0 && buffer[len - 1] == '0') {
+            len--;
+        }
+
+        if (len && buffer[len - 1] == '.') {
+            buffer[len - 1] = 0;
+        } else {
+            buffer[len] = 0;
+        }
+        
+        qstring_append(str, buffer);
+        break;
+    }
+    case QTYPE_QBOOL: {
+        QBool *val = qobject_to_qbool(obj);
+
+        if (qbool_get_int(val)) {
+            qstring_append(str, "true");
+        } else {
+            qstring_append(str, "false");
+        }
+        break;
+    }
+    case QTYPE_NONE:
+        break;
+    }
+}
+
+QString *qobject_to_json(const QObject *obj)
+{
+    QString *str = qstring_new();
+
+    to_json(obj, str);
+
+    return str;
+}
diff --git a/qjson.h b/qjson.h
index 38be643..7fce742 100644
--- a/qjson.h
+++ b/qjson.h
@@ -15,9 +15,12 @@
 #define QJSON_H
 
 #include "qobject.h"
+#include "qstring.h"
 
 QObject *qobject_from_json(const char *string);
 QObject *qobject_from_jsonf(const char *string, ...)
     __attribute__((__format__ (__printf__, 1, 2)));
 
+QString *qobject_to_json(const QObject *obj);
+
 #endif /* QJSON_H */
-- 
1.6.2.5

[Qemu-devel] [PATCH 3/3] Add test suite for json marshalling

[Anthony Liguori]

By reusing the qjson test suite.  After checking that we can demarshal, marshal
again and compared to the expected decoded value.  This doesn't work so well
for floats because they cannot be accurately represented in decimal but we
try our best.

Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
---
 check-qjson.c |   80 +++++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 files changed, 75 insertions(+), 5 deletions(-)

diff --git a/check-qjson.c b/check-qjson.c
index f763de6..4b591a5 100644
--- a/check-qjson.c
+++ b/check-qjson.c
@@ -27,12 +27,13 @@ START_TEST(escaped_string)
     struct {
         const char *encoded;
         const char *decoded;
+        int skip;
     } test_cases[] = {
         { "\"\\\"\"", "\"" },
         { "\"hello world \\\"embedded string\\\"\"",
           "hello world \"embedded string\"" },
         { "\"hello world\\nwith new line\"", "hello world\nwith new line" },
-        { "\"single byte utf-8 \\u0020\"", "single byte utf-8  " },
+        { "\"single byte utf-8 \\u0020\"", "single byte utf-8  ", .skip = 1 },
         { "\"double byte utf-8 \\u00A2\"", "double byte utf-8 \xc2\xa2" },
         { "\"triple byte utf-8 \\u20AC\"", "triple byte utf-8 \xe2\x82\xac" },
         {}
@@ -50,6 +51,13 @@ START_TEST(escaped_string)
         str = qobject_to_qstring(obj);
         fail_unless(strcmp(qstring_get_str(str), test_cases[i].decoded) == 0);
 
+        if (test_cases[i].skip == 0) {
+            str = qobject_to_json(obj);
+            fail_unless(strcmp(qstring_get_str(str), test_cases[i].encoded) == 0);
+
+            qobject_decref(obj);
+        }
+
         QDECREF(str);
     }
 }
@@ -80,6 +88,11 @@ START_TEST(simple_string)
         str = qobject_to_qstring(obj);
         fail_unless(strcmp(qstring_get_str(str), test_cases[i].decoded) == 0);
 
+        str = qobject_to_json(obj);
+        fail_unless(strcmp(qstring_get_str(str), test_cases[i].encoded) == 0);
+
+        qobject_decref(obj);
+        
         QDECREF(str);
     }
 }
@@ -149,12 +162,13 @@ START_TEST(simple_number)
     struct {
         const char *encoded;
         int64_t decoded;
+        int skip;
     } test_cases[] = {
         { "0", 0 },
         { "1234", 1234 },
         { "1", 1 },
         { "-32", -32 },
-        { "-0", 0 },
+        { "-0", 0, .skip = 1 },
         { },
     };
 
@@ -168,6 +182,13 @@ START_TEST(simple_number)
 
         qint = qobject_to_qint(obj);
         fail_unless(qint_get_int(qint) == test_cases[i].decoded);
+        if (test_cases[i].skip == 0) {
+            QString *str;
+
+            str = qobject_to_json(obj);
+            fail_unless(strcmp(qstring_get_str(str), test_cases[i].encoded) == 0);
+            QDECREF(str);
+        }
 
         QDECREF(qint);
     }
@@ -180,11 +201,12 @@ START_TEST(float_number)
     struct {
         const char *encoded;
         double decoded;
+        int skip;
     } test_cases[] = {
         { "32.43", 32.43 },
         { "0.222", 0.222 },
         { "-32.12313", -32.12313 },
-        { "-32.20e-10", -32.20e-10 },
+        { "-32.20e-10", -32.20e-10, .skip = 1 },
         { },
     };
 
@@ -199,6 +221,14 @@ START_TEST(float_number)
         qfloat = qobject_to_qfloat(obj);
         fail_unless(qfloat_get_double(qfloat) == test_cases[i].decoded);
 
+        if (test_cases[i].skip == 0) {
+            QString *str;
+
+            str = qobject_to_json(obj);
+            fail_unless(strcmp(qstring_get_str(str), test_cases[i].encoded) == 0);
+            QDECREF(str);
+        }
+
         QDECREF(qfloat);
     }
 }
@@ -246,6 +276,7 @@ START_TEST(keyword_literal)
 {
     QObject *obj;
     QBool *qbool;
+    QString *str;
 
     obj = qobject_from_json("true");
     fail_unless(obj != NULL);
@@ -254,6 +285,10 @@ START_TEST(keyword_literal)
     qbool = qobject_to_qbool(obj);
     fail_unless(qbool_get_int(qbool) != 0);
 
+    str = qobject_to_json(obj);
+    fail_unless(strcmp(qstring_get_str(str), "true") == 0);
+    QDECREF(str);
+
     QDECREF(qbool);
 
     obj = qobject_from_json("false");
@@ -263,6 +298,10 @@ START_TEST(keyword_literal)
     qbool = qobject_to_qbool(obj);
     fail_unless(qbool_get_int(qbool) == 0);
 
+    str = qobject_to_json(obj);
+    fail_unless(strcmp(qstring_get_str(str), "false") == 0);
+    QDECREF(str);
+
     QDECREF(qbool);
 
     obj = qobject_from_jsonf("%i", false);
@@ -385,7 +424,7 @@ START_TEST(simple_dict)
         LiteralQObject decoded;
     } test_cases[] = {
         {
-            .encoded = "{\"foo\":42,\"bar\":\"hello world\"}",
+            .encoded = "{\"foo\": 42, \"bar\": \"hello world\"}",
             .decoded = QLIT_QDICT(((LiteralQDictEntry[]){
                         { "foo", QLIT_QINT(42) },
                         { "bar", QLIT_QSTR("hello world") },
@@ -397,7 +436,7 @@ START_TEST(simple_dict)
                         { }
                     })),
         }, {
-            .encoded = "{\"foo\":43}",
+            .encoded = "{\"foo\": 43}",
             .decoded = QLIT_QDICT(((LiteralQDictEntry[]){
                         { "foo", QLIT_QINT(43) },
                         { }
@@ -408,6 +447,7 @@ START_TEST(simple_dict)
 
     for (i = 0; test_cases[i].encoded; i++) {
         QObject *obj;
+        QString *str;
 
         obj = qobject_from_json(test_cases[i].encoded);
         fail_unless(obj != NULL);
@@ -415,7 +455,16 @@ START_TEST(simple_dict)
 
         fail_unless(compare_litqobj_to_qobj(&test_cases[i].decoded, obj) == 1);
 
+        str = qobject_to_json(obj);
+        qobject_decref(obj);
+
+        obj = qobject_from_json(qstring_get_str(str));
+        fail_unless(obj != NULL);
+        fail_unless(qobject_type(obj) == QTYPE_QDICT);
+
+        fail_unless(compare_litqobj_to_qobj(&test_cases[i].decoded, obj) == 1);
         qobject_decref(obj);
+        QDECREF(str);
     }
 }
 END_TEST
@@ -453,6 +502,7 @@ START_TEST(simple_list)
 
     for (i = 0; test_cases[i].encoded; i++) {
         QObject *obj;
+        QString *str;
 
         obj = qobject_from_json(test_cases[i].encoded);
         fail_unless(obj != NULL);
@@ -460,7 +510,16 @@ START_TEST(simple_list)
 
         fail_unless(compare_litqobj_to_qobj(&test_cases[i].decoded, obj) == 1);
 
+        str = qobject_to_json(obj);
+        qobject_decref(obj);
+
+        obj = qobject_from_json(qstring_get_str(str));
+        fail_unless(obj != NULL);
+        fail_unless(qobject_type(obj) == QTYPE_QLIST);
+
+        fail_unless(compare_litqobj_to_qobj(&test_cases[i].decoded, obj) == 1);
         qobject_decref(obj);
+        QDECREF(str);
     }
 }
 END_TEST
@@ -512,6 +571,7 @@ START_TEST(simple_whitespace)
 
     for (i = 0; test_cases[i].encoded; i++) {
         QObject *obj;
+        QString *str;
 
         obj = qobject_from_json(test_cases[i].encoded);
         fail_unless(obj != NULL);
@@ -519,7 +579,17 @@ START_TEST(simple_whitespace)
 
         fail_unless(compare_litqobj_to_qobj(&test_cases[i].decoded, obj) == 1);
 
+        str = qobject_to_json(obj);
         qobject_decref(obj);
+
+        obj = qobject_from_json(qstring_get_str(str));
+        fail_unless(obj != NULL);
+        fail_unless(qobject_type(obj) == QTYPE_QLIST);
+
+        fail_unless(compare_litqobj_to_qobj(&test_cases[i].decoded, obj) == 1);
+
+        qobject_decref(obj);
+        QDECREF(str);
     }
 }
 END_TEST
-- 
1.6.2.5

Re: [Qemu-devel] [PATCH 3/3] Add test suite for json marshalling

[Jamie Lokier]

Anthony Liguori wrote:
> After checking that we can demarshal, marshal again and compared to
> the expected decoded value.  This doesn't work so well for floats
> because they cannot be accurately represented in decimal but we try
> our best.

Good sprintf/scanf/strtod implementations do guarantee that what's
printed and then parsed gets back the same floating point value, as
long as you have printed sufficient decimal digits.

I'm not sure if FLT_DIG/DLB_DIG are the right number of digits,
though.  Glibc's documentation of those is confusing and they might
mean something a little different.

-- Jamie

Re: [Qemu-devel] [PATCH 3/3] Add test suite for json marshalling

[Anthony Liguori]

Jamie Lokier wrote:
> Anthony Liguori wrote:
>   
>> After checking that we can demarshal, marshal again and compared to
>> the expected decoded value.  This doesn't work so well for floats
>> because they cannot be accurately represented in decimal but we try
>> our best.
>>     
>
> Good sprintf/scanf/strtod implementations do guarantee that what's
> printed and then parsed gets back the same floating point value, as
> long as you have printed sufficient decimal digits.
>
> I'm not sure if FLT_DIG/DLB_DIG are the right number of digits,
> though.  Glibc's documentation of those is confusing and they might
> mean something a little different.
>   

Eh, I played around quite a bit and the results were disappointing.

$ printf "%f\n" 43.32
43.320000
$ printf "%0.32f\n" 43.32
43.31999999999999999972244424384371

Regards,

Anthony Liguori
> -- Jamie
>
>
>   

Re: [Qemu-devel] [PATCH 3/3] Add test suite for json marshalling

[Jamie Lokier]

Anthony Liguori wrote:
> Jamie Lokier wrote:
> >Anthony Liguori wrote:
> >  
> >>After checking that we can demarshal, marshal again and compared to
> >>the expected decoded value.  This doesn't work so well for floats
> >>because they cannot be accurately represented in decimal but we try
> >>our best.
> >>    
> >
> >Good sprintf/scanf/strtod implementations do guarantee that what's
> >printed and then parsed gets back the same floating point value, as
> >long as you have printed sufficient decimal digits.
> >
> >I'm not sure if FLT_DIG/DLB_DIG are the right number of digits,
> >though.  Glibc's documentation of those is confusing and they might
> >mean something a little different.
> >  
> 
> Eh, I played around quite a bit and the results were disappointing.
> 
> $ printf "%f\n" 43.32
> 43.320000
> $ printf "%0.32f\n" 43.32
> 43.31999999999999999972244424384371

True enough.  I'm not sure how to ask for the minimum number of digits
for unambiguous representation.

Point still stands though: Try *parsing* both those outputs, and the
double result is exactly the same in each case.

$ printf '%0.32f\n' $(printf '%0.32f\n' 43.32)
43.31999999999999999972244424384371

So you can do this:

    unambiguous decimal text -> double -> same text

and this:

    double -> unambiguous decimal text -> same double

As long as "unambiguous decimal" was printed with enough digits.


-- Jamie

Re: [Qemu-devel] [PATCH 3/3] Add test suite for json marshalling

[Markus Armbruster]

Jamie Lokier <jamie@shareable.org> writes:

> Anthony Liguori wrote:
>> After checking that we can demarshal, marshal again and compared to
>> the expected decoded value.  This doesn't work so well for floats
>> because they cannot be accurately represented in decimal but we try
>> our best.
>
> Good sprintf/scanf/strtod implementations do guarantee that what's
> printed and then parsed gets back the same floating point value, as
> long as you have printed sufficient decimal digits.
>
> I'm not sure if FLT_DIG/DLB_DIG are the right number of digits,
> though.  Glibc's documentation of those is confusing and they might
> mean something a little different.
>
> -- Jamie

Yes, conversion fp <-> decimal with sufficient precision can be made
exact.  However, it's non-trivial[1], and thus neither IEEE-754 nor ISO
C require it.  IIRC, GNU libc makes it exact.

No, FLT_DIG/DBL_DIG is not sufficient precision.  You need 7 decimal
digits for IEEE single, 17 for IEEE double[2].  FLT_DIG is 6 and DBL_DIG
is 15 on my system.

An alternative to all that ingenuity is C99's hexadecimal format.


[1] Steele & White: How to print floating-point numbers accurately
http://portal.acm.org/citation.cfm?id=93559
Clinger: How to Read Floating Point Numbers Accurately,
http://portal.acm.org/citation.cfm?id=93557
[2] http://docs.sun.com/source/806-3568/ncg_goldberg.html#1251