[RFC PATCH 0/4] Introduce Advanced Watch Dog module

[RFC PATCH 0/4] Introduce Advanced Watch Dog module

[Zhang Chen]

From: Zhang Chen <chen.zhang@intel.com>

Advanced Watch Dog is an universal monitoring module on VMM side, it can be used to detect network down(VMM to guest, VMM to VMM, VMM to another remote server) and do previously set operation.
for example: send message to admin, notify another VMM, send qmp command to qemu do some operation like restart the VM, build VMM heartbeat system, etc.
It make user have basic VM/Host network monitoring tools and basic false tolerance and recovery solution.

Demo usage(for COLO heartbeat):

In primary node:

-chardev socket,id=h1,host=3.3.3.3,port=9009,server,nowait
-chardev socket,id=heartbeat0,host=3.3.3.3,port=4445
-object iothread,id=iothread2
-object advanced-watchdog,id=heart1,server=on,awd_node=h1,notification_node=heartbeat0,opt_script=colo_opt_script_path,iothread=iothread1,pulse_interval=1000,timeout=5000

In secondary node:

-monitor tcp::4445,server,nowait 
-chardev socket,id=h1,host=3.3.3.3,port=9009,reconnect=1
-chardev socket,id=heart1,host=3.3.3.8,port=4445
-object iothread,id=iothread1
-object advanced-watchdog,id=heart1,server=off,awd_node=h1,notification_node=heart1,opt_script=colo_secondary_opt_script,iothread=iothread1,timeout=10000


Zhang Chen (4):
  net/awd.c: Introduce Advanced Watch Dog module framework
  net/awd.c: Initailize input/output chardev
  net/awd.c: Load advanced watch dog worker thread job
  vl.c: Make Advanced Watch Dog delayed initialization

 net/Makefile.objs |   1 +
 net/awd.c         | 486 ++++++++++++++++++++++++++++++++++++++++++++++
 qemu-options.hx   |   6 +
 vl.c              |   7 +
 4 files changed, 500 insertions(+)
 create mode 100644 net/awd.c

-- 
2.17.1

[RFC PATCH 1/4] net/awd.c: Introduce Advanced Watch Dog module framework

[Zhang Chen]

From: Zhang Chen <chen.zhang@intel.com>

This patch introduce a new module named Advanced Watch Dog,
and defined the input and output parameter. AWD use standard chardev
as the way of communicationg with the outside world.
Demo command:
-object advanced-watchdog,id=heart1,server=on,awd_node=h1,notification_node=heartbeat0,opt_script=opt_script_path,iothread=iothread1,pulse_interval=1000,timeout=5000

Signed-off-by: Zhang Chen <chen.zhang@intel.com>
---
 net/Makefile.objs |   1 +
 net/awd.c         | 261 ++++++++++++++++++++++++++++++++++++++++++++++
 qemu-options.hx   |   6 ++
 3 files changed, 268 insertions(+)
 create mode 100644 net/awd.c

diff --git a/net/Makefile.objs b/net/Makefile.objs
index c5d076d19c..139b1394e9 100644
--- a/net/Makefile.objs
+++ b/net/Makefile.objs
@@ -19,6 +19,7 @@ common-obj-y += colo-compare.o
 common-obj-y += colo.o
 common-obj-y += filter-rewriter.o
 common-obj-y += filter-replay.o
+common-obj-y += awd.o
 
 tap-obj-$(CONFIG_LINUX) = tap-linux.o
 tap-obj-$(CONFIG_BSD) = tap-bsd.o
diff --git a/net/awd.c b/net/awd.c
new file mode 100644
index 0000000000..d42b4a7372
--- /dev/null
+++ b/net/awd.c
@@ -0,0 +1,261 @@
+/*
+ * Advanced Watch Dog
+ *
+ * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
+ * (a.k.a. Fault Tolerance or Continuous Replication)
+ *
+ * Copyright (c) 2019 Intel Corporation
+ *
+ * Author: Zhang Chen <chen.zhang@intel.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/error-report.h"
+#include "trace.h"
+#include "qemu-common.h"
+#include "qapi/error.h"
+#include "net/net.h"
+#include "qom/object_interfaces.h"
+#include "qom/object.h"
+#include "chardev/char-fe.h"
+#include "qemu/sockets.h"
+#include "sysemu/iothread.h"
+
+#define TYPE_AWD  "advanced-watchdog"
+#define AWD(obj)  OBJECT_CHECK(AwdState, (obj), TYPE_AWD)
+
+#define AWD_READ_LEN_MAX NET_BUFSIZE
+/* Default advanced watchdog pulse interval */
+#define AWD_PULSE_INTERVAL_DEFAULT 5000
+/* Default advanced watchdog timeout */
+#define AWD_TIMEOUT_DEFAULT 2000
+
+typedef struct AwdState {
+    Object parent;
+
+    bool server;
+    char *awd_node;
+    char *notification_node;
+    char *opt_script;
+    uint32_t pulse_interval;
+    uint32_t timeout;
+    IOThread *iothread;
+} AwdState;
+
+typedef struct AwdClass {
+    ObjectClass parent_class;
+} AwdClass;
+
+static char *awd_get_node(Object *obj, Error **errp)
+{
+    AwdState *s = AWD(obj);
+
+    return g_strdup(s->awd_node);
+}
+
+static void awd_set_node(Object *obj, const char *value, Error **errp)
+{
+    AwdState *s = AWD(obj);
+
+    g_free(s->awd_node);
+    s->awd_node = g_strdup(value);
+}
+
+static char *noti_get_node(Object *obj, Error **errp)
+{
+    AwdState *s = AWD(obj);
+
+    return g_strdup(s->notification_node);
+}
+
+static void noti_set_node(Object *obj, const char *value, Error **errp)
+{
+    AwdState *s = AWD(obj);
+
+    g_free(s->notification_node);
+    s->notification_node = g_strdup(value);
+}
+
+static char *opt_script_get_node(Object *obj, Error **errp)
+{
+    AwdState *s = AWD(obj);
+
+    return g_strdup(s->opt_script);
+}
+
+static void opt_script_set_node(Object *obj, const char *value, Error **errp)
+{
+    AwdState *s = AWD(obj);
+
+    g_free(s->opt_script);
+    s->opt_script = g_strdup(value);
+}
+
+static bool awd_get_server(Object *obj, Error **errp)
+{
+    AwdState *s = AWD(obj);
+
+    return s->server;
+}
+
+static void awd_set_server(Object *obj, bool value, Error **errp)
+{
+    AwdState *s = AWD(obj);
+
+    s->server = value;
+}
+
+static void awd_get_interval(Object *obj, Visitor *v,
+                                   const char *name, void *opaque,
+                                   Error **errp)
+{
+    AwdState *s = AWD(obj);
+    uint32_t value = s->pulse_interval;
+
+    visit_type_uint32(v, name, &value, errp);
+}
+
+static void awd_set_interval(Object *obj, Visitor *v,
+                                   const char *name, void *opaque,
+                                   Error **errp)
+{
+    AwdState *s = AWD(obj);
+    Error *local_err = NULL;
+    uint32_t value;
+
+    visit_type_uint32(v, name, &value, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    if (!value) {
+        error_setg(&local_err, "Property '%s.%s' requires a positive value",
+                   object_get_typename(obj), name);
+        goto out;
+    }
+    s->pulse_interval = value;
+
+out:
+    error_propagate(errp, local_err);
+}
+
+static void awd_get_timeout(Object *obj, Visitor *v,
+                            const char *name, void *opaque,
+                            Error **errp)
+{
+    AwdState *s = AWD(obj);
+    uint32_t value = s->timeout;
+
+    visit_type_uint32(v, name, &value, errp);
+}
+
+static void awd_set_timeout(Object *obj, Visitor *v,
+                            const char *name, void *opaque,
+                            Error **errp)
+{
+    AwdState *s = AWD(obj);
+    Error *local_err = NULL;
+    uint32_t value;
+
+    visit_type_uint32(v, name, &value, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    if (!value) {
+        error_setg(&local_err, "Property '%s.%s' requires a positive value",
+                   object_get_typename(obj), name);
+        goto out;
+    }
+    s->timeout = value;
+
+out:
+    error_propagate(errp, local_err);
+}
+
+static void awd_complete(UserCreatable *uc, Error **errp)
+{
+    AwdState *s = AWD(uc);
+
+    if (!s->awd_node || !s->iothread ||
+        !s->notification_node || !s->opt_script) {
+        error_setg(errp, "advanced-watchdog needs 'awd_node', "
+                   "'notification_node', 'opt_script' "
+                   "and 'server' property set");
+        return;
+    }
+
+    return;
+}
+
+static void awd_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+    ucc->complete = awd_complete;
+}
+
+static void awd_init(Object *obj)
+{
+    AwdState *s = AWD(obj);
+
+    object_property_add_str(obj, "awd_node",
+                            awd_get_node, awd_set_node,
+                            NULL);
+
+    object_property_add_str(obj, "notification_node",
+                            noti_get_node, noti_set_node,
+                            NULL);
+
+    object_property_add_str(obj, "opt_script",
+                            opt_script_get_node, opt_script_set_node,
+                            NULL);
+
+    object_property_add_bool(obj, "server",
+                             awd_get_server,
+                             awd_set_server, NULL);
+
+    object_property_add(obj, "pulse_interval", "uint32",
+                        awd_get_interval,
+                        awd_set_interval, NULL, NULL, NULL);
+
+    object_property_add(obj, "timeout", "uint32",
+                        awd_get_timeout,
+                        awd_set_timeout, NULL, NULL, NULL);
+
+    object_property_add_link(obj, "iothread", TYPE_IOTHREAD,
+                            (Object **)&s->iothread,
+                            object_property_allow_set_link,
+                            OBJ_PROP_LINK_STRONG, NULL);
+}
+
+static void awd_finalize(Object *obj)
+{
+    AwdState *s = AWD(obj);
+
+    g_free(s->awd_node);
+    g_free(s->notification_node);
+}
+
+static const TypeInfo awd_info = {
+    .name = TYPE_AWD,
+    .parent = TYPE_OBJECT,
+    .instance_size = sizeof(AwdState),
+    .instance_init = awd_init,
+    .instance_finalize = awd_finalize,
+    .class_size = sizeof(AwdClass),
+    .class_init = awd_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_USER_CREATABLE },
+        { }
+    }
+};
+
+static void register_types(void)
+{
+    type_register_static(&awd_info);
+}
+
+type_init(register_types);
diff --git a/qemu-options.hx b/qemu-options.hx
index 793d70ff93..a4c1e2e2d1 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -4520,6 +4520,12 @@ Dump the network traffic on netdev @var{dev} to the file specified by
 The file format is libpcap, so it can be analyzed with tools such as tcpdump
 or Wireshark.
 
+@item -object advanced-watchdog,id=@var{id},awd_node=@var{chardevid},notification_node=@var{chardevid},server=@var{server},iothread=@var{id}[,pulse_interval=@var{time_ms},timeout=@var{time_ms}]
+
+Advanced Watch Dog is an universal monitoring module on VMM side, it can be used to detect network down(VMM to guest, VMM to VMM, VMM to another remote server) and do previously set operation.
+for example: send message to admin, notify another VMM, send qmp command to qemu do some operation like restart the VM, build VMM heartbeat system, etc.
+It make user have basic VM/Host network monitoring tools and basic false tolerance and recovery solution.
+
 @item -object colo-compare,id=@var{id},primary_in=@var{chardevid},secondary_in=@var{chardevid},outdev=@var{chardevid},iothread=@var{id}[,vnet_hdr_support][,notify_dev=@var{id}]
 
 Colo-compare gets packet from primary_in@var{chardevid} and secondary_in@var{chardevid}, than compare primary packet with
-- 
2.17.1

Re: [RFC PATCH 1/4] net/awd.c: Introduce Advanced Watch Dog module framework

[Paolo Bonzini]

On 16/10/19 13:22, Zhang Chen wrote:
> From: Zhang Chen <chen.zhang@intel.com>
> 
> This patch introduce a new module named Advanced Watch Dog, and
> defined the input and output parameter. AWD use standard chardev as
> the way of communicationg with the outside world. Demo command: 
> -object
> advanced-watchdog,id=heart1,server=on,awd_node=h1,notification_node=heartbeat0,opt_script=opt_script_path,iothread=iothread1,pulse_interval=1000,timeout=5000
Is there a standard or specification for how to interact with this?  Is
this something that was developed explicitly for these patches, or did
it exist before?

Paolo

RE: [RFC PATCH 1/4] net/awd.c: Introduce Advanced Watch Dog module framework

[Zhang, Chen]

> -----Original Message-----
> From: Paolo Bonzini <pbonzini@redhat.com>
> Sent: Tuesday, October 22, 2019 1:24 AM
> To: Zhang, Chen <chen.zhang@intel.com>; Jason Wang
> <jasowang@redhat.com>; qemu-dev <qemu-devel@nongnu.org>
> Cc: Zhang Chen <zhangckid@gmail.com>
> Subject: Re: [RFC PATCH 1/4] net/awd.c: Introduce Advanced Watch Dog
> module framework
> 
> On 16/10/19 13:22, Zhang Chen wrote:
> > From: Zhang Chen <chen.zhang@intel.com>
> >
> > This patch introduce a new module named Advanced Watch Dog, and
> > defined the input and output parameter. AWD use standard chardev as
> > the way of communicationg with the outside world. Demo command:
> > -object
> > advanced-
> watchdog,id=heart1,server=on,awd_node=h1,notification_node=he
> > artbeat0,opt_script=opt_script_path,iothread=iothread1,pulse_interval=
> > 1000,timeout=5000
> Is there a standard or specification for how to interact with this?  Is this
> something that was developed explicitly for these patches, or did it exist
> before?
> 

Hi Paolo,

Thanks you for your attention.
Current AWD patch just accept any input as the signal to refresh the watchdog timer,
and we can also make a certain interactive protocol here. For the output user can pre-write
some command or some messages in the AWD opt-script. We noticed that there is no way
for VMM communicate directly, maybe some people think we don't need such things(up layer
software like openstack can handle it). But we engaged with real customer found that in some cases,
they need a lightweight and efficient mechanism to solve some practical problems(openstack is too heavy).
For example in COLO (https://wiki.qemu.org/Features/COLO).
We need heartbeat service to detect two node status, current implement need another application
running in the host to do this job, if found issue it will send message to local Qemu by qmp command,
This is very complicated in actual scenarios, we have to bind each heartbeat service with
the VM instance and handle lots of internal errors. Advanced watch dog can solve this issue in most cases.
At the same time, AWD also can be used in a variety of scenarios, it can active detect Qemu status and 
do some simple automatic processing for example kill itself when it loss control with openstack(just need simple paired service). 
 

Thanks
Zhang Chen

> Paolo

Re: [RFC PATCH 1/4] net/awd.c: Introduce Advanced Watch Dog module framework

[Philippe Mathieu-Daudé]

Hi Chen,

On 10/16/19 1:22 PM, Zhang Chen wrote:
> From: Zhang Chen <chen.zhang@intel.com>
> 
> This patch introduce a new module named Advanced Watch Dog,
> and defined the input and output parameter. AWD use standard chardev
> as the way of communicationg with the outside world.
> Demo command:
> -object advanced-watchdog,id=heart1,server=on,awd_node=h1,notification_node=heartbeat0,opt_script=opt_script_path,iothread=iothread1,pulse_interval=1000,timeout=5000
> 
> Signed-off-by: Zhang Chen <chen.zhang@intel.com>
> ---
>   net/Makefile.objs |   1 +
>   net/awd.c         | 261 ++++++++++++++++++++++++++++++++++++++++++++++
>   qemu-options.hx   |   6 ++
>   3 files changed, 268 insertions(+)
>   create mode 100644 net/awd.c
> 
> diff --git a/net/Makefile.objs b/net/Makefile.objs
> index c5d076d19c..139b1394e9 100644
> --- a/net/Makefile.objs
> +++ b/net/Makefile.objs
> @@ -19,6 +19,7 @@ common-obj-y += colo-compare.o
>   common-obj-y += colo.o
>   common-obj-y += filter-rewriter.o
>   common-obj-y += filter-replay.o
> +common-obj-y += awd.o
Can you add a net/Kconfig file introducing the ADVANCED_WATCHDOG selector?

config COLO_ADVANCED_WATCHDOG
     bool
     default n

Then use here:

     common-obj-$(COLO_ADVANCED_WATCHDOG) += awd.o

Thanks,

Phil.

>   
>   tap-obj-$(CONFIG_LINUX) = tap-linux.o
>   tap-obj-$(CONFIG_BSD) = tap-bsd.o
[...]

RE: [RFC PATCH 1/4] net/awd.c: Introduce Advanced Watch Dog module framework

[Zhang, Chen]

> -----Original Message-----
> From: Philippe Mathieu-Daudé <philmd@redhat.com>
> Sent: Wednesday, October 23, 2019 7:01 PM
> To: Zhang, Chen <chen.zhang@intel.com>; Jason Wang
> <jasowang@redhat.com>; Paolo Bonzini <pbonzini@redhat.com>; qemu-
> dev <qemu-devel@nongnu.org>
> Cc: Zhang Chen <zhangckid@gmail.com>
> Subject: Re: [RFC PATCH 1/4] net/awd.c: Introduce Advanced Watch Dog
> module framework
> 
> Hi Chen,
> 
> On 10/16/19 1:22 PM, Zhang Chen wrote:
> > From: Zhang Chen <chen.zhang@intel.com>
> >
> > This patch introduce a new module named Advanced Watch Dog, and
> > defined the input and output parameter. AWD use standard chardev as
> > the way of communicationg with the outside world.
> > Demo command:
> > -object
> > advanced-
> watchdog,id=heart1,server=on,awd_node=h1,notification_node=he
> > artbeat0,opt_script=opt_script_path,iothread=iothread1,pulse_interval=
> > 1000,timeout=5000
> >
> > Signed-off-by: Zhang Chen <chen.zhang@intel.com>
> > ---
> >   net/Makefile.objs |   1 +
> >   net/awd.c         | 261
> ++++++++++++++++++++++++++++++++++++++++++++++
> >   qemu-options.hx   |   6 ++
> >   3 files changed, 268 insertions(+)
> >   create mode 100644 net/awd.c
> >
> > diff --git a/net/Makefile.objs b/net/Makefile.objs index
> > c5d076d19c..139b1394e9 100644
> > --- a/net/Makefile.objs
> > +++ b/net/Makefile.objs
> > @@ -19,6 +19,7 @@ common-obj-y += colo-compare.o
> >   common-obj-y += colo.o
> >   common-obj-y += filter-rewriter.o
> >   common-obj-y += filter-replay.o
> > +common-obj-y += awd.o
> Can you add a net/Kconfig file introducing the ADVANCED_WATCHDOG
> selector?
> 
> config COLO_ADVANCED_WATCHDOG
>      bool
>      default n
> 
> Then use here:
> 
>      common-obj-$(COLO_ADVANCED_WATCHDOG) += awd.o
> 

Sure, but AWD is a universal module,  COLO is just the first user.
Maybe use "config ADVANCED_WATCHDOG" is better.

Thanks
Zhang Chen

> Thanks,
> 
> Phil.
> 
> >
> >   tap-obj-$(CONFIG_LINUX) = tap-linux.o
> >   tap-obj-$(CONFIG_BSD) = tap-bsd.o
> [...]

Re: [RFC PATCH 1/4] net/awd.c: Introduce Advanced Watch Dog module framework

[Philippe Mathieu-Daudé]

On 10/23/19 1:09 PM, Zhang, Chen wrote:
>> -----Original Message-----
>> From: Philippe Mathieu-Daudé <philmd@redhat.com>
>> Sent: Wednesday, October 23, 2019 7:01 PM
>> To: Zhang, Chen <chen.zhang@intel.com>; Jason Wang
>> <jasowang@redhat.com>; Paolo Bonzini <pbonzini@redhat.com>; qemu-
>> dev <qemu-devel@nongnu.org>
>> Cc: Zhang Chen <zhangckid@gmail.com>
>> Subject: Re: [RFC PATCH 1/4] net/awd.c: Introduce Advanced Watch Dog
>> module framework
>>
>> Hi Chen,
>>
>> On 10/16/19 1:22 PM, Zhang Chen wrote:
>>> From: Zhang Chen <chen.zhang@intel.com>
>>>
>>> This patch introduce a new module named Advanced Watch Dog, and
>>> defined the input and output parameter. AWD use standard chardev as
>>> the way of communicationg with the outside world.
>>> Demo command:
>>> -object
>>> advanced-
>> watchdog,id=heart1,server=on,awd_node=h1,notification_node=he
>>> artbeat0,opt_script=opt_script_path,iothread=iothread1,pulse_interval=
>>> 1000,timeout=5000
>>>
>>> Signed-off-by: Zhang Chen <chen.zhang@intel.com>
>>> ---
>>>    net/Makefile.objs |   1 +
>>>    net/awd.c         | 261
>> ++++++++++++++++++++++++++++++++++++++++++++++
>>>    qemu-options.hx   |   6 ++
>>>    3 files changed, 268 insertions(+)
>>>    create mode 100644 net/awd.c
>>>
>>> diff --git a/net/Makefile.objs b/net/Makefile.objs index
>>> c5d076d19c..139b1394e9 100644
>>> --- a/net/Makefile.objs
>>> +++ b/net/Makefile.objs
>>> @@ -19,6 +19,7 @@ common-obj-y += colo-compare.o
>>>    common-obj-y += colo.o
>>>    common-obj-y += filter-rewriter.o
>>>    common-obj-y += filter-replay.o
>>> +common-obj-y += awd.o
>> Can you add a net/Kconfig file introducing the ADVANCED_WATCHDOG
>> selector?
>>
>> config COLO_ADVANCED_WATCHDOG
>>       bool
>>       default n
>>
>> Then use here:
>>
>>       common-obj-$(COLO_ADVANCED_WATCHDOG) += awd.o
>>
> 
> Sure, but AWD is a universal module,  COLO is just the first user.
> Maybe use "config ADVANCED_WATCHDOG" is better.

Oh I see, better then.

Then we might add (later)

   config COLO
       ...
       select ADVANCED_WATCHDOG

Thanks!

Phil.

>>>    tap-obj-$(CONFIG_LINUX) = tap-linux.o
>>>    tap-obj-$(CONFIG_BSD) = tap-bsd.o
>> [...]

RE: [RFC PATCH 1/4] net/awd.c: Introduce Advanced Watch Dog module framework

[Zhang, Chen]

> -----Original Message-----
> From: Philippe Mathieu-Daudé <philmd@redhat.com>
> Sent: Wednesday, October 23, 2019 7:39 PM
> To: Zhang, Chen <chen.zhang@intel.com>; Jason Wang
> <jasowang@redhat.com>; Paolo Bonzini <pbonzini@redhat.com>; qemu-
> dev <qemu-devel@nongnu.org>
> Cc: Zhang Chen <zhangckid@gmail.com>
> Subject: Re: [RFC PATCH 1/4] net/awd.c: Introduce Advanced Watch Dog
> module framework
> 
> On 10/23/19 1:09 PM, Zhang, Chen wrote:
> >> -----Original Message-----
> >> From: Philippe Mathieu-Daudé <philmd@redhat.com>
> >> Sent: Wednesday, October 23, 2019 7:01 PM
> >> To: Zhang, Chen <chen.zhang@intel.com>; Jason Wang
> >> <jasowang@redhat.com>; Paolo Bonzini <pbonzini@redhat.com>; qemu-
> dev
> >> <qemu-devel@nongnu.org>
> >> Cc: Zhang Chen <zhangckid@gmail.com>
> >> Subject: Re: [RFC PATCH 1/4] net/awd.c: Introduce Advanced Watch Dog
> >> module framework
> >>
> >> Hi Chen,
> >>
> >> On 10/16/19 1:22 PM, Zhang Chen wrote:
> >>> From: Zhang Chen <chen.zhang@intel.com>
> >>>
> >>> This patch introduce a new module named Advanced Watch Dog, and
> >>> defined the input and output parameter. AWD use standard chardev as
> >>> the way of communicationg with the outside world.
> >>> Demo command:
> >>> -object
> >>> advanced-
> >> watchdog,id=heart1,server=on,awd_node=h1,notification_node=he
> >>> artbeat0,opt_script=opt_script_path,iothread=iothread1,pulse_interva
> >>> l=
> >>> 1000,timeout=5000
> >>>
> >>> Signed-off-by: Zhang Chen <chen.zhang@intel.com>
> >>> ---
> >>>    net/Makefile.objs |   1 +
> >>>    net/awd.c         | 261
> >> ++++++++++++++++++++++++++++++++++++++++++++++
> >>>    qemu-options.hx   |   6 ++
> >>>    3 files changed, 268 insertions(+)
> >>>    create mode 100644 net/awd.c
> >>>
> >>> diff --git a/net/Makefile.objs b/net/Makefile.objs index
> >>> c5d076d19c..139b1394e9 100644
> >>> --- a/net/Makefile.objs
> >>> +++ b/net/Makefile.objs
> >>> @@ -19,6 +19,7 @@ common-obj-y += colo-compare.o
> >>>    common-obj-y += colo.o
> >>>    common-obj-y += filter-rewriter.o
> >>>    common-obj-y += filter-replay.o
> >>> +common-obj-y += awd.o
> >> Can you add a net/Kconfig file introducing the ADVANCED_WATCHDOG
> >> selector?
> >>
> >> config COLO_ADVANCED_WATCHDOG
> >>       bool
> >>       default n
> >>
> >> Then use here:
> >>
> >>       common-obj-$(COLO_ADVANCED_WATCHDOG) += awd.o
> >>
> >
> > Sure, but AWD is a universal module,  COLO is just the first user.
> > Maybe use "config ADVANCED_WATCHDOG" is better.
> 
> Oh I see, better then.
> 
> Then we might add (later)
> 
>    config COLO
>        ...
>        select ADVANCED_WATCHDOG
> 

I noticed that we haven't the "net/kconfig" file, the AWD isn't a HW in the hw/net.
Do you means we can add the selector in "configuration" file?
We can add some related configuration command like "--enable-advanced-watchdog" and default is "no".

Thanks
Zhang Chen

> Thanks!
> 
> Phil.
> 
> >>>    tap-obj-$(CONFIG_LINUX) = tap-linux.o
> >>>    tap-obj-$(CONFIG_BSD) = tap-bsd.o
> >> [...]

[RFC PATCH 2/4] net/awd.c: Initailize input/output chardev

[Zhang Chen]

From: Zhang Chen <chen.zhang@intel.com>

Find and check the chardev awd_node and notification_node,
The awd_node used for keep connect with outside(like VM client/other
host/Remote server), and the notification_node used for do some
operation when disconnect event occur.

Signed-off-by: Zhang Chen <chen.zhang@intel.com>
---
 net/awd.c | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/net/awd.c b/net/awd.c
index d42b4a7372..ad3d39c982 100644
--- a/net/awd.c
+++ b/net/awd.c
@@ -42,6 +42,8 @@ typedef struct AwdState {
     char *opt_script;
     uint32_t pulse_interval;
     uint32_t timeout;
+    CharBackend chr_awd_node;
+    CharBackend chr_notification_node;
     IOThread *iothread;
 } AwdState;
 
@@ -175,9 +177,30 @@ out:
     error_propagate(errp, local_err);
 }
 
+static int find_and_check_chardev(Chardev **chr,
+                                  char *chr_name,
+                                  Error **errp)
+{
+    *chr = qemu_chr_find(chr_name);
+    if (*chr == NULL) {
+        error_setg(errp, "Device '%s' not found",
+                   chr_name);
+        return 1;
+    }
+
+    if (!qemu_chr_has_feature(*chr, QEMU_CHAR_FEATURE_RECONNECTABLE)) {
+        error_setg(errp, "chardev \"%s\" is not reconnectable",
+                   chr_name);
+        return 1;
+    }
+
+    return 0;
+}
+
 static void awd_complete(UserCreatable *uc, Error **errp)
 {
     AwdState *s = AWD(uc);
+    Chardev *chr;
 
     if (!s->awd_node || !s->iothread ||
         !s->notification_node || !s->opt_script) {
@@ -187,6 +210,20 @@ static void awd_complete(UserCreatable *uc, Error **errp)
         return;
     }
 
+    if (find_and_check_chardev(&chr, s->awd_node, errp) ||
+        !qemu_chr_fe_init(&s->chr_awd_node, chr, errp)) {
+        error_setg(errp, "advanced-watchdog can't find chardev awd_node: %s",
+                   s->awd_node);
+        return;
+    }
+
+    if (find_and_check_chardev(&chr, s->notification_node, errp) ||
+        !qemu_chr_fe_init(&s->chr_notification_node, chr, errp)) {
+        error_setg(errp, "advanced-watchdog can't find "
+                   "chardev notification_node: %s", s->notification_node);
+        return;
+    }
+
     return;
 }
 
-- 
2.17.1

[RFC PATCH 3/4] net/awd.c: Load advanced watch dog worker thread job

[Zhang Chen]

From: Zhang Chen <chen.zhang@intel.com>

This patch load pulse_timer and timeout_timer in the new iothread.
The pulse timer will send pulse info to awd_node, and the timeout timer
will check the reply pulse from awd_node. If timeout occur, it will send
opt_script's data to the notification_node.

Signed-off-by: Zhang Chen <chen.zhang@intel.com>
---
 net/awd.c | 188 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 188 insertions(+)

diff --git a/net/awd.c b/net/awd.c
index ad3d39c982..ea7f628437 100644
--- a/net/awd.c
+++ b/net/awd.c
@@ -40,17 +40,137 @@ typedef struct AwdState {
     char *awd_node;
     char *notification_node;
     char *opt_script;
+    char *opt_script_data;
     uint32_t pulse_interval;
     uint32_t timeout;
     CharBackend chr_awd_node;
     CharBackend chr_notification_node;
+    SocketReadState awd_rs;
+
+    QEMUTimer *pulse_timer;
+    QEMUTimer *timeout_timer;
     IOThread *iothread;
+    GMainContext *worker_context;
 } AwdState;
 
 typedef struct AwdClass {
     ObjectClass parent_class;
 } AwdClass;
 
+static int awd_chr_send(AwdState *s,
+                        const uint8_t *buf,
+                        uint32_t size)
+{
+    int ret = 0;
+    uint32_t len = htonl(size);
+
+    if (!size) {
+        return 0;
+    }
+
+    ret = qemu_chr_fe_write_all(&s->chr_awd_node, (uint8_t *)&len,
+                                sizeof(len));
+    if (ret != sizeof(len)) {
+        goto err;
+    }
+
+    ret = qemu_chr_fe_write_all(&s->chr_awd_node, (uint8_t *)buf,
+                                size);
+    if (ret != size) {
+        goto err;
+    }
+
+    return 0;
+
+err:
+    return ret < 0 ? ret : -EIO;
+}
+
+static int awd_chr_can_read(void *opaque)
+{
+    return AWD_READ_LEN_MAX;
+}
+
+static void awd_node_in(void *opaque, const uint8_t *buf, int size)
+{
+    AwdState *s = AWD(opaque);
+    int ret;
+
+    ret = net_fill_rstate(&s->awd_rs, buf, size);
+    if (ret == -1) {
+        qemu_chr_fe_set_handlers(&s->chr_awd_node, NULL, NULL, NULL, NULL,
+                                 NULL, NULL, true);
+        error_report("advanced-watchdog get pulse error");
+    }
+}
+
+static void awd_send_pulse(void *opaque)
+{
+    AwdState *s = opaque;
+    char buf[] = "advanced-watchdog pulse";
+
+    awd_chr_send(s, (uint8_t *)buf, sizeof(buf));
+}
+
+static void awd_regular_pulse(void *opaque)
+{
+    AwdState *s = opaque;
+
+    awd_send_pulse(s);
+    timer_mod(s->pulse_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+              s->pulse_interval);
+}
+
+static void awd_timeout(void *opaque)
+{
+    AwdState *s = opaque;
+    int ret = 0;
+
+    ret = qemu_chr_fe_write_all(&s->chr_notification_node,
+                                (uint8_t *)s->opt_script_data,
+                                strlen(s->opt_script_data));
+    if (ret) {
+        error_report("advanced-watchdog notification failure");
+    }
+}
+
+static void awd_timer_init(AwdState *s)
+{
+    AioContext *ctx = iothread_get_aio_context(s->iothread);
+
+    s->timeout_timer = aio_timer_new(ctx, QEMU_CLOCK_VIRTUAL, SCALE_MS,
+                                     awd_timeout, s);
+
+    s->pulse_timer = aio_timer_new(ctx, QEMU_CLOCK_VIRTUAL, SCALE_MS,
+                                      awd_regular_pulse, s);
+
+    if (!s->pulse_interval) {
+        s->pulse_interval = AWD_PULSE_INTERVAL_DEFAULT;
+    }
+
+    if (!s->timeout) {
+        s->timeout = AWD_TIMEOUT_DEFAULT;
+    }
+
+    timer_mod(s->pulse_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+              s->pulse_interval);
+}
+
+static void awd_timer_del(AwdState *s)
+{
+    if (s->pulse_timer) {
+        timer_del(s->pulse_timer);
+        timer_free(s->pulse_timer);
+        s->pulse_timer = NULL;
+    }
+
+    if (s->timeout_timer) {
+        timer_del(s->timeout_timer);
+        timer_free(s->timeout_timer);
+        s->timeout_timer = NULL;
+    }
+ }
+
 static char *awd_get_node(Object *obj, Error **errp)
 {
     AwdState *s = AWD(obj);
@@ -177,6 +297,22 @@ out:
     error_propagate(errp, local_err);
 }
 
+static void awd_rs_finalize(SocketReadState *awd_rs)
+{
+    AwdState *s = container_of(awd_rs, AwdState, awd_rs);
+
+    if (!s->server) {
+        char buf[] = "advanced-watchdog reply pulse";
+
+        awd_chr_send(s, (uint8_t *)buf, sizeof(buf));
+    }
+
+    timer_mod(s->timeout_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+              s->timeout);
+
+    error_report("advanced-watchdog got message : %s", awd_rs->buf);
+}
+
 static int find_and_check_chardev(Chardev **chr,
                                   char *chr_name,
                                   Error **errp)
@@ -197,6 +333,41 @@ static int find_and_check_chardev(Chardev **chr,
     return 0;
 }
 
+static void awd_iothread(AwdState *s)
+{
+    object_ref(OBJECT(s->iothread));
+    s->worker_context = iothread_get_g_main_context(s->iothread);
+
+    qemu_chr_fe_set_handlers(&s->chr_awd_node, awd_chr_can_read,
+                             awd_node_in, NULL, NULL,
+                             s, s->worker_context, true);
+
+    awd_timer_init(s);
+}
+
+static int get_opt_script_data(AwdState *s)
+{
+    FILE *opt_fd;
+    long fsize;
+
+    opt_fd = fopen(s->opt_script, "r");
+    if (opt_fd == NULL) {
+        error_report("advanced-watchdog can't read "
+                     "opt_script: %s", s->opt_script);
+        return -1;
+    }
+
+    fseek(opt_fd, 0, SEEK_END);
+    fsize = ftell(opt_fd);
+    fseek(opt_fd, 0, SEEK_SET);
+    s->opt_script_data = malloc(fsize + 1);
+    fread(s->opt_script_data, 1, fsize, opt_fd);
+
+    fclose(opt_fd);
+
+    return 0;
+}
+
 static void awd_complete(UserCreatable *uc, Error **errp)
 {
     AwdState *s = AWD(uc);
@@ -224,6 +395,16 @@ static void awd_complete(UserCreatable *uc, Error **errp)
         return;
     }
 
+    if (get_opt_script_data(s)) {
+        error_setg(errp, "advanced-watchdog can't get "
+                   "opt script data: %s", s->opt_script);
+        return;
+    }
+
+    net_socket_rs_init(&s->awd_rs, awd_rs_finalize, false);
+
+    awd_iothread(s);
+
     return;
 }
 
@@ -272,6 +453,13 @@ static void awd_finalize(Object *obj)
 {
     AwdState *s = AWD(obj);
 
+    qemu_chr_fe_deinit(&s->chr_awd_node, false);
+    qemu_chr_fe_deinit(&s->chr_notification_node, false);
+
+    if (s->iothread) {
+        awd_timer_del(s);
+    }
+
     g_free(s->awd_node);
     g_free(s->notification_node);
 }
-- 
2.17.1

[RFC PATCH 4/4] vl.c: Make Advanced Watch Dog delayed initialization

[Zhang Chen]

From: Zhang Chen <chen.zhang@intel.com>

Because Advanced Watch Dog module needs chardev socket
to initialize properly before.

Signed-off-by: Zhang Chen <chen.zhang@intel.com>
---
 vl.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/vl.c b/vl.c
index 4489cfb2bb..4094f61d28 100644
--- a/vl.c
+++ b/vl.c
@@ -2683,6 +2683,13 @@ static bool object_create_initial(const char *type, QemuOpts *opts)
         return false;
     }
 
+    /*
+     * Reason: Advanced Watch Dog property "chardev".
+     */
+    if (g_str_equal(type, "advanced-watchdog")) {
+        return false;
+    }
+
     /* Memory allocation by backends needs to be done
      * after configure_accelerator() (due to the tcg_enabled()
      * checks at memory_region_init_*()).
-- 
2.17.1

Re: [RFC PATCH 0/4] Introduce Advanced Watch Dog module

[no-reply]

Patchew URL: https://patchew.org/QEMU/20191016112209.9024-1-chen.zhang@intel.com/



Hi,

This series failed the docker-quick@centos7 build test. Please find the testing commands and
their output below. If you have Docker installed, you can probably reproduce it
locally.

=== TEST SCRIPT BEGIN ===
#!/bin/bash
make docker-image-centos7 V=1 NETWORK=1
time make docker-test-quick@centos7 SHOW_ENV=1 J=14 NETWORK=1
=== TEST SCRIPT END ===

Clone of 'https://git.qemu.org/git/dtc.git' into submodule path 'dtc' failed
failed to update submodule dtc
Submodule 'dtc' (https://git.qemu.org/git/dtc.git) unregistered for path 'dtc'
make[1]: *** [/var/tmp/patchew-tester-tmp-2eq69u33/src/docker-src.2019-10-16-11.34.59.10123] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-2eq69u33/src'
make: *** [docker-run-test-quick@centos7] Error 2

real    0m3.762s
user    0m2.349s


The full log is available at
http://patchew.org/logs/20191016112209.9024-1-chen.zhang@intel.com/testing.docker-quick@centos7/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to patchew-devel@redhat.com

Re: [RFC PATCH 0/4] Introduce Advanced Watch Dog module

[Zhang Chen]

[-- Attachment #1: Type: text/plain, Size: 1365 bytes --]

On Wed, Oct 16, 2019 at 11:35 PM <no-reply@patchew.org> wrote:

> Patchew URL:
> https://patchew.org/QEMU/20191016112209.9024-1-chen.zhang@intel.com/
>
>
>
> Hi,
>
> This series failed the docker-quick@centos7 build test. Please find the
> testing commands and
> their output below. If you have Docker installed, you can probably
> reproduce it
> locally.
>
> === TEST SCRIPT BEGIN ===
> #!/bin/bash
> make docker-image-centos7 V=1 NETWORK=1
> time make docker-test-quick@centos7 SHOW_ENV=1 J=14 NETWORK=1
> === TEST SCRIPT END ===
>
> Clone of 'https://git.qemu.org/git/dtc.git' into submodule path 'dtc'
> failed
> failed to update submodule dtc
> Submodule 'dtc' (https://git.qemu.org/git/dtc.git) unregistered for path
> 'dtc'
> make[1]: ***
> [/var/tmp/patchew-tester-tmp-2eq69u33/src/docker-src.2019-10-16-11.34.59.10123]
> Error 1
> make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-2eq69u33/src'
> make: *** [docker-run-test-quick@centos7] Error 2
>
>
It looks this problem dose not caused by my patch series?

Thanks
Zhang Chen


> real    0m3.762s
> user    0m2.349s
>
>
> The full log is available at
>
> http://patchew.org/logs/20191016112209.9024-1-chen.zhang@intel.com/testing.docker-quick@centos7/?type=message
> .
> ---
> Email generated automatically by Patchew [https://patchew.org/].
> Please send your feedback to patchew-devel@redhat.com

[-- Attachment #2: Type: text/html, Size: 2557 bytes --]

Re: [RFC PATCH 0/4] Introduce Advanced Watch Dog module

[Zhang Chen]

On Wed, Oct 16, 2019 at 11:35 PM <no-reply@patchew.org> wrote:
>
> Patchew URL: https://patchew.org/QEMU/20191016112209.9024-1-chen.zhang@intel.com/
>
>
>
> Hi,
>
> This series failed the docker-quick@centos7 build test. Please find the testing commands and
> their output below. If you have Docker installed, you can probably reproduce it
> locally.
>
> === TEST SCRIPT BEGIN ===
> #!/bin/bash
> make docker-image-centos7 V=1 NETWORK=1
> time make docker-test-quick@centos7 SHOW_ENV=1 J=14 NETWORK=1
> === TEST SCRIPT END ===
>
> Clone of 'https://git.qemu.org/git/dtc.git' into submodule path 'dtc' failed
> failed to update submodule dtc
> Submodule 'dtc' (https://git.qemu.org/git/dtc.git) unregistered for path 'dtc'
> make[1]: *** [/var/tmp/patchew-tester-tmp-2eq69u33/src/docker-src.2019-10-16-11.34.59.10123] Error 1
> make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-2eq69u33/src'
> make: *** [docker-run-test-quick@centos7] Error 2
>

It looks this problem dose not caused by my patch series?

Thanks
Zhang Chen

> real    0m3.762s
> user    0m2.349s
>
>
> The full log is available at
> http://patchew.org/logs/20191016112209.9024-1-chen.zhang@intel.com/testing.docker-quick@centos7/?type=message.
> ---
> Email generated automatically by Patchew [https://patchew.org/].
> Please send your feedback to patchew-devel@redhat.com