* [PATCH 1/2] libxl: fix fd and timer event handling
2015-02-03 0:00 [PATCH 0/2] libxl: fix handling of fd and timer registrations Jim Fehlig
@ 2015-02-03 0:00 ` Jim Fehlig
2015-02-03 0:00 ` [PATCH 2/2] libxl: Move setup of child processing code to driver initialization Jim Fehlig
2015-02-06 12:32 ` [PATCH 0/2] libxl: fix handling of fd and timer registrations Anthony PERARD
2 siblings, 0 replies; 6+ messages in thread
From: Jim Fehlig @ 2015-02-03 0:00 UTC (permalink / raw)
To: libvir-list; +Cc: Jim Fehlig, xen-devel
Long ago I incorrectly associated libxl fd and timer registrations
with per-domain libxl_ctx objects. When creating a libxlDomainObjPrivate,
a libxl_ctx is allocated, and libxl_osevent_register_hooks is called
passing a pointer to the libxlDomainObjPrivate. When an fd or timer
registration occurred, the registration callback received the
libxlDomainObjPrivate, containing the per-domain libxl_ctx. This
libxl_ctx was then used when informing libxl about fd events or
timer expirations.
The problem with this approach is that fd and timer registrations do not
share the same lifespan as libxlDomainObjPrivate, and hence the per-domain
libxl_ctx ojects. The result is races between per-domain libxl_ctx's being
destoryed and events firing on associated fds/timers, typically manifesting
as an assert in libxl
libxl_internal.h:2788: libxl__ctx_unlock: Assertion `!r' failed
There is no need to associate libxlDomainObjPrivate objects with libxl's
desire to use libvirt's event loop. Instead, the driver-wide libxl_ctx can
be used for the fd and timer registrations.
This patch moves the fd and timer handling code away from the
domain-specific code in libxl_domain.c into libxl_driver.c. While at it,
function names were changed a bit to better describe their purpose.
The unnecessary locking was also removed since the code simply provides a
wrapper over the event loop interface. Indeed the locks may have been
causing some deadlocks when repeatedly creating/destroying muliple domains.
There have also been rumors about such deadlocks during parallel OpenStack
Tempest runs.
Signed-off-by: Jim Fehlig <jfehlig@suse.com>
---
src/libxl/libxl_domain.c | 234 +----------------------------------------------
src/libxl/libxl_driver.c | 201 +++++++++++++++++++++++++++++++++++++++-
2 files changed, 201 insertions(+), 234 deletions(-)
diff --git a/src/libxl/libxl_domain.c b/src/libxl/libxl_domain.c
index 856cfb4..c44799b 100644
--- a/src/libxl/libxl_domain.c
+++ b/src/libxl/libxl_domain.c
@@ -1,7 +1,7 @@
/*
* libxl_domain.c: libxl domain object private state
*
- * Copyright (C) 2011-2014 SUSE LINUX Products GmbH, Nuernberg, Germany.
+ * Copyright (C) 2011-2015 SUSE LINUX Products GmbH, Nuernberg, Germany.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -46,16 +46,6 @@ VIR_ENUM_IMPL(libxlDomainJob, LIBXL_JOB_LAST,
"modify",
);
-/* Object used to store info related to libxl event registrations */
-typedef struct _libxlEventHookInfo libxlEventHookInfo;
-typedef libxlEventHookInfo *libxlEventHookInfoPtr;
-struct _libxlEventHookInfo {
- libxlEventHookInfoPtr next;
- libxlDomainObjPrivatePtr priv;
- void *xl_priv;
- int id;
-};
-
static virClassPtr libxlDomainObjPrivateClass;
static void
@@ -75,227 +65,6 @@ libxlDomainObjPrivateOnceInit(void)
VIR_ONCE_GLOBAL_INIT(libxlDomainObjPrivate)
-static void
-libxlDomainObjFDEventHookInfoFree(void *obj)
-{
- VIR_FREE(obj);
-}
-
-static void
-libxlDomainObjTimerEventHookInfoFree(void *obj)
-{
- libxlEventHookInfoPtr info = obj;
-
- /* Drop reference on libxlDomainObjPrivate */
- virObjectUnref(info->priv);
- VIR_FREE(info);
-}
-
-static void
-libxlDomainObjFDEventCallback(int watch ATTRIBUTE_UNUSED,
- int fd,
- int vir_events,
- void *fd_info)
-{
- libxlEventHookInfoPtr info = fd_info;
- int events = 0;
-
- virObjectLock(info->priv);
- if (vir_events & VIR_EVENT_HANDLE_READABLE)
- events |= POLLIN;
- if (vir_events & VIR_EVENT_HANDLE_WRITABLE)
- events |= POLLOUT;
- if (vir_events & VIR_EVENT_HANDLE_ERROR)
- events |= POLLERR;
- if (vir_events & VIR_EVENT_HANDLE_HANGUP)
- events |= POLLHUP;
-
- virObjectUnlock(info->priv);
- libxl_osevent_occurred_fd(info->priv->ctx, info->xl_priv, fd, 0, events);
-}
-
-static int
-libxlDomainObjFDRegisterEventHook(void *priv,
- int fd,
- void **hndp,
- short events,
- void *xl_priv)
-{
- int vir_events = VIR_EVENT_HANDLE_ERROR;
- libxlEventHookInfoPtr info;
-
- if (VIR_ALLOC(info) < 0)
- return -1;
-
- info->priv = priv;
- info->xl_priv = xl_priv;
-
- if (events & POLLIN)
- vir_events |= VIR_EVENT_HANDLE_READABLE;
- if (events & POLLOUT)
- vir_events |= VIR_EVENT_HANDLE_WRITABLE;
-
- info->id = virEventAddHandle(fd, vir_events, libxlDomainObjFDEventCallback,
- info, libxlDomainObjFDEventHookInfoFree);
- if (info->id < 0) {
- VIR_FREE(info);
- return -1;
- }
-
- *hndp = info;
-
- return 0;
-}
-
-static int
-libxlDomainObjFDModifyEventHook(void *priv ATTRIBUTE_UNUSED,
- int fd ATTRIBUTE_UNUSED,
- void **hndp,
- short events)
-{
- libxlEventHookInfoPtr info = *hndp;
- int vir_events = VIR_EVENT_HANDLE_ERROR;
-
- virObjectLock(info->priv);
- if (events & POLLIN)
- vir_events |= VIR_EVENT_HANDLE_READABLE;
- if (events & POLLOUT)
- vir_events |= VIR_EVENT_HANDLE_WRITABLE;
-
- virEventUpdateHandle(info->id, vir_events);
- virObjectUnlock(info->priv);
-
- return 0;
-}
-
-static void
-libxlDomainObjFDDeregisterEventHook(void *priv ATTRIBUTE_UNUSED,
- int fd ATTRIBUTE_UNUSED,
- void *hnd)
-{
- libxlEventHookInfoPtr info = hnd;
- libxlDomainObjPrivatePtr p = info->priv;
-
- virObjectLock(p);
- virEventRemoveHandle(info->id);
- virObjectUnlock(p);
-}
-
-static void
-libxlDomainObjTimerCallback(int timer ATTRIBUTE_UNUSED, void *timer_info)
-{
- libxlEventHookInfoPtr info = timer_info;
- libxlDomainObjPrivatePtr p = info->priv;
-
- virObjectLock(p);
- /*
- * libxl expects the event to be deregistered when calling
- * libxl_osevent_occurred_timeout, but we dont want the event info
- * destroyed. Disable the timeout and only remove it after returning
- * from libxl.
- */
- virEventUpdateTimeout(info->id, -1);
- virObjectUnlock(p);
- libxl_osevent_occurred_timeout(p->ctx, info->xl_priv);
- virObjectLock(p);
- virEventRemoveTimeout(info->id);
- virObjectUnlock(p);
-}
-
-static int
-libxlDomainObjTimeoutRegisterEventHook(void *priv,
- void **hndp,
- struct timeval abs_t,
- void *xl_priv)
-{
- libxlEventHookInfoPtr info;
- struct timeval now;
- struct timeval res;
- static struct timeval zero;
- int timeout;
-
- if (VIR_ALLOC(info) < 0)
- return -1;
-
- info->priv = priv;
- /*
- * Also take a reference on the domain object. Reference is dropped in
- * libxlDomainObjEventHookInfoFree, ensuring the domain object outlives the
- * timeout event objects.
- */
- virObjectRef(info->priv);
- info->xl_priv = xl_priv;
-
- gettimeofday(&now, NULL);
- timersub(&abs_t, &now, &res);
- /* Ensure timeout is not overflowed */
- if (timercmp(&res, &zero, <)) {
- timeout = 0;
- } else if (res.tv_sec > INT_MAX / 1000) {
- timeout = INT_MAX;
- } else {
- timeout = res.tv_sec * 1000 + (res.tv_usec + 999) / 1000;
- }
- info->id = virEventAddTimeout(timeout, libxlDomainObjTimerCallback,
- info, libxlDomainObjTimerEventHookInfoFree);
- if (info->id < 0) {
- virObjectUnref(info->priv);
- VIR_FREE(info);
- return -1;
- }
-
- *hndp = info;
-
- return 0;
-}
-
-/*
- * Note: There are two changes wrt timeouts starting with xen-unstable
- * changeset 26469:
- *
- * 1. Timeout modify callbacks will only be invoked with an abs_t of {0,0},
- * i.e. make the timeout fire immediately. Prior to this commit, timeout
- * modify callbacks were never invoked.
- *
- * 2. Timeout deregister hooks will no longer be called.
- */
-static int
-libxlDomainObjTimeoutModifyEventHook(void *priv ATTRIBUTE_UNUSED,
- void **hndp,
- struct timeval abs_t ATTRIBUTE_UNUSED)
-{
- libxlEventHookInfoPtr info = *hndp;
-
- virObjectLock(info->priv);
- /* Make the timeout fire */
- virEventUpdateTimeout(info->id, 0);
- virObjectUnlock(info->priv);
-
- return 0;
-}
-
-static void
-libxlDomainObjTimeoutDeregisterEventHook(void *priv ATTRIBUTE_UNUSED,
- void *hnd)
-{
- libxlEventHookInfoPtr info = hnd;
- libxlDomainObjPrivatePtr p = info->priv;
-
- virObjectLock(p);
- virEventRemoveTimeout(info->id);
- virObjectUnlock(p);
-}
-
-
-static const libxl_osevent_hooks libxl_event_callbacks = {
- .fd_register = libxlDomainObjFDRegisterEventHook,
- .fd_modify = libxlDomainObjFDModifyEventHook,
- .fd_deregister = libxlDomainObjFDDeregisterEventHook,
- .timeout_register = libxlDomainObjTimeoutRegisterEventHook,
- .timeout_modify = libxlDomainObjTimeoutModifyEventHook,
- .timeout_deregister = libxlDomainObjTimeoutDeregisterEventHook,
-};
-
static int
libxlDomainObjInitJob(libxlDomainObjPrivatePtr priv)
{
@@ -796,7 +565,6 @@ libxlDomainObjPrivateInitCtx(virDomainObjPtr vm)
goto cleanup;
}
- libxl_osevent_register_hooks(priv->ctx, &libxl_event_callbacks, priv);
libxl_childproc_setmode(priv->ctx, &libxl_child_hooks, priv);
ret = 0;
diff --git a/src/libxl/libxl_driver.c b/src/libxl/libxl_driver.c
index c95b387..00dab98 100644
--- a/src/libxl/libxl_driver.c
+++ b/src/libxl/libxl_driver.c
@@ -2,7 +2,7 @@
* libxl_driver.c: core driver methods for managing libxenlight domains
*
* Copyright (C) 2006-2014 Red Hat, Inc.
- * Copyright (C) 2011-2014 SUSE LINUX Products GmbH, Nuernberg, Germany.
+ * Copyright (C) 2011-2015 SUSE LINUX Products GmbH, Nuernberg, Germany.
* Copyright (C) 2011 Univention GmbH.
*
* This library is free software; you can redistribute it and/or
@@ -80,6 +80,15 @@ VIR_LOG_INIT("libxl.libxl_driver");
static libxlDriverPrivatePtr libxl_driver;
+/* Object used to store info related to libxl event registrations */
+typedef struct _libxlOSEventHookInfo libxlOSEventHookInfo;
+typedef libxlOSEventHookInfo *libxlOSEventHookInfoPtr;
+struct _libxlOSEventHookInfo {
+ libxl_ctx *ctx;
+ void *xl_priv;
+ int id;
+};
+
/* Function declarations */
static int
libxlDomainManagedSaveLoad(virDomainObjPtr vm,
@@ -87,6 +96,183 @@ libxlDomainManagedSaveLoad(virDomainObjPtr vm,
/* Function definitions */
+static void
+libxlOSEventHookInfoFree(void *obj)
+{
+ VIR_FREE(obj);
+}
+
+static void
+libxlFDEventCallback(int watch ATTRIBUTE_UNUSED,
+ int fd,
+ int vir_events,
+ void *fd_info)
+{
+ libxlOSEventHookInfoPtr info = fd_info;
+ int events = 0;
+
+ if (vir_events & VIR_EVENT_HANDLE_READABLE)
+ events |= POLLIN;
+ if (vir_events & VIR_EVENT_HANDLE_WRITABLE)
+ events |= POLLOUT;
+ if (vir_events & VIR_EVENT_HANDLE_ERROR)
+ events |= POLLERR;
+ if (vir_events & VIR_EVENT_HANDLE_HANGUP)
+ events |= POLLHUP;
+
+ libxl_osevent_occurred_fd(info->ctx, info->xl_priv, fd, 0, events);
+}
+
+static int
+libxlFDRegisterEventHook(void *priv,
+ int fd,
+ void **hndp,
+ short events,
+ void *xl_priv)
+{
+ int vir_events = VIR_EVENT_HANDLE_ERROR;
+ libxlOSEventHookInfoPtr info;
+
+ if (VIR_ALLOC(info) < 0)
+ return -1;
+
+ info->ctx = priv;
+ info->xl_priv = xl_priv;
+
+ if (events & POLLIN)
+ vir_events |= VIR_EVENT_HANDLE_READABLE;
+ if (events & POLLOUT)
+ vir_events |= VIR_EVENT_HANDLE_WRITABLE;
+
+ info->id = virEventAddHandle(fd, vir_events, libxlFDEventCallback,
+ info, libxlOSEventHookInfoFree);
+ if (info->id < 0) {
+ VIR_FREE(info);
+ return -1;
+ }
+
+ *hndp = info;
+
+ return 0;
+}
+
+static int
+libxlFDModifyEventHook(void *priv ATTRIBUTE_UNUSED,
+ int fd ATTRIBUTE_UNUSED,
+ void **hndp,
+ short events)
+{
+ libxlOSEventHookInfoPtr info = *hndp;
+ int vir_events = VIR_EVENT_HANDLE_ERROR;
+
+ if (events & POLLIN)
+ vir_events |= VIR_EVENT_HANDLE_READABLE;
+ if (events & POLLOUT)
+ vir_events |= VIR_EVENT_HANDLE_WRITABLE;
+
+ virEventUpdateHandle(info->id, vir_events);
+
+ return 0;
+}
+
+static void
+libxlFDDeregisterEventHook(void *priv ATTRIBUTE_UNUSED,
+ int fd ATTRIBUTE_UNUSED,
+ void *hnd)
+{
+ libxlOSEventHookInfoPtr info = hnd;
+
+ virEventRemoveHandle(info->id);
+}
+
+static void
+libxlTimerCallback(int timer ATTRIBUTE_UNUSED, void *timer_info)
+{
+ libxlOSEventHookInfoPtr info = timer_info;
+
+ /*
+ * libxl expects the event to be deregistered when calling
+ * libxl_osevent_occurred_timeout, but we dont want the event info
+ * destroyed. Disable the timeout and only remove it after returning
+ * from libxl.
+ */
+ virEventUpdateTimeout(info->id, -1);
+ libxl_osevent_occurred_timeout(info->ctx, info->xl_priv);
+ virEventRemoveTimeout(info->id);
+}
+
+static int
+libxlTimeoutRegisterEventHook(void *priv,
+ void **hndp,
+ struct timeval abs_t,
+ void *xl_priv)
+{
+ libxlOSEventHookInfoPtr info;
+ struct timeval now;
+ struct timeval res;
+ static struct timeval zero;
+ int timeout;
+
+ if (VIR_ALLOC(info) < 0)
+ return -1;
+
+ info->ctx = priv;
+ info->xl_priv = xl_priv;
+
+ gettimeofday(&now, NULL);
+ timersub(&abs_t, &now, &res);
+ /* Ensure timeout is not overflowed */
+ if (timercmp(&res, &zero, <)) {
+ timeout = 0;
+ } else if (res.tv_sec > INT_MAX / 1000) {
+ timeout = INT_MAX;
+ } else {
+ timeout = res.tv_sec * 1000 + (res.tv_usec + 999) / 1000;
+ }
+ info->id = virEventAddTimeout(timeout, libxlTimerCallback,
+ info, libxlOSEventHookInfoFree);
+ if (info->id < 0) {
+ VIR_FREE(info);
+ return -1;
+ }
+
+ *hndp = info;
+
+ return 0;
+}
+
+/*
+ * Note: There are two changes wrt timeouts starting with xen-unstable
+ * changeset 26469:
+ *
+ * 1. Timeout modify callbacks will only be invoked with an abs_t of {0,0},
+ * i.e. make the timeout fire immediately. Prior to this commit, timeout
+ * modify callbacks were never invoked.
+ *
+ * 2. Timeout deregister hooks will no longer be called.
+ */
+static int
+libxlTimeoutModifyEventHook(void *priv ATTRIBUTE_UNUSED,
+ void **hndp,
+ struct timeval abs_t ATTRIBUTE_UNUSED)
+{
+ libxlOSEventHookInfoPtr info = *hndp;
+
+ /* Make the timeout fire */
+ virEventUpdateTimeout(info->id, 0);
+
+ return 0;
+}
+
+static void
+libxlTimeoutDeregisterEventHook(void *priv ATTRIBUTE_UNUSED,
+ void *hnd)
+{
+ libxlOSEventHookInfoPtr info = hnd;
+
+ virEventRemoveTimeout(info->id);
+}
+
static virDomainObjPtr
libxlDomObjFromDomain(virDomainPtr dom)
{
@@ -277,6 +463,16 @@ libxlDriverShouldLoad(bool privileged)
return ret;
}
+/* Callbacks wrapping libvirt's event loop interface */
+static const libxl_osevent_hooks libxl_osevent_callbacks = {
+ .fd_register = libxlFDRegisterEventHook,
+ .fd_modify = libxlFDModifyEventHook,
+ .fd_deregister = libxlFDDeregisterEventHook,
+ .timeout_register = libxlTimeoutRegisterEventHook,
+ .timeout_modify = libxlTimeoutModifyEventHook,
+ .timeout_deregister = libxlTimeoutDeregisterEventHook,
+};
+
static int
libxlStateInitialize(bool privileged,
virStateInhibitCallback callback ATTRIBUTE_UNUSED,
@@ -322,6 +518,9 @@ libxlStateInitialize(bool privileged,
if (!(cfg = libxlDriverConfigNew()))
goto error;
+ /* Register the callbacks providing access to libvirt's event loop */
+ libxl_osevent_register_hooks(cfg->ctx, &libxl_osevent_callbacks, cfg->ctx);
+
libxl_driver->config = cfg;
if (virFileMakePath(cfg->stateDir) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
--
1.8.4.5
^ permalink raw reply related [flat|nested] 6+ messages in thread* Re: [PATCH 0/2] libxl: fix handling of fd and timer registrations
2015-02-03 0:00 [PATCH 0/2] libxl: fix handling of fd and timer registrations Jim Fehlig
2015-02-03 0:00 ` [PATCH 1/2] libxl: fix fd and timer event handling Jim Fehlig
2015-02-03 0:00 ` [PATCH 2/2] libxl: Move setup of child processing code to driver initialization Jim Fehlig
@ 2015-02-06 12:32 ` Anthony PERARD
2015-02-09 16:21 ` Jim Fehlig
2 siblings, 1 reply; 6+ messages in thread
From: Anthony PERARD @ 2015-02-06 12:32 UTC (permalink / raw)
To: Jim Fehlig; +Cc: libvir-list, xen-devel
[-- Attachment #1: Type: text/plain, Size: 1475 bytes --]
On Mon, Feb 02, 2015 at 05:00:34PM -0700, Jim Fehlig wrote:
> This small series fixes some assertions we occasionally see in the
> libxl driver when running libvirt-TCK. The assertions were due to
> races between destroying per-domain libxl_ctx and receiving fd and
> timer callbacks associated with them. The races are masked by
> setting DEBUG loglevel in libvirtd.conf, so often missed by
> automated test setups that want DEBUG loglevel.
>
> Patch 1 actually fixes the assertions. Patch2 fixes a stupid mistake.
> See the commit messages for details.
>
> Jim Fehlig (2):
> libxl: fix fd and timer event handling
> libxl: Move setup of child processing code to driver initialization
>
> src/libxl/libxl_domain.c | 244 +----------------------------------------------
> src/libxl/libxl_driver.c | 212 +++++++++++++++++++++++++++++++++++++++-
> 2 files changed, 212 insertions(+), 244 deletions(-)
Hi Jim,
I gave a try to those two patches with OpenStack. Assuming I haven't make any
mistake, it make things worse.
Environment:
Ubuntu 14.04
with Xen package install (xen 4.4)
libvirt: master (47dd6c4)
Installed OpenStack via DevStack
Test: ./run_tempest.sh tempest.api.compute
Result:
without the patches, the tests run fine, they all succeed.
with the patches, the tests fail AND libvirt became unresponsible.
Running `virsh -c xen: list` does not return. (or any virsh command)
I have attach a backtrace, if that can help.
--
Anthony PERARD
[-- Attachment #2: libvirt-bt.log --]
[-- Type: text/plain, Size: 62289 bytes --]
Thread 30 (Thread 0x7fa9cad9f700 (LWP 5872)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cfef8001 in virDomainObjListFindByName (doms=0x7fa9b8189460, name=0x7fa9a000d5e0 "instance-00000020") at conf/domain_conf.c:1114
#6 0x00007fa9c118b786 in libxlDomainLookupByName (conn=0x7fa9a0007630, name=<optimised out>) at libxl/libxl_driver.c:954
#7 0x00007fa9cff71356 in virDomainLookupByName (conn=0x7fa9a0007630, name=0x7fa9a000d5e0 "instance-00000020") at libvirt-domain.c:427
#8 0x00007fa9d16fd492 in remoteDispatchDomainLookupByName (server=<optimised out>, msg=<optimised out>, ret=0x7fa9a0006770, args=0x7fa9a0000f90, rerr=0x7fa9cad9ec40,
client=0x7fa9d36470d0) at remote_dispatch.h:5429
#9 remoteDispatchDomainLookupByNameHelper (server=<optimised out>, client=0x7fa9d36470d0, msg=<optimised out>, rerr=0x7fa9cad9ec40, args=0x7fa9a0000f90, ret=0x7fa9a0006770)
at remote_dispatch.h:5409
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3645fb0, client=0x7fa9d36470d0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36470d0, msg=0x7fa9d3645fb0) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d361c5f0) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9cad9f700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 29 (Thread 0x7fa9ca59e700 (LWP 5873)):
#0 0x00007fa9cf91dcbd in poll () at ../sysdeps/unix/syscall-template.S:81
#1 0x00007fa9cd452fff in ?? () from /usr/lib/libxenlight-4.4.so
#2 0x00007fa9cd45441b in ?? () from /usr/lib/libxenlight-4.4.so
#3 0x00007fa9cd438d27 in ?? () from /usr/lib/libxenlight-4.4.so
#4 0x00007fa9cd439228 in libxl_domain_create_new () from /usr/lib/libxenlight-4.4.so
#5 0x00007fa9c11846d8 in libxlDomainStart (driver=driver@entry=0x7fa9b818ff60, vm=vm@entry=0x7fa9b80f0770, start_paused=start_paused@entry=false,
restore_fd=restore_fd@entry=-1) at libxl/libxl_domain.c:1044
#6 0x00007fa9c1188134 in libxlDomainCreateWithFlags (dom=0x7fa9b0004440, flags=0) at libxl/libxl_driver.c:2587
#7 0x00007fa9cff8466d in virDomainCreateWithFlags (domain=domain@entry=0x7fa9b0004440, flags=0) at libvirt-domain.c:6898
#8 0x00007fa9d16ee4a8 in remoteDispatchDomainCreateWithFlags (server=<optimised out>, msg=<optimised out>, ret=0x7fa9b000a560, args=0x7fa9b00031d0, rerr=0x7fa9ca59dc40,
client=<optimised out>) at remote_dispatch.h:3252
#9 remoteDispatchDomainCreateWithFlagsHelper (server=<optimised out>, client=<optimised out>, msg=<optimised out>, rerr=0x7fa9ca59dc40, args=0x7fa9b00031d0,
ret=0x7fa9b000a560) at remote_dispatch.h:3229
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3644440, client=0x7fa9d36429b0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36429b0, msg=0x7fa9d3644440) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d361c490) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9ca59e700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 28 (Thread 0x7fa9c9d9d700 (LWP 5874)):
#0 0x00007fa9cf91dcbd in poll () at ../sysdeps/unix/syscall-template.S:81
#1 0x00007fa9cd452fff in ?? () from /usr/lib/libxenlight-4.4.so
#2 0x00007fa9cd45441b in ?? () from /usr/lib/libxenlight-4.4.so
#3 0x00007fa9cd438d27 in ?? () from /usr/lib/libxenlight-4.4.so
#4 0x00007fa9c11842c9 in libxlDomainStart (driver=driver@entry=0x7fa9b818ff60, vm=vm@entry=0x7fa9b0001900, start_paused=start_paused@entry=false, restore_fd=89,
restore_fd@entry=-1) at libxl/libxl_domain.c:1049
#5 0x00007fa9c1188134 in libxlDomainCreateWithFlags (dom=0x7fa9b40034f0, flags=0) at libxl/libxl_driver.c:2587
#6 0x00007fa9cff8466d in virDomainCreateWithFlags (domain=domain@entry=0x7fa9b40034f0, flags=0) at libvirt-domain.c:6898
#7 0x00007fa9d16ee4a8 in remoteDispatchDomainCreateWithFlags (server=<optimised out>, msg=<optimised out>, ret=0x7fa9b4007930, args=0x7fa9b4004610, rerr=0x7fa9c9d9cc40,
client=<optimised out>) at remote_dispatch.h:3252
#8 remoteDispatchDomainCreateWithFlagsHelper (server=<optimised out>, client=<optimised out>, msg=<optimised out>, rerr=0x7fa9c9d9cc40, args=0x7fa9b4004610,
ret=0x7fa9b4007930) at remote_dispatch.h:3229
#9 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3647270, client=0x7fa9d36429b0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#10 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36429b0, msg=0x7fa9d3647270) at rpc/virnetserverprogram.c:307
#11 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#12 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#13 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d361c330) at util/virthreadpool.c:144
#14 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#15 0x00007fa9cfc00182 in start_thread (arg=0x7fa9c9d9d700) at pthread_create.c:312
#16 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 27 (Thread 0x7fa9c959c700 (LWP 5875)):
#0 0x00007fa9cf91dcbd in poll () at ../sysdeps/unix/syscall-template.S:81
#1 0x00007fa9cd452fff in ?? () from /usr/lib/libxenlight-4.4.so
#2 0x00007fa9cd45441b in ?? () from /usr/lib/libxenlight-4.4.so
#3 0x00007fa9cd438d27 in ?? () from /usr/lib/libxenlight-4.4.so
#4 0x00007fa9cd439228 in libxl_domain_create_new () from /usr/lib/libxenlight-4.4.so
#5 0x00007fa9c11846d8 in libxlDomainStart (driver=driver@entry=0x7fa9b818ff60, vm=vm@entry=0x7fa9b4003ec0, start_paused=start_paused@entry=false,
restore_fd=restore_fd@entry=-1) at libxl/libxl_domain.c:1044
#6 0x00007fa9c1188134 in libxlDomainCreateWithFlags (dom=0x7fa9a80038a0, flags=0) at libxl/libxl_driver.c:2587
#7 0x00007fa9cff8466d in virDomainCreateWithFlags (domain=domain@entry=0x7fa9a80038a0, flags=0) at libvirt-domain.c:6898
#8 0x00007fa9d16ee4a8 in remoteDispatchDomainCreateWithFlags (server=<optimised out>, msg=<optimised out>, ret=0x7fa9a8002430, args=0x7fa9a8004590, rerr=0x7fa9c959bc40,
client=<optimised out>) at remote_dispatch.h:3252
#9 remoteDispatchDomainCreateWithFlagsHelper (server=<optimised out>, client=<optimised out>, msg=<optimised out>, rerr=0x7fa9c959bc40, args=0x7fa9a8004590,
ret=0x7fa9a8002430) at remote_dispatch.h:3229
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d36447b0, client=0x7fa9d36429b0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36429b0, msg=0x7fa9d36447b0) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d361c490) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9c959c700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 26 (Thread 0x7fa9c8d9b700 (LWP 5876)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b0008e90) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b0008e90) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b0008e80) at util/virobject.c:323
#5 0x00007fa9cff0c418 in virDomainListPopulate (payload=0x7fa9b0008e80, name=<optimised out>, opaque=0x7fa9c8d9aa60) at conf/domain_conf.c:21474
#6 0x00007fa9cfe9fccc in virHashForEach (table=0x7fa9b81894b0, iter=iter@entry=0x7fa9cff0c3d0 <virDomainListPopulate>, data=data@entry=0x7fa9c8d9aa60) at util/virhash.c:521
#7 0x00007fa9cff20e32 in virDomainObjListExport (doms=0x7fa9b8189460, conn=<optimised out>, domains=0x7fa9c8d9ab40, filter=<optimised out>, flags=<optimised out>)
at conf/domain_conf.c:21600
#8 0x00007fa9cff842b3 in virConnectListAllDomains (conn=0x7fa9b40009a0, domains=0x7fa9c8d9ab40, flags=3) at libvirt-domain.c:6794
#9 0x00007fa9d16e646f in remoteDispatchConnectListAllDomains (server=<optimised out>, msg=<optimised out>, ret=0x7fa9ac00f4e0, args=0x7fa9ac00c740, rerr=0x7fa9c8d9ac40,
client=0x7fa9d36429b0) at remote.c:1580
#10 remoteDispatchConnectListAllDomainsHelper (server=<optimised out>, client=0x7fa9d36429b0, msg=<optimised out>, rerr=0x7fa9c8d9ac40, args=0x7fa9ac00c740,
ret=0x7fa9ac00f4e0) at remote_dispatch.h:1072
#11 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3641aa0, client=0x7fa9d36429b0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#12 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36429b0, msg=0x7fa9d3641aa0) at rpc/virnetserverprogram.c:307
#13 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#14 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#15 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d361c5f0) at util/virthreadpool.c:144
#16 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#17 0x00007fa9cfc00182 in start_thread (arg=0x7fa9c8d9b700) at pthread_create.c:312
#18 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 25 (Thread 0x7fa9c859a700 (LWP 5877)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cff0caf0 in virDomainObjListAdd (doms=0x7fa9b8189460, def=def@entry=0x7fa9b81a3810, xmlopt=0x7fa9b81a5740, flags=flags@entry=0,
oldDef=oldDef@entry=0x7fa9c8599a40) at conf/domain_conf.c:2453
#6 0x00007fa9c1187e67 in libxlDomainDefineXMLFlags (conn=0x7fa9a0007630,
xml=0x7fa9b80eec00 "<domain type=\"xen\">\n <uuid>0083e6c1-d8ca-4803-9c14-cca1317e2f1c</uuid>\n <name>instance-00000024</name>\n <memory>65536</memory>\n <vcpu cpuset=\"0-7\">1</vcpu>\n <metadata>\n <nova:instance xmlns:no"..., flags=<optimised out>) at libxl/libxl_driver.c:2626
#7 0x00007fa9cff836b4 in virDomainDefineXML (conn=0x7fa9a0007630,
xml=0x7fa9b80eec00 "<domain type=\"xen\">\n <uuid>0083e6c1-d8ca-4803-9c14-cca1317e2f1c</uuid>\n <name>instance-00000024</name>\n <memory>65536</memory>\n <vcpu cpuset=\"0-7\">1</vcpu>\n <metadata>\n <nova:instance xmlns:no"...) at libvirt-domain.c:6464
#8 0x00007fa9d16fe8e2 in remoteDispatchDomainDefineXML (server=<optimised out>, msg=<optimised out>, ret=0x7fa9b81a0ab0, args=0x7fa9b8150180, rerr=0x7fa9c8599c40,
client=0x7fa9d36470d0) at remote_dispatch.h:3376
#9 remoteDispatchDomainDefineXMLHelper (server=<optimised out>, client=0x7fa9d36470d0, msg=<optimised out>, rerr=0x7fa9c8599c40, args=0x7fa9b8150180, ret=0x7fa9b81a0ab0)
at remote_dispatch.h:3356
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3646120, client=0x7fa9d36470d0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36470d0, msg=0x7fa9d3646120) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d361c490) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9c859a700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 24 (Thread 0x7fa9c7d99700 (LWP 5878)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cff0bdf3 in virDomainObjListNumOfDomains (doms=0x7fa9b8189460, active=<optimised out>, filter=<optimised out>, conn=<optimised out>)
at conf/domain_conf.c:20606
#6 0x00007fa9cff705be in virConnectNumOfDomains (conn=0x7fa9a0007630) at libvirt-domain.c:97
#7 0x00007fa9d16faf39 in remoteDispatchConnectNumOfDomains (server=<optimised out>, msg=<optimised out>, ret=0x7fa990000b10, rerr=0x7fa9c7d98c40, client=0x7fa9d36470d0)
at remote_dispatch.h:2099
#8 remoteDispatchConnectNumOfDomainsHelper (server=<optimised out>, client=0x7fa9d36470d0, msg=<optimised out>, rerr=0x7fa9c7d98c40, args=<optimised out>,
ret=0x7fa990000b10) at remote_dispatch.h:2080
#9 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d36437b0, client=0x7fa9d36470d0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#10 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36470d0, msg=0x7fa9d36437b0) at rpc/virnetserverprogram.c:307
#11 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#12 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#13 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d361c5f0) at util/virthreadpool.c:144
#14 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#15 0x00007fa9cfc00182 in start_thread (arg=0x7fa9c7d99700) at pthread_create.c:312
#16 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 23 (Thread 0x7fa9c7598700 (LWP 5879)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cfef8001 in virDomainObjListFindByName (doms=0x7fa9b8189460, name=0x7fa99c0097e0 "instance-00000023") at conf/domain_conf.c:1114
#6 0x00007fa9c118b786 in libxlDomainLookupByName (conn=0x7fa9a4000b40, name=<optimised out>) at libxl/libxl_driver.c:954
#7 0x00007fa9cff71356 in virDomainLookupByName (conn=0x7fa9a4000b40, name=0x7fa99c0097e0 "instance-00000023") at libvirt-domain.c:427
#8 0x00007fa9d16fd492 in remoteDispatchDomainLookupByName (server=<optimised out>, msg=<optimised out>, ret=0x7fa99c001a40, args=0x7fa99c0031d0, rerr=0x7fa9c7597c40,
client=0x7fa9d364eb40) at remote_dispatch.h:5429
#9 remoteDispatchDomainLookupByNameHelper (server=<optimised out>, client=0x7fa9d364eb40, msg=<optimised out>, rerr=0x7fa9c7597c40, args=0x7fa99c0031d0, ret=0x7fa99c001a40)
at remote_dispatch.h:5409
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3644020, client=0x7fa9d364eb40, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d364eb40, msg=0x7fa9d3644020) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d361c490) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9c7598700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 22 (Thread 0x7fa9c6d97700 (LWP 5880)):
#0 0x00007fa9cf91dcbd in poll () at ../sysdeps/unix/syscall-template.S:81
#1 0x00007fa9cd452fff in ?? () from /usr/lib/libxenlight-4.4.so
#2 0x00007fa9cd45441b in ?? () from /usr/lib/libxenlight-4.4.so
#3 0x00007fa9cd42cc36 in libxl_domain_destroy () from /usr/lib/libxenlight-4.4.so
#4 0x00007fa9c118ac2f in libxlDomainDestroyFlags (dom=<optimised out>, flags=<optimised out>) at libxl/libxl_driver.c:1226
#5 0x00007fa9cff7157c in virDomainDestroy (domain=domain@entry=0x7fa998001b50) at libvirt-domain.c:481
#6 0x00007fa9d16fe7bc in remoteDispatchDomainDestroy (server=<optimised out>, msg=<optimised out>, args=<optimised out>, rerr=0x7fa9c6d96c40, client=0x7fa9d36429b0)
at remote_dispatch.h:3479
#7 remoteDispatchDomainDestroyHelper (server=<optimised out>, client=0x7fa9d36429b0, msg=<optimised out>, rerr=0x7fa9c6d96c40, args=<optimised out>, ret=<optimised out>)
at remote_dispatch.h:3457
#8 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d36483a0, client=0x7fa9d36429b0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#9 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36429b0, msg=0x7fa9d36483a0) at rpc/virnetserverprogram.c:307
#10 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#11 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#12 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d361c5f0) at util/virthreadpool.c:144
#13 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#14 0x00007fa9cfc00182 in start_thread (arg=0x7fa9c6d97700) at pthread_create.c:312
#15 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 21 (Thread 0x7fa9c6596700 (LWP 5881)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cfef8001 in virDomainObjListFindByName (doms=0x7fa9b8189460, name=0x7fa9a40008e0 "instance-00000025") at conf/domain_conf.c:1114
#6 0x00007fa9c118b786 in libxlDomainLookupByName (conn=0x7fa96c000b80, name=<optimised out>) at libxl/libxl_driver.c:954
#7 0x00007fa9cff71356 in virDomainLookupByName (conn=0x7fa96c000b80, name=0x7fa9a40008e0 "instance-00000025") at libvirt-domain.c:427
#8 0x00007fa9d16fd492 in remoteDispatchDomainLookupByName (server=<optimised out>, msg=<optimised out>, ret=0x7fa9a40011d0, args=0x7fa9a40099d0, rerr=0x7fa9c6595c40,
client=0x7fa9d36452c0) at remote_dispatch.h:5429
#9 remoteDispatchDomainLookupByNameHelper (server=<optimised out>, client=0x7fa9d36452c0, msg=<optimised out>, rerr=0x7fa9c6595c40, args=0x7fa9a40099d0, ret=0x7fa9a40011d0)
at remote_dispatch.h:5409
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3645250, client=0x7fa9d36452c0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36452c0, msg=0x7fa9d3645250) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d361c490) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9c6596700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 20 (Thread 0x7fa9d1693700 (LWP 23669)):
#0 0x00007fa9cfc073bd in read () at ../sysdeps/unix/syscall-template.S:81
#1 0x00007fa9cbe4280d in ?? () from /usr/lib/libxenstore.so.3.0
#2 0x00007fa9cbe428bc in ?? () from /usr/lib/libxenstore.so.3.0
#3 0x00007fa9cbe42bc1 in ?? () from /usr/lib/libxenstore.so.3.0
#4 0x00007fa9cfc00182 in start_thread (arg=0x7fa9d1693700) at pthread_create.c:312
#5 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 19 (Thread 0x7fa9d168a700 (LWP 23712)):
#0 0x00007fa9cfc073bd in read () at ../sysdeps/unix/syscall-template.S:81
#1 0x00007fa9cbe4280d in ?? () from /usr/lib/libxenstore.so.3.0
#2 0x00007fa9cbe428bc in ?? () from /usr/lib/libxenstore.so.3.0
#3 0x00007fa9cbe42bc1 in ?? () from /usr/lib/libxenstore.so.3.0
#4 0x00007fa9cfc00182 in start_thread (arg=0x7fa9d168a700) at pthread_create.c:312
#5 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 18 (Thread 0x7fa9d166f700 (LWP 24994)):
#0 0x00007fa9cfc073bd in read () at ../sysdeps/unix/syscall-template.S:81
#1 0x00007fa9cbe4280d in ?? () from /usr/lib/libxenstore.so.3.0
#2 0x00007fa9cbe428bc in ?? () from /usr/lib/libxenstore.so.3.0
#3 0x00007fa9cbe42bc1 in ?? () from /usr/lib/libxenstore.so.3.0
#4 0x00007fa9cfc00182 in start_thread (arg=0x7fa9d166f700) at pthread_create.c:312
#5 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 17 (Thread 0x7fa9d165c700 (LWP 27073)):
#0 0x00007fa9cfc073bd in read () at ../sysdeps/unix/syscall-template.S:81
#1 0x00007fa9cbe4280d in ?? () from /usr/lib/libxenstore.so.3.0
#2 0x00007fa9cbe428bc in ?? () from /usr/lib/libxenstore.so.3.0
#3 0x00007fa9cbe42bc1 in ?? () from /usr/lib/libxenstore.so.3.0
#4 0x00007fa9cfc00182 in start_thread (arg=0x7fa9d165c700) at pthread_create.c:312
#5 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 16 (Thread 0x7fa9bfb9f700 (LWP 28515)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cfef8001 in virDomainObjListFindByName (doms=0x7fa9b8189460, name=0x7fa980001a10 "instance-00000023") at conf/domain_conf.c:1114
#6 0x00007fa9c118b786 in libxlDomainLookupByName (conn=0x7fa9a0007630, name=<optimised out>) at libxl/libxl_driver.c:954
#7 0x00007fa9cff71356 in virDomainLookupByName (conn=0x7fa9a0007630, name=0x7fa980001a10 "instance-00000023") at libvirt-domain.c:427
#8 0x00007fa9d16fd492 in remoteDispatchDomainLookupByName (server=<optimised out>, msg=<optimised out>, ret=0x7fa980000ed0, args=0x7fa980001a30, rerr=0x7fa9bfb9ec40,
client=0x7fa9d36470d0) at remote_dispatch.h:5429
#9 remoteDispatchDomainLookupByNameHelper (server=<optimised out>, client=0x7fa9d36470d0, msg=<optimised out>, rerr=0x7fa9bfb9ec40, args=0x7fa980001a30, ret=0x7fa980000ed0)
at remote_dispatch.h:5409
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3644820, client=0x7fa9d36470d0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36470d0, msg=0x7fa9d3644820) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d361e380) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9bfb9f700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 15 (Thread 0x7fa9bf39e700 (LWP 28518)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cfef8001 in virDomainObjListFindByName (doms=0x7fa9b8189460, name=0x7fa97c000b90 "instance-0000001f") at conf/domain_conf.c:1114
#6 0x00007fa9c118b786 in libxlDomainLookupByName (conn=0x7fa9a0007630, name=<optimised out>) at libxl/libxl_driver.c:954
#7 0x00007fa9cff71356 in virDomainLookupByName (conn=0x7fa9a0007630, name=0x7fa97c000b90 "instance-0000001f") at libvirt-domain.c:427
#8 0x00007fa9d16fd492 in remoteDispatchDomainLookupByName (server=<optimised out>, msg=<optimised out>, ret=0x7fa97c000ac0, args=0x7fa97c000b70, rerr=0x7fa9bf39dc40,
client=0x7fa9d36470d0) at remote_dispatch.h:5429
#9 remoteDispatchDomainLookupByNameHelper (server=<optimised out>, client=0x7fa9d36470d0, msg=<optimised out>, rerr=0x7fa9bf39dc40, args=0x7fa97c000b70, ret=0x7fa97c000ac0)
at remote_dispatch.h:5409
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d364f440, client=0x7fa9d36470d0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36470d0, msg=0x7fa9d364f440) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d3647250) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9bf39e700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 14 (Thread 0x7fa9beb9d700 (LWP 28610)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cfef8001 in virDomainObjListFindByName (doms=0x7fa9b8189460, name=0x7fa984001ba0 "instance-00000024") at conf/domain_conf.c:1114
#6 0x00007fa9c118b786 in libxlDomainLookupByName (conn=0x7fa9a4000b40, name=<optimised out>) at libxl/libxl_driver.c:954
#7 0x00007fa9cff71356 in virDomainLookupByName (conn=0x7fa9a4000b40, name=0x7fa984001ba0 "instance-00000024") at libvirt-domain.c:427
#8 0x00007fa9d16fd492 in remoteDispatchDomainLookupByName (server=<optimised out>, msg=<optimised out>, ret=0x7fa984000b10, args=0x7fa984001b80, rerr=0x7fa9beb9cc40,
client=0x7fa9d364eb40) at remote_dispatch.h:5429
#9 remoteDispatchDomainLookupByNameHelper (server=<optimised out>, client=0x7fa9d364eb40, msg=<optimised out>, rerr=0x7fa9beb9cc40, args=0x7fa984001b80, ret=0x7fa984000b10)
at remote_dispatch.h:5409
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3643700, client=0x7fa9d364eb40, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d364eb40, msg=0x7fa9d3643700) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d3619740) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9beb9d700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 13 (Thread 0x7fa9be39c700 (LWP 29578)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cfef8001 in virDomainObjListFindByName (doms=0x7fa9b8189460, name=0x7fa994000ac0 "instance-0000001f") at conf/domain_conf.c:1114
#6 0x00007fa9c118b786 in libxlDomainLookupByName (conn=0x7fa9a4000b40, name=<optimised out>) at libxl/libxl_driver.c:954
#7 0x00007fa9cff71356 in virDomainLookupByName (conn=0x7fa9a4000b40, name=0x7fa994000ac0 "instance-0000001f") at libvirt-domain.c:427
#8 0x00007fa9d16fd492 in remoteDispatchDomainLookupByName (server=<optimised out>, msg=<optimised out>, ret=0x7fa994000a70, args=0x7fa994000aa0, rerr=0x7fa9be39bc40,
client=0x7fa9d364eb40) at remote_dispatch.h:5429
#9 remoteDispatchDomainLookupByNameHelper (server=<optimised out>, client=0x7fa9d364eb40, msg=<optimised out>, rerr=0x7fa9be39bc40, args=0x7fa994000aa0, ret=0x7fa994000a70)
at remote_dispatch.h:5409
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d36492c0, client=0x7fa9d364eb40, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d364eb40, msg=0x7fa9d36492c0) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d3620d80) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9be39c700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 12 (Thread 0x7fa9bdb9b700 (LWP 29584)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cfef8001 in virDomainObjListFindByName (doms=0x7fa9b8189460, name=0x7fa9740008c0 "instance-00000020") at conf/domain_conf.c:1114
#6 0x00007fa9c118b786 in libxlDomainLookupByName (conn=0x7fa9a4000b40, name=<optimised out>) at libxl/libxl_driver.c:954
#7 0x00007fa9cff71356 in virDomainLookupByName (conn=0x7fa9a4000b40, name=0x7fa9740008c0 "instance-00000020") at libvirt-domain.c:427
#8 0x00007fa9d16fd492 in remoteDispatchDomainLookupByName (server=<optimised out>, msg=<optimised out>, ret=0x7fa974000960, args=0x7fa9740008e0, rerr=0x7fa9bdb9ac40,
client=0x7fa9d364eb40) at remote_dispatch.h:5429
#9 remoteDispatchDomainLookupByNameHelper (server=<optimised out>, client=0x7fa9d364eb40, msg=<optimised out>, rerr=0x7fa9bdb9ac40, args=0x7fa9740008e0, ret=0x7fa974000960)
at remote_dispatch.h:5409
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d36440c0, client=0x7fa9d364eb40, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d364eb40, msg=0x7fa9d36440c0) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d3620440) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9bdb9b700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 11 (Thread 0x7fa9bd39a700 (LWP 29585)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cff0bdf3 in virDomainObjListNumOfDomains (doms=0x7fa9b8189460, active=<optimised out>, filter=<optimised out>, conn=<optimised out>)
at conf/domain_conf.c:20606
#6 0x00007fa9cff705be in virConnectNumOfDomains (conn=0x7fa9a4000b40) at libvirt-domain.c:97
#7 0x00007fa9d16faf39 in remoteDispatchConnectNumOfDomains (server=<optimised out>, msg=<optimised out>, ret=0x7fa9680008e0, rerr=0x7fa9bd399c40, client=0x7fa9d364eb40)
at remote_dispatch.h:2099
#8 remoteDispatchConnectNumOfDomainsHelper (server=<optimised out>, client=0x7fa9d364eb40, msg=<optimised out>, rerr=0x7fa9bd399c40, args=<optimised out>,
ret=0x7fa9680008e0) at remote_dispatch.h:2080
#9 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3648f60, client=0x7fa9d364eb40, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#10 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d364eb40, msg=0x7fa9d3648f60) at rpc/virnetserverprogram.c:307
#11 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#12 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#13 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d3612030) at util/virthreadpool.c:144
#14 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#15 0x00007fa9cfc00182 in start_thread (arg=0x7fa9bd39a700) at pthread_create.c:312
#16 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 10 (Thread 0x7fa9bcb99700 (LWP 29586)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cfef8001 in virDomainObjListFindByName (doms=0x7fa9b8189460, name=0x7fa96c0008e0 "instance-0000001f") at conf/domain_conf.c:1114
#6 0x00007fa9c118b786 in libxlDomainLookupByName (conn=0x7fa96c000b80, name=<optimised out>) at libxl/libxl_driver.c:954
#7 0x00007fa9cff71356 in virDomainLookupByName (conn=0x7fa96c000b80, name=0x7fa96c0008e0 "instance-0000001f") at libvirt-domain.c:427
#8 0x00007fa9d16fd492 in remoteDispatchDomainLookupByName (server=<optimised out>, msg=<optimised out>, ret=0x7fa96c003240, args=0x7fa96c0008c0, rerr=0x7fa9bcb98c40,
client=0x7fa9d36452c0) at remote_dispatch.h:5429
#9 remoteDispatchDomainLookupByNameHelper (server=<optimised out>, client=0x7fa9d36452c0, msg=<optimised out>, rerr=0x7fa9bcb98c40, args=0x7fa96c0008c0, ret=0x7fa96c003240)
at remote_dispatch.h:5409
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3645530, client=0x7fa9d36452c0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36452c0, msg=0x7fa9d3645530) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d3613b20) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9bcb99700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 9 (Thread 0x7fa967fff700 (LWP 29668)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cfef8001 in virDomainObjListFindByName (doms=0x7fa9b8189460, name=0x7fa9600008e0 "instance-00000024") at conf/domain_conf.c:1114
#6 0x00007fa9c118b786 in libxlDomainLookupByName (conn=0x7fa96c000b80, name=<optimised out>) at libxl/libxl_driver.c:954
#7 0x00007fa9cff71356 in virDomainLookupByName (conn=0x7fa96c000b80, name=0x7fa9600008e0 "instance-00000024") at libvirt-domain.c:427
#8 0x00007fa9d16fd492 in remoteDispatchDomainLookupByName (server=<optimised out>, msg=<optimised out>, ret=0x7fa960000b80, args=0x7fa9600008c0, rerr=0x7fa967ffec40,
client=0x7fa9d36452c0) at remote_dispatch.h:5429
#9 remoteDispatchDomainLookupByNameHelper (server=<optimised out>, client=0x7fa9d36452c0, msg=<optimised out>, rerr=0x7fa967ffec40, args=0x7fa9600008c0, ret=0x7fa960000b80)
at remote_dispatch.h:5409
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d36455a0, client=0x7fa9d36452c0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36452c0, msg=0x7fa9d36455a0) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d3619b80) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa967fff700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 8 (Thread 0x7fa9677fe700 (LWP 29744)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cff0caf0 in virDomainObjListAdd (doms=0x7fa9b8189460, def=def@entry=0x7fa95800a930, xmlopt=0x7fa9b81a5740, flags=flags@entry=0,
oldDef=oldDef@entry=0x7fa9677fda40) at conf/domain_conf.c:2453
#6 0x00007fa9c1187e67 in libxlDomainDefineXMLFlags (conn=0x7fa96c000b80,
xml=0x7fa958000990 "<domain type=\"xen\">\n <uuid>0ef9e67d-eb9a-48e4-b773-ead7f0167009</uuid>\n <name>instance-00000028</name>\n <memory>65536</memory>\n <vcpu cpuset=\"0-7\">1</vcpu>\n <metadata>\n <nova:instance xmlns:no"..., flags=<optimised out>) at libxl/libxl_driver.c:2626
#7 0x00007fa9cff836b4 in virDomainDefineXML (conn=0x7fa96c000b80,
xml=0x7fa958000990 "<domain type=\"xen\">\n <uuid>0ef9e67d-eb9a-48e4-b773-ead7f0167009</uuid>\n <name>instance-00000028</name>\n <memory>65536</memory>\n <vcpu cpuset=\"0-7\">1</vcpu>\n <metadata>\n <nova:instance xmlns:no"...) at libvirt-domain.c:6464
#8 0x00007fa9d16fe8e2 in remoteDispatchDomainDefineXML (server=<optimised out>, msg=<optimised out>, ret=0x7fa958000960, args=0x7fa9580008e0, rerr=0x7fa9677fdc40,
client=0x7fa9d36452c0) at remote_dispatch.h:3376
#9 remoteDispatchDomainDefineXMLHelper (server=<optimised out>, client=0x7fa9d36452c0, msg=<optimised out>, rerr=0x7fa9677fdc40, args=0x7fa9580008e0, ret=0x7fa958000960)
at remote_dispatch.h:3356
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3651080, client=0x7fa9d36452c0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36452c0, msg=0x7fa9d3651080) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d3619060) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9677fe700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 7 (Thread 0x7fa966ffd700 (LWP 29848)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cff0caf0 in virDomainObjListAdd (doms=0x7fa9b8189460, def=def@entry=0x7fa95c00a930, xmlopt=0x7fa9b81a5740, flags=flags@entry=0,
oldDef=oldDef@entry=0x7fa966ffca40) at conf/domain_conf.c:2453
#6 0x00007fa9c1187e67 in libxlDomainDefineXMLFlags (conn=0x7fa96c000b80,
xml=0x7fa95c000990 "<domain type=\"xen\">\n <uuid>d64983a4-b85c-4793-a526-7b451517c55b</uuid>\n <name>instance-00000026</name>\n <memory>65536</memory>\n <vcpu cpuset=\"0-7\">1</vcpu>\n <metadata>\n <nova:instance xmlns:no"..., flags=<optimised out>) at libxl/libxl_driver.c:2626
#7 0x00007fa9cff836b4 in virDomainDefineXML (conn=0x7fa96c000b80,
xml=0x7fa95c000990 "<domain type=\"xen\">\n <uuid>d64983a4-b85c-4793-a526-7b451517c55b</uuid>\n <name>instance-00000026</name>\n <memory>65536</memory>\n <vcpu cpuset=\"0-7\">1</vcpu>\n <metadata>\n <nova:instance xmlns:no"...) at libvirt-domain.c:6464
#8 0x00007fa9d16fe8e2 in remoteDispatchDomainDefineXML (server=<optimised out>, msg=<optimised out>, ret=0x7fa95c000960, args=0x7fa95c0008e0, rerr=0x7fa966ffcc40,
client=0x7fa9d36452c0) at remote_dispatch.h:3376
#9 remoteDispatchDomainDefineXMLHelper (server=<optimised out>, client=0x7fa9d36452c0, msg=<optimised out>, rerr=0x7fa966ffcc40, args=0x7fa95c0008e0, ret=0x7fa95c000960)
at remote_dispatch.h:3356
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3648fd0, client=0x7fa9d36452c0, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d36452c0, msg=0x7fa9d3648fd0) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d361b550) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa966ffd700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 6 (Thread 0x7fa9667fc700 (LWP 30108)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cff0caf0 in virDomainObjListAdd (doms=0x7fa9b8189460, def=def@entry=0x7fa95000ad40, xmlopt=0x7fa9b81a5740, flags=flags@entry=0,
oldDef=oldDef@entry=0x7fa9667fba40) at conf/domain_conf.c:2453
#6 0x00007fa9c1187e67 in libxlDomainDefineXMLFlags (conn=0x7fa950000b80,
xml=0x7fa950000e90 "<domain type=\"xen\">\n <uuid>3c260037-874b-4ec8-8704-05cf2fe9ee98</uuid>\n <name>instance-00000029</name>\n <memory>65536</memory>\n <vcpu cpuset=\"0-7\">1</vcpu>\n <metadata>\n <nova:instance xmlns:no"..., flags=<optimised out>) at libxl/libxl_driver.c:2626
#7 0x00007fa9cff836b4 in virDomainDefineXML (conn=0x7fa950000b80,
xml=0x7fa950000e90 "<domain type=\"xen\">\n <uuid>3c260037-874b-4ec8-8704-05cf2fe9ee98</uuid>\n <name>instance-00000029</name>\n <memory>65536</memory>\n <vcpu cpuset=\"0-7\">1</vcpu>\n <metadata>\n <nova:instance xmlns:no"...) at libvirt-domain.c:6464
#8 0x00007fa9d16fe8e2 in remoteDispatchDomainDefineXML (server=<optimised out>, msg=<optimised out>, ret=0x7fa950003240, args=0x7fa9500008e0, rerr=0x7fa9667fbc40,
client=0x7fa9d3652320) at remote_dispatch.h:3376
#9 remoteDispatchDomainDefineXMLHelper (server=<optimised out>, client=0x7fa9d3652320, msg=<optimised out>, rerr=0x7fa9667fbc40, args=0x7fa9500008e0, ret=0x7fa950003240)
at remote_dispatch.h:3356
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3652780, client=0x7fa9d3652320, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d3652320, msg=0x7fa9d3652780) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d3622cb0) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9667fc700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 5 (Thread 0x7fa965ffb700 (LWP 30109)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cff0caf0 in virDomainObjListAdd (doms=0x7fa9b8189460, def=def@entry=0x7fa95400ab40, xmlopt=0x7fa9b81a5740, flags=flags@entry=0,
oldDef=oldDef@entry=0x7fa965ffaa40) at conf/domain_conf.c:2453
#6 0x00007fa9c1187e67 in libxlDomainDefineXMLFlags (conn=0x7fa950000b80,
xml=0x7fa954000db0 "<domain type=\"xen\">\n <uuid>6207e6ef-543d-49bb-a06e-60fcbee4652e</uuid>\n <name>instance-00000027</name>\n <memory>65536</memory>\n <vcpu cpuset=\"0-7\">1</vcpu>\n <metadata>\n <nova:instance xmlns:no"..., flags=<optimised out>) at libxl/libxl_driver.c:2626
#7 0x00007fa9cff836b4 in virDomainDefineXML (conn=0x7fa950000b80,
xml=0x7fa954000db0 "<domain type=\"xen\">\n <uuid>6207e6ef-543d-49bb-a06e-60fcbee4652e</uuid>\n <name>instance-00000027</name>\n <memory>65536</memory>\n <vcpu cpuset=\"0-7\">1</vcpu>\n <metadata>\n <nova:instance xmlns:no"...) at libvirt-domain.c:6464
#8 0x00007fa9d16fe8e2 in remoteDispatchDomainDefineXML (server=<optimised out>, msg=<optimised out>, ret=0x7fa954000d80, args=0x7fa9540008e0, rerr=0x7fa965ffac40,
client=0x7fa9d3652320) at remote_dispatch.h:3376
#9 remoteDispatchDomainDefineXMLHelper (server=<optimised out>, client=0x7fa9d3652320, msg=<optimised out>, rerr=0x7fa965ffac40, args=0x7fa9540008e0, ret=0x7fa954000d80)
at remote_dispatch.h:3356
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3645430, client=0x7fa9d3652320, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d3652320, msg=0x7fa9d3645430) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d3624110) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa965ffb700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 4 (Thread 0x7fa9657fa700 (LWP 30110)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cfef8001 in virDomainObjListFindByName (doms=0x7fa9b8189460, name=0x7fa9480008c0 "instance-00000028") at conf/domain_conf.c:1114
#6 0x00007fa9c118b786 in libxlDomainLookupByName (conn=0x7fa950000b80, name=<optimised out>) at libxl/libxl_driver.c:954
#7 0x00007fa9cff71356 in virDomainLookupByName (conn=0x7fa950000b80, name=0x7fa9480008c0 "instance-00000028") at libvirt-domain.c:427
#8 0x00007fa9d16fd492 in remoteDispatchDomainLookupByName (server=<optimised out>, msg=<optimised out>, ret=0x7fa948000d80, args=0x7fa9480008e0, rerr=0x7fa9657f9c40,
client=0x7fa9d3652320) at remote_dispatch.h:5429
#9 remoteDispatchDomainLookupByNameHelper (server=<optimised out>, client=0x7fa9d3652320, msg=<optimised out>, rerr=0x7fa9657f9c40, args=0x7fa9480008e0, ret=0x7fa948000d80)
at remote_dispatch.h:5409
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3652550, client=0x7fa9d3652320, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d3652320, msg=0x7fa9d3652550) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d36178b0) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa9657fa700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 3 (Thread 0x7fa964ff9700 (LWP 30111)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cfef8001 in virDomainObjListFindByName (doms=0x7fa9b8189460, name=0x7fa94c0008c0 "instance-00000026") at conf/domain_conf.c:1114
#6 0x00007fa9c118b786 in libxlDomainLookupByName (conn=0x7fa950000b80, name=<optimised out>) at libxl/libxl_driver.c:954
#7 0x00007fa9cff71356 in virDomainLookupByName (conn=0x7fa950000b80, name=0x7fa94c0008c0 "instance-00000026") at libvirt-domain.c:427
#8 0x00007fa9d16fd492 in remoteDispatchDomainLookupByName (server=<optimised out>, msg=<optimised out>, ret=0x7fa94c000960, args=0x7fa94c0008e0, rerr=0x7fa964ff8c40,
client=0x7fa9d3652320) at remote_dispatch.h:5429
#9 remoteDispatchDomainLookupByNameHelper (server=<optimised out>, client=0x7fa9d3652320, msg=<optimised out>, rerr=0x7fa964ff8c40, args=0x7fa94c0008e0, ret=0x7fa94c000960)
at remote_dispatch.h:5409
#10 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d3653e10, client=0x7fa9d3652320, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#11 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d3652320, msg=0x7fa9d3653e10) at rpc/virnetserverprogram.c:307
#12 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#13 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#14 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d3613910) at util/virthreadpool.c:144
#15 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#16 0x00007fa9cfc00182 in start_thread (arg=0x7fa964ff9700) at pthread_create.c:312
#17 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 2 (Thread 0x7fa947fff700 (LWP 30112)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fa9cfc02657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fa9cfc02480 in __GI___pthread_mutex_lock (mutex=0x7fa9b8189470) at ../nptl/pthread_mutex_lock.c:79
#3 0x00007fa9cfeda44d in virMutexLock (m=m@entry=0x7fa9b8189470) at util/virthread.c:88
#4 0x00007fa9cfec172e in virObjectLock (anyobj=anyobj@entry=0x7fa9b8189460) at util/virobject.c:323
#5 0x00007fa9cff0bdf3 in virDomainObjListNumOfDomains (doms=0x7fa9b8189460, active=<optimised out>, filter=<optimised out>, conn=<optimised out>)
at conf/domain_conf.c:20606
#6 0x00007fa9cff705be in virConnectNumOfDomains (conn=0x7fa950000b80) at libvirt-domain.c:97
#7 0x00007fa9d16faf39 in remoteDispatchConnectNumOfDomains (server=<optimised out>, msg=<optimised out>, ret=0x7fa9400008e0, rerr=0x7fa947ffec40, client=0x7fa9d3652320)
at remote_dispatch.h:2099
#8 remoteDispatchConnectNumOfDomainsHelper (server=<optimised out>, client=0x7fa9d3652320, msg=<optimised out>, rerr=0x7fa947ffec40, args=<optimised out>,
ret=0x7fa9400008e0) at remote_dispatch.h:2080
#9 0x00007fa9d1717992 in virNetServerProgramDispatchCall (msg=0x7fa9d36451e0, client=0x7fa9d3652320, server=0x7fa9d362b2f0, prog=0x7fa9d363ee90)
at rpc/virnetserverprogram.c:437
#10 virNetServerProgramDispatch (prog=0x7fa9d363ee90, server=server@entry=0x7fa9d362b2f0, client=0x7fa9d3652320, msg=0x7fa9d36451e0) at rpc/virnetserverprogram.c:307
#11 0x00007fa9d17119ed in virNetServerProcessMsg (msg=<optimised out>, prog=<optimised out>, client=<optimised out>, srv=0x7fa9d362b2f0) at rpc/virnetserver.c:172
#12 virNetServerHandleJob (jobOpaque=<optimised out>, opaque=0x7fa9d362b2f0) at rpc/virnetserver.c:193
#13 0x00007fa9cfedad25 in virThreadPoolWorker (opaque=opaque@entry=0x7fa9d3623290) at util/virthreadpool.c:144
#14 0x00007fa9cfeda21e in virThreadHelper (data=<optimised out>) at util/virthread.c:197
#15 0x00007fa9cfc00182 in start_thread (arg=0x7fa947fff700) at pthread_create.c:312
#16 0x00007fa9cf92b00d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
Thread 1 (Thread 0x7fa9d169c8c0 (LWP 5871)):
#0 0x00007fa9cf91dcbd in poll () at ../sysdeps/unix/syscall-template.S:81
#1 0x00007fa9cfe97b9a in poll (__timeout=-1, __nfds=10, __fds=<optimised out>) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2 virEventPollRunOnce () at util/vireventpoll.c:641
#3 0x00007fa9cfe965c2 in virEventRunDefaultImpl () at util/virevent.c:308
#4 0x00007fa9d1712fed in virNetServerRun (srv=0x7fa9d362b2f0) at rpc/virnetserver.c:1139
#5 0x00007fa9d16d9bdd in main (argc=<optimised out>, argv=<optimised out>) at libvirtd.c:1486
[-- Attachment #3: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 6+ messages in thread