From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([2001:4830:134:3::10]:35457) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1bNzgW-0005Su-6P for qemu-devel@nongnu.org; Fri, 15 Jul 2016 05:45:29 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1bNzgS-00053m-0Q for qemu-devel@nongnu.org; Fri, 15 Jul 2016 05:45:27 -0400 Received: from mx-v6.kamp.de ([2a02:248:0:51::16]:39351 helo=mx01.kamp.de) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1bNzgR-00053O-N4 for qemu-devel@nongnu.org; Fri, 15 Jul 2016 05:45:23 -0400 From: Peter Lieven Date: Fri, 15 Jul 2016 11:45:11 +0200 Message-Id: <1468575911-20656-1-git-send-email-pl@kamp.de> Subject: [Qemu-devel] [PATCH] vnc-tight: fix regression with libxenstore List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: qemu-devel@nongnu.org Cc: pbonzini@redhat.com, kraxel@redhat.com, jgross@suse.com, Peter Lieven commit 095497ff added thread local storage for the color counting palette. Unfortunately, a VncPalette is about 7kB on a x86_64 system. This memory is reserved from the stack of every thread and it exhausted the stack space of a libxenstore thread. Fix this by allocating memory only for the VNC encoding thread. Fixes: 095497ffc66b7f031ff2a17f1e50f5cb105ce588 Reported-by: Juergen Gross Tested-by: Juergen Gross Signed-off-by: Peter Lieven --- ui/vnc-enc-tight.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/ui/vnc-enc-tight.c b/ui/vnc-enc-tight.c index b8581dd..2b58739 100644 --- a/ui/vnc-enc-tight.c +++ b/ui/vnc-enc-tight.c @@ -1457,11 +1457,17 @@ static int send_sub_rect_jpeg(VncState *vs, int x, int y, int w, int h, } #endif -static __thread VncPalette color_count_palette; +static __thread VncPalette *color_count_palette; +static __thread Notifier vnc_tight_cleanup_notifier; + +static void vnc_tight_cleanup(Notifier *n, void *value) +{ + g_free(color_count_palette); + color_count_palette = NULL; +} static int send_sub_rect(VncState *vs, int x, int y, int w, int h) { - VncPalette *palette = &color_count_palette; uint32_t bg = 0, fg = 0; int colors; int ret = 0; @@ -1470,6 +1476,12 @@ static int send_sub_rect(VncState *vs, int x, int y, int w, int h) bool allow_jpeg = true; #endif + if (!color_count_palette) { + color_count_palette = g_malloc(sizeof(VncPalette)); + vnc_tight_cleanup_notifier.notify = vnc_tight_cleanup; + qemu_thread_atexit_add(&vnc_tight_cleanup_notifier); + } + vnc_framebuffer_update(vs, x, y, w, h, vs->tight.type); vnc_tight_start(vs); @@ -1490,17 +1502,19 @@ static int send_sub_rect(VncState *vs, int x, int y, int w, int h) } #endif - colors = tight_fill_palette(vs, x, y, w * h, &bg, &fg, palette); + colors = tight_fill_palette(vs, x, y, w * h, &bg, &fg, color_count_palette); #ifdef CONFIG_VNC_JPEG if (allow_jpeg && vs->tight.quality != (uint8_t)-1) { - ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors, palette, - force_jpeg); + ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors, + color_count_palette, force_jpeg); } else { - ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette); + ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, + color_count_palette); } #else - ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette); + ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, + color_count_palette); #endif return ret; -- 1.9.1