From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([208.118.235.92]:56362) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1UIF3V-0004eN-9J for qemu-devel@nongnu.org; Wed, 20 Mar 2013 05:11:36 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1UIF3Q-0002j5-Q9 for qemu-devel@nongnu.org; Wed, 20 Mar 2013 05:11:33 -0400 Received: from nodalink.pck.nerim.net ([62.212.105.220]:33392 helo=paradis.irqsave.net) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1UIF3Q-0002iQ-AG for qemu-devel@nongnu.org; Wed, 20 Mar 2013 05:11:28 -0400 From: =?UTF-8?q?Beno=C3=AEt=20Canet?= Date: Wed, 20 Mar 2013 10:12:13 +0100 Message-Id: <1363770734-30970-1-git-send-email-benoit@irqsave.net> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: quoted-printable Subject: [Qemu-devel] [PATCH] Fix I/O throttling pathologic oscillating behavior List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: qemu-devel@nongnu.org Cc: kwolf@redhat.com, wuzhy@linux.vnet.ibm.com, =?UTF-8?q?Beno=C3=AEt=20Canet?= , stefanha@redhat.com Limiting a virtio device backed by a QED file at 150 iops with "-drive file=3Dtest.qed,if=3Dvirtio,cache=3Dnone,iops=3D150" and running= in the guest the load.c program lead to an iops oscillating behavior which amplify its= elf as the time goes on. At first an extract of "iostats -x -d 2" would look like this: Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-s= z vdb 0,00 4,11 0,10 149,09 0,39 614,02 8,2= 4 vdb 0,00 0,00 0,00 163,27 0,00 653,06 8,0= 0 vdb 0,00 0,00 0,00 164,65 0,00 658,59 8,0= 0 vdb 0,00 0,00 0,00 84,85 0,00 339,39 8,0= 0 vdb 0,00 0,00 0,00 170,71 0,00 682,83 8,0= 0 vdb 0,00 0,00 0,00 185,71 0,00 742,86 8,0= 0 vdb 0,00 0,00 0,00 174,75 0,00 698,99 8,0= 0 vdb 0,00 0,00 0,00 87,88 0,00 351,52 8,0= 0 w/s seems ok After a few moment it would look like this: Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-s= z vdb 0,00 0,00 0,00 249,00 0,00 996,00 8,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 260,00 0,00 1040,00 8,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 250,00 0,00 1000,00 8,0= 0 vdb 0,00 0,00 0,00 249,49 0,00 997,98 8,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 Here w/s start to oscillate on a few second cycle. Waiting around ten hours leads to this. Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-s= z db 0,00 0,00 0,00 0,00 0,00 0,00 0,00 vdb 0,00 0,00 0,00 267,68 0,00 1070,71 8,0= 0 vdb 0,00 0,00 0,00 1184,38 0,00 4737,50 8,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 1415,15 0,00 5660,61 8,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 0,00 0,00 0,00 0,0= 0 vdb 0,00 0,00 0,00 939,18 0,00 3756,70 8,0= 0 vdb 0,00 0,00 0,00 523,71 0,00 2094,85 8,0= 0 It seems to get worse with the amplitude and duration of the cycle growin= g. As the cause of the amplification of the oscillation seems to be the grow= ing of the cycle duration I wrote the "block: fix bdrv_exceed_iops_limits wait computation" patch which solve this problem. load.c - beware of hardcoded "/dev/vdb" -------------------------------------------------------------------------= ------ #define _GNU_SOURCE #include #include #include #include #include #include #define BLOCKSIZE 4096 #define THREAD_COUNT 50 #define IOS 100000 pthread_mutex_t mutex; pthread_cond_t condition; void *io_thread_body(void *opaque) { int j, fd; int i =3D (int) *(int *) opaque; void *buffer; posix_memalign(&buffer, BLOCKSIZE, BLOCKSIZE); fd =3D open("/dev/vdb", O_RDWR|O_DIRECT); if (fd < 0) { printf("Fail to open /dev/vdb in O_DIRECT\n"); goto exit; } printf("Waiting for signal\n"); pthread_mutex_lock(&mutex); pthread_cond_wait(&condition, &mutex); pthread_mutex_unlock(&mutex); printf("Starting ios\n"); while (1) { off_t offset =3D (i * IOS * BLOCKSIZE) + j * BLOCKSIZE; pwrite(fd, buffer, BLOCKSIZE, offset); j++; } exit: free(opaque); pthread_exit(NULL); } int main(int argc, char **argv) { int i; pthread_t threads[THREAD_COUNT]; pthread_attr_t attr; pthread_mutex_init(&mutex, NULL); pthread_cond_init(&condition, NULL); pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); printf("Creating threads\n"); for (i =3D 0; i < THREAD_COUNT; i++) { int *j =3D malloc(sizeof(j)); *j =3D i; pthread_create(&threads[i], &attr, io_thread_body, j); } sleep(1); printf("Waking up threads\n"); pthread_cond_broadcast(&condition); printf("Waiting threads to finish\n"); for (i =3D 0; i < THREAD_COUNT; i++) { pthread_join(threads[i], NULL); } pthread_attr_destroy(&attr); pthread_mutex_destroy(&mutex); pthread_cond_destroy(&condition); return 0; } -------------------------------------------------------------------------= ------ Beno=C3=AEt Canet (1): block: fix bdrv_exceed_iops_limits wait computation block.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- 1.7.10.4