[PATCH v4 0/3] scripts/simplebench: add bench_write_req.py test

[PATCH v4 0/3] scripts/simplebench: add bench_write_req.py test

[Andrey Shinkevich]

The script 'bench_write_req.py' allows comparing performances of write request for two
qemu-img binary files. If you made a change in QEMU code and want to check the write
requests performance, you will want to build two qemu-img binary files with and without
your change. Then you specify paths to those binary files and put them as parameters to
the bench_write_req.py script. You may see other supported parameters in the USAGE help.

v4:
  01: 'if/else requests' blocks moved from patch 0001 to 0003.

v3: Based on the Vladimir's review
  01: The test results were amended in the patch description.
  02: The python format string syntax changed to the newer one f''.
  03: The 'empty_disk' test parameter fixed to True.
  04: The function bench_write_req() was supplied with commentary.
  05: The subprocess.call() was replaced with subprocess.run().
  06: The exception handling was improved.
  07: The v2 only patch was split into three in the series.

Andrey Shinkevich (3):
  scripts/simplebench: compare write request performance
  scripts/simplebench: allow writing to non-empty image
  scripts/simplebench: add unaligned data case to bench_write_req

 scripts/simplebench/bench_write_req.py | 206 +++++++++++++++++++++++++++++++++
 1 file changed, 206 insertions(+)
 create mode 100755 scripts/simplebench/bench_write_req.py

-- 
1.8.3.1

[PATCH v4 1/3] scripts/simplebench: compare write request performance

[Andrey Shinkevich]

The script 'bench_write_req.py' allows comparing performances of write
request for two qemu-img binary files.
An example with (qemu-img binary 1) and without (qemu-img binary 2) the
applied patch "qcow2: skip writing zero buffers to empty COW areas"
(git commit ID: c8bb23cbdbe32f5) has the following results:

SSD:
-----------------  -------------------  -------------------
                   <qemu-img binary 1>  <qemu-img binary 2>
<simple case>      0.34 +- 0.01         10.57 +- 0.96
<general case>     0.33 +- 0.01         9.15 +- 0.85
<cluster middle>   0.33 +- 0.00         8.72 +- 0.05
<cluster overlap>  7.43 +- 1.19         14.35 +- 1.00
-----------------  -------------------  -------------------
HDD:
-----------------  -------------------  -------------------
                   <qemu-img binary 1>  <qemu-img binary 2>
<simple case>      32.61 +- 1.17        55.11 +- 1.15
<general case>     54.28 +- 8.82        60.11 +- 2.76
<cluster middle>   57.93 +- 0.47        58.53 +- 0.51
<cluster overlap>  11.47 +- 0.94        17.29 +- 4.40
-----------------  -------------------  -------------------

Suggested-by: Denis V. Lunev <den@openvz.org>
Suggested-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
---
 scripts/simplebench/bench_write_req.py | 173 +++++++++++++++++++++++++++++++++
 1 file changed, 173 insertions(+)
 create mode 100755 scripts/simplebench/bench_write_req.py

diff --git a/scripts/simplebench/bench_write_req.py b/scripts/simplebench/bench_write_req.py
new file mode 100755
index 0000000..a285ef1
--- /dev/null
+++ b/scripts/simplebench/bench_write_req.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+#
+# Test to compare performance of write requests for two qemu-img binary files.
+#
+# Copyright (c) 2020 Virtuozzo International GmbH.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+import sys
+import os
+import subprocess
+import simplebench
+
+
+def bench_func(env, case):
+    """ Handle one "cell" of benchmarking table. """
+    return bench_write_req(env['qemu_img'], env['image_name'],
+                           case['block_size'], case['block_offset'],
+                           case['requests'])
+
+
+def qemu_img_pipe(*args):
+    '''Run qemu-img and return its output'''
+    subp = subprocess.Popen(list(args),
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT,
+                            universal_newlines=True)
+    exitcode = subp.wait()
+    if exitcode < 0:
+        sys.stderr.write('qemu-img received signal %i: %s\n'
+                         % (-exitcode, ' '.join(list(args))))
+    return subp.communicate()[0]
+
+
+def bench_write_req(qemu_img, image_name, block_size, block_offset, requests):
+    """Benchmark write requests
+
+    The function creates a QCOW2 image with the given path/name and fills it
+    with random data optionally. Then it runs the 'qemu-img bench' command and
+    makes series of write requests on the image clusters. Finally, it returns
+    the total time of the write operations on the disk.
+
+    qemu_img     -- path to qemu_img executable file
+    image_name   -- QCOW2 image name to create
+    block_size   -- size of a block to write to clusters
+    block_offset -- offset of the block in clusters
+    requests     -- number of write requests per cluster
+
+    Returns {'seconds': int} on success and {'error': str} on failure.
+    Return value is compatible with simplebench lib.
+    """
+
+    if not os.path.isfile(qemu_img):
+        print(f'File not found: {qemu_img}')
+        sys.exit(1)
+
+    image_dir = os.path.dirname(os.path.abspath(image_name))
+    if not os.path.isdir(image_dir):
+        print(f'Path not found: {image_name}')
+        sys.exit(1)
+
+    cluster_size = 1024 * 1024
+    image_size = 1024 * cluster_size
+    seek = 4
+    dd_count = int(image_size / cluster_size) - seek
+
+    args_create = [qemu_img, 'create', '-f', 'qcow2', '-o',
+                   f'cluster_size={cluster_size}',
+                   image_name, str(image_size)]
+
+    count = requests * int(image_size / cluster_size)
+    step = str(cluster_size)
+    offset = str(block_offset)
+    cnt = str(count)
+    size = []
+    if block_size:
+        size = ['-s', f'{block_size}']
+
+    args_bench = [qemu_img, 'bench', '-w', '-n', '-t', 'none', '-c', cnt,
+                  '-S', step, '-o', offset, '-f', 'qcow2', image_name]
+    if block_size:
+        args_bench.extend(size)
+
+    try:
+        qemu_img_pipe(*args_create)
+    except OSError as e:
+        os.remove(image_name)
+        return {'error': 'qemu_img create failed: ' + str(e)}
+
+    try:
+        ret = qemu_img_pipe(*args_bench)
+    finally:
+        os.remove(image_name)
+        if not ret:
+            return {'error': 'qemu_img bench failed'}
+        if 'seconds' in ret:
+            ret_list = ret.split()
+            index = ret_list.index('seconds.')
+            return {'seconds': float(ret_list[index-1])}
+        else:
+            return {'error': 'qemu_img bench failed: ' + ret}
+
+
+if __name__ == '__main__':
+
+    if len(sys.argv) < 4:
+        program = os.path.basename(sys.argv[0])
+        print(f'USAGE: {program} <path to qemu-img binary file> '
+              '<path to another qemu-img to compare performance with> '
+              '<full or relative name for QCOW2 image to create>')
+        exit(1)
+
+    # Test-cases are "rows" in benchmark resulting table, 'id' is a caption
+    # for the row, other fields are handled by bench_func.
+    test_cases = [
+        {
+            'id': '<simple case>',
+            'block_size': 0,
+            'block_offset': 0,
+            'requests': 10
+        },
+        {
+            'id': '<general case>',
+            'block_size': 4096,
+            'block_offset': 0,
+            'requests': 10
+        },
+        {
+            'id': '<cluster middle>',
+            'block_size': 4096,
+            'block_offset': 524288,
+            'requests': 10
+        },
+        {
+            'id': '<cluster overlap>',
+            'block_size': 524288,
+            'block_offset': 4096,
+            'requests': 2
+        },
+    ]
+
+    # Test-envs are "columns" in benchmark resulting table, 'id is a caption
+    # for the column, other fields are handled by bench_func.
+    # Set the paths below to desired values
+    test_envs = [
+        {
+            'id': '<qemu-img binary 1>',
+            'qemu_img': f'{sys.argv[1]}',
+            'image_name': f'{sys.argv[3]}'
+        },
+        {
+            'id': '<qemu-img binary 2>',
+            'qemu_img': f'{sys.argv[2]}',
+            'image_name': f'{sys.argv[3]}'
+        },
+    ]
+
+    result = simplebench.bench(bench_func, test_envs, test_cases, count=3,
+                               initial_run=False)
+    print(simplebench.ascii(result))
-- 
1.8.3.1

Re: [PATCH v4 1/3] scripts/simplebench: compare write request performance

[Vladimir Sementsov-Ogievskiy]

12.07.2020 20:49, Andrey Shinkevich wrote:
> The script 'bench_write_req.py' allows comparing performances of write
> request for two qemu-img binary files.
> An example with (qemu-img binary 1) and without (qemu-img binary 2) the
> applied patch "qcow2: skip writing zero buffers to empty COW areas"
> (git commit ID: c8bb23cbdbe32f5) has the following results:
> 
> SSD:
> -----------------  -------------------  -------------------
>                     <qemu-img binary 1>  <qemu-img binary 2>
> <simple case>      0.34 +- 0.01         10.57 +- 0.96
> <general case>     0.33 +- 0.01         9.15 +- 0.85
> <cluster middle>   0.33 +- 0.00         8.72 +- 0.05
> <cluster overlap>  7.43 +- 1.19         14.35 +- 1.00
> -----------------  -------------------  -------------------
> HDD:
> -----------------  -------------------  -------------------
>                     <qemu-img binary 1>  <qemu-img binary 2>
> <simple case>      32.61 +- 1.17        55.11 +- 1.15
> <general case>     54.28 +- 8.82        60.11 +- 2.76
> <cluster middle>   57.93 +- 0.47        58.53 +- 0.51
> <cluster overlap>  11.47 +- 0.94        17.29 +- 4.40
> -----------------  -------------------  -------------------
> 
> Suggested-by: Denis V. Lunev <den@openvz.org>
> Suggested-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
> Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>

Andrey wants to drop 02,03 in v5, so this patch is a candidate for v5. Below my notes.

> ---
>   scripts/simplebench/bench_write_req.py | 173 +++++++++++++++++++++++++++++++++
>   1 file changed, 173 insertions(+)
>   create mode 100755 scripts/simplebench/bench_write_req.py
> 
> diff --git a/scripts/simplebench/bench_write_req.py b/scripts/simplebench/bench_write_req.py
> new file mode 100755
> index 0000000..a285ef1
> --- /dev/null
> +++ b/scripts/simplebench/bench_write_req.py
> @@ -0,0 +1,173 @@
> +#!/usr/bin/env python3
> +#
> +# Test to compare performance of write requests for two qemu-img binary files.

Let's note that patch is intended to check benefit of c8bb23cbdbe
"qcow2: skip writing zero buffers to empty COW areas"

> +#
> +# Copyright (c) 2020 Virtuozzo International GmbH.
> +#
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation; either version 2 of the License, or
> +# (at your option) any later version.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program.  If not, see <http://www.gnu.org/licenses/>.
> +#
> +
> +
> +import sys
> +import os
> +import subprocess
> +import simplebench
> +
> +
> +def bench_func(env, case):
> +    """ Handle one "cell" of benchmarking table. """
> +    return bench_write_req(env['qemu_img'], env['image_name'],
> +                           case['block_size'], case['block_offset'],
> +                           case['requests'])
> +
> +
> +def qemu_img_pipe(*args):
> +    '''Run qemu-img and return its output'''
> +    subp = subprocess.Popen(list(args),
> +                            stdout=subprocess.PIPE,
> +                            stderr=subprocess.STDOUT,
> +                            universal_newlines=True)
> +    exitcode = subp.wait()
> +    if exitcode < 0:
> +        sys.stderr.write('qemu-img received signal %i: %s\n'
> +                         % (-exitcode, ' '.join(list(args))))
> +    return subp.communicate()[0]
> +
> +
> +def bench_write_req(qemu_img, image_name, block_size, block_offset, requests):
> +    """Benchmark write requests
> +
> +    The function creates a QCOW2 image with the given path/name and fills it
> +    with random data optionally. 

No, it doesn't fill..

> + Then it runs the 'qemu-img bench' command and
> +    makes series of write requests on the image clusters. Finally, it returns
> +    the total time of the write operations on the disk.
> +
> +    qemu_img     -- path to qemu_img executable file
> +    image_name   -- QCOW2 image name to create
> +    block_size   -- size of a block to write to clusters
> +    block_offset -- offset of the block in clusters
> +    requests     -- number of write requests per cluster
> +
> +    Returns {'seconds': int} on success and {'error': str} on failure.
> +    Return value is compatible with simplebench lib.
> +    """
> +
> +    if not os.path.isfile(qemu_img):
> +        print(f'File not found: {qemu_img}')
> +        sys.exit(1)
> +
> +    image_dir = os.path.dirname(os.path.abspath(image_name))
> +    if not os.path.isdir(image_dir):
> +        print(f'Path not found: {image_name}')
> +        sys.exit(1)
> +
> +    cluster_size = 1024 * 1024
> +    image_size = 1024 * cluster_size
> +    seek = 4
> +    dd_count = int(image_size / cluster_size) - seek

seek and dd_count are unused

> +
> +    args_create = [qemu_img, 'create', '-f', 'qcow2', '-o',
> +                   f'cluster_size={cluster_size}',
> +                   image_name, str(image_size)]
> +
> +    count = requests * int(image_size / cluster_size)

requests is number of requests per cluster..

> +    step = str(cluster_size)

but step is one cluster. So, we have several requests per cluster, but still, step is cluster?

It sounds strange to me. Assume requests = 5 and we have image with 5 clusters. count would be 5 * 5 = 25. Trying to make 25 iterations with step=cluster will go far beyond the image size..

> +    offset = str(block_offset)
> +    cnt = str(count)

extra variables. Let's just use str() in args below.

> +    size = []
> +    if block_size:
> +        size = ['-s', f'{block_size}']
> +
> +    args_bench = [qemu_img, 'bench', '-w', '-n', '-t', 'none', '-c', cnt,
> +                  '-S', step, '-o', offset, '-f', 'qcow2', image_name]
> +    if block_size:
> +        args_bench.extend(size)

1. You may just write here

if block_size:
    args_bench += ['-s', f'{block_size}']

No reason to create extra "size" variable to be used only here.

2. Why do you need this logic? If user pass block_size = 0, we instead rely on default bufsize of img_bench() which is 4096. And in two test-cases you use explicit 4096 constant, and in one you use 0 to be implicitly changed to same 4096. Let's instead specify block_size explicitly.


> +
> +    try:
> +        qemu_img_pipe(*args_create)
> +    except OSError as e:
> +        os.remove(image_name)
> +        return {'error': 'qemu_img create failed: ' + str(e)}
> +
> +    try:
> +        ret = qemu_img_pipe(*args_bench)
> +    finally:
> +        os.remove(image_name)
> +        if not ret:

ret may be not defined, if exception shot prior to assignment of ret, isn't it?

I suggest to not bother with "finally", and just make similar "except" like for previous case, and then just parse ret at function end.

> +            return {'error': 'qemu_img bench failed'}
> +        if 'seconds' in ret:
> +            ret_list = ret.split()
> +            index = ret_list.index('seconds.')
> +            return {'seconds': float(ret_list[index-1])}
> +        else:
> +            return {'error': 'qemu_img bench failed: ' + ret}
> +
> +
> +if __name__ == '__main__':
> +
> +    if len(sys.argv) < 4:
> +        program = os.path.basename(sys.argv[0])
> +        print(f'USAGE: {program} <path to qemu-img binary file> '
> +              '<path to another qemu-img to compare performance with> '
> +              '<full or relative name for QCOW2 image to create>')
> +        exit(1)
> +
> +    # Test-cases are "rows" in benchmark resulting table, 'id' is a caption
> +    # for the row, other fields are handled by bench_func.
> +    test_cases = [
> +        {
> +            'id': '<simple case>',
> +            'block_size': 0,
> +            'block_offset': 0,
> +            'requests': 10
> +        },
> +        {
> +            'id': '<general case>',
> +            'block_size': 4096,
> +            'block_offset': 0,
> +            'requests': 10
> +        },

Hmm I don't understand, why simple case and general case are different? As I already noted, if you don't specify -s for bench, as you do if block_size is 0, the default value is 4096 anyway in img_bench(). So, how there can be so different results in commit message? Or what am I missing?

> +        {
> +            'id': '<cluster middle>',
> +            'block_size': 4096,
> +            'block_offset': 524288,
> +            'requests': 10
> +        },
> +        {
> +            'id': '<cluster overlap>',

What is overlapping here? you just write half of cluster with a small offset from start of cluster. I'm OK with the case itself, I just don't understand the naming.

> +            'block_size': 524288,
> +            'block_offset': 4096,
> +            'requests': 2
> +        },
> +    ]
> +
> +    # Test-envs are "columns" in benchmark resulting table, 'id is a caption
> +    # for the column, other fields are handled by bench_func.
> +    # Set the paths below to desired values
> +    test_envs = [
> +        {
> +            'id': '<qemu-img binary 1>',
> +            'qemu_img': f'{sys.argv[1]}',
> +            'image_name': f'{sys.argv[3]}'
> +        },
> +        {
> +            'id': '<qemu-img binary 2>',
> +            'qemu_img': f'{sys.argv[2]}',
> +            'image_name': f'{sys.argv[3]}'
> +        },
> +    ]
> +
> +    result = simplebench.bench(bench_func, test_envs, test_cases, count=3,
> +                               initial_run=False)
> +    print(simplebench.ascii(result))
> 


-- 
Best regards,
Vladimir

[PATCH v4 2/3] scripts/simplebench: allow writing to non-empty image

[Andrey Shinkevich]

Add 'empty_image' parameter to the function bench_write_req() and to
the test cases that will allow writing to the non-empty clusters of the
image if the 'empty_image' parameter set to False.

Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
---
 scripts/simplebench/bench_write_req.py | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/scripts/simplebench/bench_write_req.py b/scripts/simplebench/bench_write_req.py
index a285ef1..f758f90 100755
--- a/scripts/simplebench/bench_write_req.py
+++ b/scripts/simplebench/bench_write_req.py
@@ -29,7 +29,7 @@ def bench_func(env, case):
     """ Handle one "cell" of benchmarking table. """
     return bench_write_req(env['qemu_img'], env['image_name'],
                            case['block_size'], case['block_offset'],
-                           case['requests'])
+                           case['requests'], case['empty_image'])
 
 
 def qemu_img_pipe(*args):
@@ -45,7 +45,8 @@ def qemu_img_pipe(*args):
     return subp.communicate()[0]
 
 
-def bench_write_req(qemu_img, image_name, block_size, block_offset, requests):
+def bench_write_req(qemu_img, image_name, block_size, block_offset, requests,
+                    empty_image):
     """Benchmark write requests
 
     The function creates a QCOW2 image with the given path/name and fills it
@@ -58,6 +59,7 @@ def bench_write_req(qemu_img, image_name, block_size, block_offset, requests):
     block_size   -- size of a block to write to clusters
     block_offset -- offset of the block in clusters
     requests     -- number of write requests per cluster
+    empty_image  -- if not True, fills image with random data
 
     Returns {'seconds': int} on success and {'error': str} on failure.
     Return value is compatible with simplebench lib.
@@ -96,6 +98,15 @@ def bench_write_req(qemu_img, image_name, block_size, block_offset, requests):
 
     try:
         qemu_img_pipe(*args_create)
+
+        if not empty_image:
+            dd = ['dd', 'if=/dev/urandom', f'of={image_name}',
+                  f'bs={cluster_size}', f'seek={seek}',
+                  f'count={dd_count}']
+            devnull = open('/dev/null', 'w')
+            subprocess.run(dd, stderr=devnull, stdout=devnull)
+            subprocess.run('sync')
+
     except OSError as e:
         os.remove(image_name)
         return {'error': 'qemu_img create failed: ' + str(e)}
@@ -130,25 +141,29 @@ if __name__ == '__main__':
             'id': '<simple case>',
             'block_size': 0,
             'block_offset': 0,
-            'requests': 10
+            'requests': 10,
+            'empty_image': True
         },
         {
             'id': '<general case>',
             'block_size': 4096,
             'block_offset': 0,
-            'requests': 10
+            'requests': 10,
+            'empty_image': True
         },
         {
             'id': '<cluster middle>',
             'block_size': 4096,
             'block_offset': 524288,
-            'requests': 10
+            'requests': 10,
+            'empty_image': True
         },
         {
             'id': '<cluster overlap>',
             'block_size': 524288,
             'block_offset': 4096,
-            'requests': 2
+            'requests': 2,
+            'empty_image': True
         },
     ]
 
-- 
1.8.3.1

[PATCH v4 3/3] scripts/simplebench: add unaligned data case to bench_write_req

[Andrey Shinkevich]

Add a test case with writhing data unaligned to the image clusters.
This case does not involve the COW optimization introduced with the
patch "qcow2: skip writing zero buffers to empty COW areas"
(git commit ID: c8bb23cbdbe32f5).

Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
---
 scripts/simplebench/bench_write_req.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/scripts/simplebench/bench_write_req.py b/scripts/simplebench/bench_write_req.py
index f758f90..9f3a520 100755
--- a/scripts/simplebench/bench_write_req.py
+++ b/scripts/simplebench/bench_write_req.py
@@ -58,7 +58,7 @@ def bench_write_req(qemu_img, image_name, block_size, block_offset, requests,
     image_name   -- QCOW2 image name to create
     block_size   -- size of a block to write to clusters
     block_offset -- offset of the block in clusters
-    requests     -- number of write requests per cluster
+    requests     -- number of write requests per cluster, customize if zero
     empty_image  -- if not True, fills image with random data
 
     Returns {'seconds': int} on success and {'error': str} on failure.
@@ -83,8 +83,17 @@ def bench_write_req(qemu_img, image_name, block_size, block_offset, requests,
                    f'cluster_size={cluster_size}',
                    image_name, str(image_size)]
 
-    count = requests * int(image_size / cluster_size)
-    step = str(cluster_size)
+    if requests:
+        count = requests * int(image_size / cluster_size)
+        step = str(cluster_size)
+    else:
+        # Create unaligned write requests
+        assert block_size
+        shift = int(block_size * 1.01)
+        count = int((image_size - block_offset) / shift)
+        step = str(shift)
+        depth = ['-d', '2']
+
     offset = str(block_offset)
     cnt = str(count)
     size = []
@@ -95,6 +104,8 @@ def bench_write_req(qemu_img, image_name, block_size, block_offset, requests,
                   '-S', step, '-o', offset, '-f', 'qcow2', image_name]
     if block_size:
         args_bench.extend(size)
+    if not requests:
+        args_bench.extend(depth)
 
     try:
         qemu_img_pipe(*args_create)
@@ -165,6 +176,13 @@ if __name__ == '__main__':
             'requests': 2,
             'empty_image': True
         },
+        {
+            'id': '<unaligned>',
+            'block_size': 104857600,
+            'block_offset': 524288,
+            'requests': 0,
+            'empty_image': False
+        },
     ]
 
     # Test-envs are "columns" in benchmark resulting table, 'id is a caption
-- 
1.8.3.1