[PATCH 0/5] test efi images in qemu

[PATCH 0/5] test efi images in qemu

[Ed Bartosh]

Hi,

This patchset adds testing of efi images in qemu to the wic test suite,
related oe-selftest API changes and small improvements.

The following changes since commit 4188e53f803aefdb26768abfad591283662cd27e:

  poky: make 4.10 the qemu* default (2017-03-04 23:19:03 +0000)

are available in the git repository at:

  git://git.yoctoproject.org/poky-contrib ed/wic/wip
  http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=ed/wic/wip

Ed Bartosh (5):
  qemurunner: add runqemuparams argument to commands.runqemu
  targetcontrol: add image_fstype argument to commands.runqemu
  selftest: test wic efi image in qemu
  qemurunner.py: ignore decode errors
  selftest: remove extra backslashes from debug output

 meta/lib/oeqa/selftest/base.py        |  4 ++--
 meta/lib/oeqa/selftest/wic.py         | 14 ++++++++++++++
 meta/lib/oeqa/targetcontrol.py        |  8 ++++----
 meta/lib/oeqa/utils/commands.py       |  6 +++---
 meta/lib/oeqa/utils/qemurunner.py     |  6 +++---
 meta/lib/oeqa/utils/qemutinyrunner.py |  2 +-
 6 files changed, 27 insertions(+), 13 deletions(-)

--
Regards,
Ed

[PATCH 1/5] qemurunner: add runqemuparams argument to commands.runqemu

[Ed Bartosh]

Added possibility to pass additional runqemu parameters
down the stack of APIs:
 commands.runqemu -> QemuTarget.start -> QemuRunner.start

This will be used to pass ovmf parameter in testing of
efi wic images under qemu.

Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
 meta/lib/oeqa/targetcontrol.py        | 4 ++--
 meta/lib/oeqa/utils/commands.py       | 4 ++--
 meta/lib/oeqa/utils/qemurunner.py     | 4 ++--
 meta/lib/oeqa/utils/qemutinyrunner.py | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/meta/lib/oeqa/targetcontrol.py b/meta/lib/oeqa/targetcontrol.py
index d1f441f..dbd2c7c 100644
--- a/meta/lib/oeqa/targetcontrol.py
+++ b/meta/lib/oeqa/targetcontrol.py
@@ -176,8 +176,8 @@ class QemuTarget(BaseTarget):
         bb.note("Qemu log file: %s" % self.qemulog)
         super(QemuTarget, self).deploy()
 
-    def start(self, params=None, ssh=True, extra_bootparams=None):
-        if self.runner.start(params, get_ip=ssh, extra_bootparams=extra_bootparams):
+    def start(self, params=None, ssh=True, extra_bootparams=None, runqemuparams=''):
+        if self.runner.start(params, get_ip=ssh, extra_bootparams=extra_bootparams, runqemuparams=runqemuparams):
             if ssh:
                 self.ip = self.runner.ip
                 self.server_ip = self.runner.server_ip
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index 0425c9f..73ede23 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -218,7 +218,7 @@ def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec=
 
 
 @contextlib.contextmanager
-def runqemu(pn, ssh=True):
+def runqemu(pn, ssh=True, runqemuparams=''):
 
     import bb.tinfoil
     import bb.build
@@ -260,7 +260,7 @@ def runqemu(pn, ssh=True):
     try:
         qemu.deploy()
         try:
-            qemu.start(ssh=ssh)
+            qemu.start(ssh=ssh, runqemuparams=runqemuparams)
         except bb.build.FuncFailed:
             raise Exception('Failed to start QEMU - see the logs in %s' % logdir)
 
diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py
index 21bc35a..19f0f92 100644
--- a/meta/lib/oeqa/utils/qemurunner.py
+++ b/meta/lib/oeqa/utils/qemurunner.py
@@ -95,7 +95,7 @@ class QemuRunner:
                 self._dump_host()
                 raise SystemExit
 
-    def start(self, qemuparams = None, get_ip = True, extra_bootparams = None):
+    def start(self, qemuparams = None, get_ip = True, extra_bootparams = None, runqemuparams=''):
         if self.display:
             os.environ["DISPLAY"] = self.display
             # Set this flag so that Qemu doesn't do any grabs as SDL grabs
@@ -136,7 +136,7 @@ class QemuRunner:
         self.origchldhandler = signal.getsignal(signal.SIGCHLD)
         signal.signal(signal.SIGCHLD, self.handleSIGCHLD)
 
-        launch_cmd = 'runqemu snapshot '
+        launch_cmd = 'runqemu snapshot %s ' % runqemuparams
         if self.use_kvm:
             logger.info('Using kvm for runqemu')
             launch_cmd += 'kvm '
diff --git a/meta/lib/oeqa/utils/qemutinyrunner.py b/meta/lib/oeqa/utils/qemutinyrunner.py
index d554f0d..ec52473 100644
--- a/meta/lib/oeqa/utils/qemutinyrunner.py
+++ b/meta/lib/oeqa/utils/qemutinyrunner.py
@@ -60,7 +60,7 @@ class QemuTinyRunner(QemuRunner):
             with open(self.logfile, "a") as f:
                 f.write("%s" % msg)
 
-    def start(self, qemuparams = None, ssh=True, extra_bootparams=None):
+    def start(self, qemuparams = None, ssh=True, extra_bootparams=None, runqemuparams=''):
 
         if self.display:
             os.environ["DISPLAY"] = self.display
-- 
2.1.4

[PATCH 2/5] targetcontrol: add image_fstype argument to commands.runqemu

[Ed Bartosh]

qemu runner picks up first fsimage type from the hard-coded
list of supported types. This makes it impossible to test
particular image type unless it's not ext4(first type in
the hardcoded list of types).

Added image_fstypes argument to commands.runqemu and QemuTarget
__init__ to specify type of the image to run qemu with.

This will be used to pass wic image type to test efi wic images.

Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
 meta/lib/oeqa/targetcontrol.py  | 4 ++--
 meta/lib/oeqa/utils/commands.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/meta/lib/oeqa/targetcontrol.py b/meta/lib/oeqa/targetcontrol.py
index dbd2c7c..0ad3a6b 100644
--- a/meta/lib/oeqa/targetcontrol.py
+++ b/meta/lib/oeqa/targetcontrol.py
@@ -113,11 +113,11 @@ class QemuTarget(BaseTarget):
 
     supported_image_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic']
 
-    def __init__(self, d):
+    def __init__(self, d, image_fstype=None):
 
         super(QemuTarget, self).__init__(d)
 
-        self.image_fstype = self.get_image_fstype(d)
+        self.image_fstype = image_fstype or self.get_image_fstype(d)
         self.qemulog = os.path.join(self.testdir, "qemu_boot_log.%s" % self.datetime)
         self.rootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"),  d.getVar("IMAGE_LINK_NAME") + '.' + self.image_fstype)
         self.kernel = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), d.getVar("KERNEL_IMAGETYPE", False) + '-' + d.getVar('MACHINE', False) + '.bin')
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index 73ede23..82c5908 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -218,7 +218,7 @@ def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec=
 
 
 @contextlib.contextmanager
-def runqemu(pn, ssh=True, runqemuparams=''):
+def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None):
 
     import bb.tinfoil
     import bb.build
@@ -240,7 +240,7 @@ def runqemu(pn, ssh=True, runqemuparams=''):
         logger.propagate = False
         logdir = recipedata.getVar("TEST_LOG_DIR")
 
-        qemu = oeqa.targetcontrol.QemuTarget(recipedata)
+        qemu = oeqa.targetcontrol.QemuTarget(recipedata, image_fstype)
     finally:
         # We need to shut down tinfoil early here in case we actually want
         # to run tinfoil-using utilities with the running QEMU instance.
-- 
2.1.4

[PATCH 3/5] selftest: test wic efi image in qemu

[Ed Bartosh]

Added test_qemu_efi test case to wic test suite.

It uses ovmf qemu extention to test mkefidisk image.

Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
 meta/lib/oeqa/selftest/wic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/meta/lib/oeqa/selftest/wic.py b/meta/lib/oeqa/selftest/wic.py
index b7fe52f..dcb88ba 100644
--- a/meta/lib/oeqa/selftest/wic.py
+++ b/meta/lib/oeqa/selftest/wic.py
@@ -547,3 +547,17 @@ part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/ --r
             status, output = qemu.run_serial(cmd)
             self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output))
             self.assertEqual(output, '/dev/root /\r\n/dev/vda3 /mnt')
+
+    def test_qemu_efi(self):
+        """Test core-image-minimal efi image under qemu"""
+        config = 'IMAGE_FSTYPES = "wic"\nWKS_FILE = "mkefidisk.wks"\n'
+        self.append_config(config)
+        self.assertEqual(0, bitbake('core-image-minimal ovmf').status)
+        self.remove_config(config)
+
+        with runqemu('core-image-minimal', ssh=False,
+                     runqemuparams='ovmf', image_fstype='wic') as qemu:
+            cmd = "grep vda. /proc/partitions  |wc -l"
+            status, output = qemu.run_serial(cmd)
+            self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output))
+            self.assertEqual(output, '3')
-- 
2.1.4

[PATCH 4/5] qemurunner.py: ignore decode errors

[Ed Bartosh]

qemu output can contain control characters. This cause qemurunner
API to crash when decoding the output to utf-8:

Traceback (most recent call last):
  File "/usr/lib64/python3.4/threading.py", line 911, in _bootstrap_inner
    self.run()
  File "meta/lib/oeqa/utils/qemurunner.py", line 472, in run
    threading.Thread.run(self)
  File "/usr/lib64/python3.4/threading.py", line 859, in run
    self._target(*self._args, **self._kwargs)
  File "meta/lib/oeqa/utils/qemurunner.py", line 465, in threadtarget
    self.eventloop()
  File "meta/lib/oeqa/utils/qemurunner.py", line 526, in eventloop
    self.logfunc(data)
  File "meta/lib/oeqa/utils/qemurunner.py", line 77, in log
    msg = msg.decode("utf-8")
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xda in position 0:
unexpected end of data

Added errors='ignore' to decode call to fix this.

Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
 meta/lib/oeqa/utils/qemurunner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py
index 19f0f92..59dc11d 100644
--- a/meta/lib/oeqa/utils/qemurunner.py
+++ b/meta/lib/oeqa/utils/qemurunner.py
@@ -74,7 +74,7 @@ class QemuRunner:
         if self.logfile:
             # It is needed to sanitize the data received from qemu
             # because is possible to have control characters
-            msg = msg.decode("utf-8")
+            msg = msg.decode("utf-8", errors='ignore')
             msg = re_control_char.sub('', msg)
             with codecs.open(self.logfile, "a", encoding="utf-8") as f:
                 f.write("%s" % msg)
-- 
2.1.4

[PATCH 5/5] selftest: remove extra backslashes from debug output

[Ed Bartosh]

Remove unneeded backslashes from the format strings that
caused debug output to look confusing:

2017-03-06 16:52:42,428 - selftest.base - DEBUG - Removing from: ...
\IMAGE_FSTYPES = "wic"
WKS_FILE = "mkefidisk.wks"

Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
 meta/lib/oeqa/selftest/base.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/lib/oeqa/selftest/base.py b/meta/lib/oeqa/selftest/base.py
index 26c93f9..47a8ea8 100644
--- a/meta/lib/oeqa/selftest/base.py
+++ b/meta/lib/oeqa/selftest/base.py
@@ -163,7 +163,7 @@ be re-executed from a clean environment to ensure accurate results.")
 
     # remove data from <builddir>/conf/selftest.inc
     def remove_config(self, data):
-        self.log.debug("Removing from: %s\n\%s\n" % (self.testinc_path, data))
+        self.log.debug("Removing from: %s\n%s\n" % (self.testinc_path, data))
         ftools.remove_from_file(self.testinc_path, data)
 
     # write to meta-sefltest/recipes-test/<recipe>/test_recipe.inc
@@ -206,7 +206,7 @@ be re-executed from a clean environment to ensure accurate results.")
 
     # remove data from <builddir>/conf/bblayers.inc
     def remove_bblayers_config(self, data):
-        self.log.debug("Removing from: %s\n\%s\n" % (self.testinc_bblayers_path, data))
+        self.log.debug("Removing from: %s\n%s\n" % (self.testinc_bblayers_path, data))
         ftools.remove_from_file(self.testinc_bblayers_path, data)
 
     # write to <builddir>/conf/machine.inc
-- 
2.1.4