[PATCH 1/3] oeqa/targetcontrol: Add support for poky-tiny in QemuTarget.

[PATCH 1/3] oeqa/targetcontrol: Add support for poky-tiny in QemuTarget.

[Lucian Musat]

Signed-off-by: Lucian Musat <george.l.musat@intel.com>
---
 meta/lib/oeqa/targetcontrol.py | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/meta/lib/oeqa/targetcontrol.py b/meta/lib/oeqa/targetcontrol.py
index 1f4770f..9a681a3 100644
--- a/meta/lib/oeqa/targetcontrol.py
+++ b/meta/lib/oeqa/targetcontrol.py
@@ -12,6 +12,7 @@ import traceback
 import sys
 from oeqa.utils.sshcontrol import SSHControl
 from oeqa.utils.qemurunner import QemuRunner
+from oeqa.utils.qemutinyrunner import QemuTinyRunner
 from oeqa.controllers.testtargetloader import TestTargetLoader
 from abc import ABCMeta, abstractmethod
 
@@ -110,7 +111,7 @@ class BaseTarget(object):
 
 class QemuTarget(BaseTarget):
 
-    supported_image_fstypes = ['ext3', 'ext4']
+    supported_image_fstypes = ['ext3', 'ext4', 'cpio.gz']
 
     def __init__(self, d):
 
@@ -120,14 +121,25 @@ class QemuTarget(BaseTarget):
         self.qemulog = os.path.join(self.testdir, "qemu_boot_log.%s" % self.datetime)
         self.origrootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True),  d.getVar("IMAGE_LINK_NAME", True) + '.' + self.image_fstype)
         self.rootfs = os.path.join(self.testdir, d.getVar("IMAGE_LINK_NAME", True) + '-testimage.' + self.image_fstype)
-
-        self.runner = QemuRunner(machine=d.getVar("MACHINE", True),
-                        rootfs=self.rootfs,
-                        tmpdir = d.getVar("TMPDIR", True),
-                        deploy_dir_image = d.getVar("DEPLOY_DIR_IMAGE", True),
-                        display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY", True),
-                        logfile = self.qemulog,
-                        boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT", True)))
+        self.kernel = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("KERNEL_IMAGETYPE") + '-' + d.getVar('MACHINE') + '.bin')
+
+        if d.getVar("DISTRO", True) == "poky-tiny":
+            self.runner = QemuTinyRunner(machine=d.getVar("MACHINE", True),
+                            rootfs=self.rootfs,
+                            tmpdir = d.getVar("TMPDIR", True),
+                            deploy_dir_image = d.getVar("DEPLOY_DIR_IMAGE", True),
+                            display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY", True),
+                            logfile = self.qemulog,
+                            kernel = self.kernel,
+                            boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT", True)))
+        else:
+            self.runner = QemuRunner(machine=d.getVar("MACHINE", True),
+                            rootfs=self.rootfs,
+                            tmpdir = d.getVar("TMPDIR", True),
+                            deploy_dir_image = d.getVar("DEPLOY_DIR_IMAGE", True),
+                            display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY", True),
+                            logfile = self.qemulog,
+                            boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT", True)))
 
     def deploy(self):
         try:
@@ -167,6 +179,9 @@ class QemuTarget(BaseTarget):
         else:
             raise bb.build.FuncFailed("%s - FAILED to re-start qemu - check the task log and the boot log" % self.pn)
 
+    def run_serial(self, command):
+        return self.runner.run_serial(command)
+
 
 class SimpleRemoteTarget(BaseTarget):
 
-- 
2.1.0

[PATCH 2/3] oeqa/utils: Add runner for poky-tiny qemu.

[Lucian Musat]

The connection and commands are done via serial.

Signed-off-by: Lucian Musat <george.l.musat@intel.com>
---
 meta/lib/oeqa/utils/qemutinyrunner.py | 170 ++++++++++++++++++++++++++++++++++
 1 file changed, 170 insertions(+)
 create mode 100644 meta/lib/oeqa/utils/qemutinyrunner.py

diff --git a/meta/lib/oeqa/utils/qemutinyrunner.py b/meta/lib/oeqa/utils/qemutinyrunner.py
new file mode 100644
index 0000000..4f95101
--- /dev/null
+++ b/meta/lib/oeqa/utils/qemutinyrunner.py
@@ -0,0 +1,170 @@
+# Copyright (C) 2015 Intel Corporation
+#
+# Released under the MIT license (see COPYING.MIT)
+
+# This module provides a class for starting qemu images of poky tiny.
+# It's used by testimage.bbclass.
+
+import subprocess
+import os
+import time
+import signal
+import re
+import socket
+import select
+import bb
+from qemurunner import QemuRunner
+
+class QemuTinyRunner(QemuRunner):
+
+    def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime):
+
+        # Popen object for runqemu
+        self.runqemu = None
+        # pid of the qemu process that runqemu will start
+        self.qemupid = None
+        # target ip - from the command line
+        self.ip = None
+        # host ip - where qemu is running
+        self.server_ip = None
+
+        self.machine = machine
+        self.rootfs = rootfs
+        self.display = display
+        self.tmpdir = tmpdir
+        self.deploy_dir_image = deploy_dir_image
+        self.logfile = logfile
+        self.boottime = boottime
+
+        self.runqemutime = 60
+        self.socketfile = "console.sock"
+        self.server_socket = None
+        self.kernel = kernel
+
+
+    def create_socket(self):
+        tries = 3
+        while tries > 0:
+            try:
+                self.server_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+                self.server_socket.connect(self.socketfile)
+                bb.note("Created listening socket for qemu serial console.")
+                tries = 0
+            except socket.error, msg:
+                self.server_socket.close()
+                bb.fatal("Failed to create listening socket.")
+                tries -= 1
+
+    def log(self, msg):
+        if self.logfile:
+            with open(self.logfile, "a") as f:
+                f.write("%s" % msg)
+
+    def start(self, qemuparams = None):
+
+        if self.display:
+            os.environ["DISPLAY"] = self.display
+        else:
+            bb.error("To start qemu I need a X desktop, please set DISPLAY correctly (e.g. DISPLAY=:1)")
+            return False
+        if not os.path.exists(self.rootfs):
+            bb.error("Invalid rootfs %s" % self.rootfs)
+            return False
+        if not os.path.exists(self.tmpdir):
+            bb.error("Invalid TMPDIR path %s" % self.tmpdir)
+            return False
+        else:
+            os.environ["OE_TMPDIR"] = self.tmpdir
+        if not os.path.exists(self.deploy_dir_image):
+            bb.error("Invalid DEPLOY_DIR_IMAGE path %s" % self.deploy_dir_image)
+            return False
+        else:
+            os.environ["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image
+
+        # Set this flag so that Qemu doesn't do any grabs as SDL grabs interact
+        # badly with screensavers.
+        os.environ["QEMU_DONT_GRAB"] = "1"
+        self.qemuparams = '--append "root=/dev/ram0 console=ttyS0" -nographic -serial unix:%s,server,nowait' % self.socketfile
+
+        launch_cmd = 'qemu-system-i386 -kernel %s -initrd %s %s' % (self.kernel, self.rootfs, self.qemuparams)
+        self.runqemu = subprocess.Popen(launch_cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,preexec_fn=os.setpgrp)
+
+        bb.note("runqemu started, pid is %s" % self.runqemu.pid)
+        bb.note("waiting at most %s seconds for qemu pid" % self.runqemutime)
+        endtime = time.time() + self.runqemutime
+        while not self.is_alive() and time.time() < endtime:
+            time.sleep(1)
+
+        if self.is_alive():
+            bb.note("qemu started - qemu procces pid is %s" % self.qemupid)
+            self.create_socket()
+        else:
+            bb.note("Qemu pid didn't appeared in %s seconds" % self.runqemutime)
+            output = self.runqemu.stdout
+            self.stop()
+            bb.note("Output from runqemu:\n%s" % output.read())
+            return False
+
+        return self.is_alive()
+
+    def run_serial(self, command):
+        self.server_socket.sendall(command+'\n')
+        data = ''
+        status = 0
+        stopread = False
+        endtime = time.time()+5
+        while time.time()<endtime and not stopread:
+                sread, _, _ = select.select([self.server_socket],[],[],5)
+                for sock in sread:
+                        answer = sock.recv(1024)
+                        if answer:
+                                data += answer
+                        else:
+                                sock.close()
+                                stopread = True
+        if not data:
+            status = 1
+        return (status, str(data))
+
+    def find_child(self,parent_pid):
+        #
+        # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd]
+        #
+        ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0]
+        processes = ps.split('\n')
+        nfields = len(processes[0].split()) - 1
+        pids = {}
+        commands = {}
+        for row in processes[1:]:
+            data = row.split(None, nfields)
+            if len(data) != 3:
+                continue
+            if data[1] not in pids:
+                pids[data[1]] = []
+
+            pids[data[1]].append(data[0])
+            commands[data[0]] = data[2]
+
+        if parent_pid not in pids:
+            return []
+
+        parents = []
+        newparents = pids[parent_pid]
+        while newparents:
+            next = []
+            for p in newparents:
+                if p in pids:
+                    for n in pids[p]:
+                        if n not in parents and n not in next:
+                            next.append(n)
+                if p not in parents:
+                    parents.append(p)
+                    newparents = next
+        #print "Children matching %s:" % str(parents)
+        for p in parents:
+            # Need to be careful here since runqemu-internal runs "ldd qemu-system-xxxx"
+            # Also, old versions of ldd (2.11) run "LD_XXXX qemu-system-xxxx"
+            basecmd = commands[p].split()[0]
+            basecmd = os.path.basename(basecmd)
+            if "qemu-system" in basecmd and "-serial unix" in commands[p]:
+                return [int(p),commands[p]]
\ No newline at end of file
-- 
2.1.0

[PATCH 3/3] oeqa/runtime: Boot test for poky-tiny.

[Lucian Musat]

Bug 6705.

Signed-off-by: Lucian Musat <george.l.musat@intel.com>
---
 meta/lib/oeqa/runtime/_qemutiny.py | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 meta/lib/oeqa/runtime/_qemutiny.py

diff --git a/meta/lib/oeqa/runtime/_qemutiny.py b/meta/lib/oeqa/runtime/_qemutiny.py
new file mode 100644
index 0000000..a3c29f3
--- /dev/null
+++ b/meta/lib/oeqa/runtime/_qemutiny.py
@@ -0,0 +1,9 @@
+import unittest
+from oeqa.oetest import oeRuntimeTest
+from oeqa.utils.qemutinyrunner import *
+
+class QemuTinyTest(oeRuntimeTest):
+
+    def test_boot_tiny(self):
+        (status, output) = self.target.run_serial('uname -a')
+        self.assertTrue("yocto-tiny" in output, msg="Cannot detect poky tiny boot!")
\ No newline at end of file
-- 
2.1.0

Re: [PATCH 2/3] oeqa/utils: Add runner for poky-tiny qemu.

[Randy Witt]

On 04/09/2015 01:08 AM, Lucian Musat wrote:
> The connection and commands are done via serial.
>
> Signed-off-by: Lucian Musat <george.l.musat@intel.com>
> ---
>   meta/lib/oeqa/utils/qemutinyrunner.py | 170 ++++++++++++++++++++++++++++++++++
>   1 file changed, 170 insertions(+)
>   create mode 100644 meta/lib/oeqa/utils/qemutinyrunner.py
>
> diff --git a/meta/lib/oeqa/utils/qemutinyrunner.py b/meta/lib/oeqa/utils/qemutinyrunner.py
> new file mode 100644
> index 0000000..4f95101
> --- /dev/null
> +++ b/meta/lib/oeqa/utils/qemutinyrunner.py
> @@ -0,0 +1,170 @@
> +# Copyright (C) 2015 Intel Corporation
> +#
> +# Released under the MIT license (see COPYING.MIT)
> +
> +# This module provides a class for starting qemu images of poky tiny.
> +# It's used by testimage.bbclass.
> +
> +import subprocess
> +import os
> +import time
> +import signal
> +import re
> +import socket
> +import select
> +import bb
> +from qemurunner import QemuRunner
> +
> +class QemuTinyRunner(QemuRunner):
> +
> +    def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime):
> +
> +        # Popen object for runqemu
> +        self.runqemu = None
> +        # pid of the qemu process that runqemu will start
> +        self.qemupid = None
> +        # target ip - from the command line
> +        self.ip = None
> +        # host ip - where qemu is running
> +        self.server_ip = None
> +
> +        self.machine = machine
> +        self.rootfs = rootfs
> +        self.display = display
> +        self.tmpdir = tmpdir
> +        self.deploy_dir_image = deploy_dir_image
> +        self.logfile = logfile
> +        self.boottime = boottime
> +
> +        self.runqemutime = 60
> +        self.socketfile = "console.sock"
> +        self.server_socket = None
> +        self.kernel = kernel
> +
> +
> +    def create_socket(self):
> +        tries = 3
> +        while tries > 0:
> +            try:
> +                self.server_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
> +                self.server_socket.connect(self.socketfile)
> +                bb.note("Created listening socket for qemu serial console.")
> +                tries = 0
> +            except socket.error, msg:
> +                self.server_socket.close()
> +                bb.fatal("Failed to create listening socket.")
> +                tries -= 1
> +
> +    def log(self, msg):
> +        if self.logfile:
> +            with open(self.logfile, "a") as f:
> +                f.write("%s" % msg)
> +
> +    def start(self, qemuparams = None):
> +
> +        if self.display:
> +            os.environ["DISPLAY"] = self.display
> +        else:
> +            bb.error("To start qemu I need a X desktop, please set DISPLAY correctly (e.g. DISPLAY=:1)")
> +            return False
> +        if not os.path.exists(self.rootfs):
> +            bb.error("Invalid rootfs %s" % self.rootfs)
> +            return False
> +        if not os.path.exists(self.tmpdir):
> +            bb.error("Invalid TMPDIR path %s" % self.tmpdir)
> +            return False
> +        else:
> +            os.environ["OE_TMPDIR"] = self.tmpdir
> +        if not os.path.exists(self.deploy_dir_image):
> +            bb.error("Invalid DEPLOY_DIR_IMAGE path %s" % self.deploy_dir_image)
> +            return False
> +        else:
> +            os.environ["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image
> +
> +        # Set this flag so that Qemu doesn't do any grabs as SDL grabs interact
> +        # badly with screensavers.
> +        os.environ["QEMU_DONT_GRAB"] = "1"
> +        self.qemuparams = '--append "root=/dev/ram0 console=ttyS0" -nographic -serial unix:%s,server,nowait' % self.socketfile
> +
> +        launch_cmd = 'qemu-system-i386 -kernel %s -initrd %s %s' % (self.kernel, self.rootfs, self.qemuparams)
> +        self.runqemu = subprocess.Popen(launch_cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,preexec_fn=os.setpgrp)
> +
> +        bb.note("runqemu started, pid is %s" % self.runqemu.pid)
> +        bb.note("waiting at most %s seconds for qemu pid" % self.runqemutime)
> +        endtime = time.time() + self.runqemutime
> +        while not self.is_alive() and time.time() < endtime:
> +            time.sleep(1)
> +
> +        if self.is_alive():
> +            bb.note("qemu started - qemu procces pid is %s" % self.qemupid)
> +            self.create_socket()
> +        else:
> +            bb.note("Qemu pid didn't appeared in %s seconds" % self.runqemutime)
> +            output = self.runqemu.stdout
> +            self.stop()
> +            bb.note("Output from runqemu:\n%s" % output.read())
> +            return False
> +
> +        return self.is_alive()
> +
> +    def run_serial(self, command):
> +        self.server_socket.sendall(command+'\n')
> +        data = ''
> +        status = 0
> +        stopread = False
> +        endtime = time.time()+5
> +        while time.time()<endtime and not stopread:
> +                sread, _, _ = select.select([self.server_socket],[],[],5)
> +                for sock in sread:
> +                        answer = sock.recv(1024)
> +                        if answer:
> +                                data += answer
> +                        else:
> +                                sock.close()
> +                                stopread = True
> +        if not data:
> +            status = 1
> +        return (status, str(data))
> +
> +    def find_child(self,parent_pid):
> +        #
> +        # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd]
> +        #
> +        ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0]
> +        processes = ps.split('\n')
> +        nfields = len(processes[0].split()) - 1
> +        pids = {}
> +        commands = {}
> +        for row in processes[1:]:
> +            data = row.split(None, nfields)
> +            if len(data) != 3:
> +                continue
> +            if data[1] not in pids:
> +                pids[data[1]] = []
> +
> +            pids[data[1]].append(data[0])
> +            commands[data[0]] = data[2]
> +
> +        if parent_pid not in pids:
> +            return []
> +
> +        parents = []
> +        newparents = pids[parent_pid]
> +        while newparents:
> +            next = []
> +            for p in newparents:
> +                if p in pids:
> +                    for n in pids[p]:
> +                        if n not in parents and n not in next:
> +                            next.append(n)
> +                if p not in parents:
> +                    parents.append(p)
> +                    newparents = next
> +        #print "Children matching %s:" % str(parents)
> +        for p in parents:
> +            # Need to be careful here since runqemu-internal runs "ldd qemu-system-xxxx"
> +            # Also, old versions of ldd (2.11) run "LD_XXXX qemu-system-xxxx"
> +            basecmd = commands[p].split()[0]
> +            basecmd = os.path.basename(basecmd)
> +            if "qemu-system" in basecmd and "-serial unix" in commands[p]:
> +                return [int(p),commands[p]]
> \ No newline at end of file
>

Lucian, since this is derived from the base class QemuRunner, could we not add 
some more parameters to the base class so that we can have more code reuse by 
calling the base class functions? Most of the lines are copied straight from 
QemuRunner. And log() was copied verbatim.

Re: [PATCH 3/3] oeqa/runtime: Boot test for poky-tiny.

[Paul Eggleton]

Hi Lucian,

On Thursday 09 April 2015 11:08:12 Lucian Musat wrote:
> Bug 6705.

* Could you please add a proper commit message to the patches in this series 
explaining what each one does (and preferably why). I appreciate most of it is 
simply adding something new, and a lot of us have gotten away with not adding 
them in the past, but we still really ought to have a commit message for every 
change. You may have seen this already but in case you haven't here is a 
guide:

http://www.openembedded.org/wiki/Commit_Patch_Message_Guidelines

* Can you also please use the standard syntax for referencing bugs in the YP 
bugzilla i.e. [YOCTO #6705]

Thanks very much,
Paul


-- 

Paul Eggleton
Intel Open Source Technology Centre