Re: [Webhob] [PATCH 3/3] Create Data Store Interface (DSI) file

[Calin Dragomir <calinx.l.dragomir@intel.com>]

Hello Cristian,

You are not wrong, but that set of data is not ready yet.
We are working on integrating that information into an event object 
which will come from Bitbake.
Until that is ready, this whole chunk of information is put here for 
testing reasons:

+                                            # The next lines are just for testing reasons
+                                            task_id=1,
+                                            path_to_sstate_obj='/home/calin',
+                                            source_url='/',
+                                            log_file='/',
+                                            work_directory='/',
+                                            script_type='python',
+                                            file_path='/',
+                                            line_number=55,
+                                            py_stack_trace='Testing traceback',
+                                            disk_io = 23.55633,
+                                            cpu_usage=43,
+                                            dependent_tasks='Task 1',
+                                            errors_no=0,
+                                            warnings_no=0,
+                                            error='None',
+                                            warning='None',
+                                            sstate_result='not_applicable')


Thank you for the feedback,
Calin


On 12.06.2013 12:53, Iorga, Cristian wrote:
> Hello Calin,
>
> I might be wrong, but this:
>
> path_to_sstate_obj='/home/calin',
>
> does not seem right.
>
> Regards,
> Cristian
>
> -----Original Message-----
> From: webhob-bounces@yoctoproject.org [mailto:webhob-bounces@yoctoproject.org] On Behalf Of Calin Dragomir
> Sent: Wednesday, June 12, 2013 12:30 PM
> To: webhob@yoctoproject.org
> Subject: [Webhob] [PATCH 3/3] Create Data Store Interface (DSI) file
>
> This adds the first version of the DSI file. It uses the Knotty code for now, but inserts task related information into the database using the Django ORM
> ---
>   bitbake/lib/bb/ui/dsi.py | 609 +++++++++++++++++++++++++++++++++++++++++++++++
>   1 file changed, 609 insertions(+)
>   create mode 100644 bitbake/lib/bb/ui/dsi.py
>
> diff --git a/bitbake/lib/bb/ui/dsi.py b/bitbake/lib/bb/ui/dsi.py new file mode 100644 index 0000000..4bcd509
> --- /dev/null
> +++ b/bitbake/lib/bb/ui/dsi.py
> @@ -0,0 +1,609 @@
> +#
> +# BitBake (No)TTY UI Implementation
> +#
> +# Handling output to TTYs or files (no TTY) # # Copyright (C) 2006-2012
> +Richard Purdie # # This program is free software; you can redistribute
> +it and/or modify # it under the terms of the GNU General Public License
> +version 2 as # published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it will be useful, # but
> +WITHOUT ANY WARRANTY; without even the implied warranty of #
> +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the # GNU
> +General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +along # with this program; if not, write to the Free Software
> +Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> +
> +from __future__ import division
> +
> +import os
> +import sys
> +import xmlrpclib
> +import logging
> +import progressbar
> +import signal
> +import bb.msg
> +import time
> +import fcntl
> +import struct
> +import copy
> +import datetime
> +
> +from bb.ui import uihelper
> +
> +logger = logging.getLogger("BitBake")
> +interactive = sys.stdout.isatty()
> +
> +class WebHOBHelper(object):
> +
> +    def __init__(self):
> +        self.configure_django()
> +        self.task_order = 0
> +        self.tasks_information = {}
> +
> +    def configure_django(self):
> +        import webhob.whbmain.settings as whb_django_settings
> +        from django.core.management import setup_environ
> +        setup_environ(whb_django_settings)
> +        # Add webhob to sys path for importing modules
> +
> + sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.d
> + irname(__file__))), 'webhob'))
> +
> +    def store_started_task(self, event, uuid):
> +        self.task_order += 1
> +        self.tasks_information[event.pid] = {
> +                'uuid': uuid,
> +                'task_id': 1,
> +                'task_executed': True,
> +                'order': self.task_order,
> +                'recipe': event._package,
> +                'task_name': event._task,
> +                'start_time': time.time(),
> +                }
> +
> +    def update_stored_tasks(self, event, uuid):
> +        self.tasks_information[event.pid].update({
> +                'outcome': event.getDisplayName().lower(),
> +                'end_time': time.time(),
> +                })
> +
> +    def write_in_database(self, tasks_information):
> +        # This needs to be imported after we have configured the Django settings file
> +        from webhob.orm.models import Tasks
> +        for pid in tasks_information.keys():
> +            task = tasks_information[pid]
> +
> +            task_obj = Tasks.objects.create(uuid=task['uuid'],
> +                                            order=task['order'],
> +                                            outcome=task['outcome'],
> +                                            recipe=task['recipe'],
> +                                            task_name=task['task_name'],
> +                                            elapsed_time=task['end_time'] - task['start_time'],
> +                                            # The next lines are just for testing reasons
> +                                            task_id=1,
> +                                            path_to_sstate_obj='/home/calin',
> +                                            source_url='/',
> +                                            log_file='/',
> +                                            work_directory='/',
> +                                            script_type='python',
> +                                            file_path='/',
> +                                            line_number=55,
> +                                            py_stack_trace='Testing traceback',
> +                                            disk_io = 23.55633,
> +                                            cpu_usage=43,
> +                                            dependent_tasks='Task 1',
> +                                            errors_no=0,
> +                                            warnings_no=0,
> +                                            error='None',
> +                                            warning='None',
> +                                            sstate_result='not_applicable')
> +            task_obj.save()
> +
> +class BBProgress(progressbar.ProgressBar):
> +    def __init__(self, msg, maxval):
> +        self.msg = msg
> +        widgets = [progressbar.Percentage(), ' ', progressbar.Bar(), ' ',
> +           progressbar.ETA()]
> +
> +        try:
> +            self._resize_default = signal.getsignal(signal.SIGWINCH)
> +        except:
> +            self._resize_default = None
> +        progressbar.ProgressBar.__init__(self, maxval, [self.msg + ":
> + "] + widgets)
> +
> +    def _handle_resize(self, signum, frame):
> +        progressbar.ProgressBar._handle_resize(self, signum, frame)
> +        if self._resize_default:
> +            self._resize_default(signum, frame)
> +    def finish(self):
> +        progressbar.ProgressBar.finish(self)
> +        if self._resize_default:
> +            signal.signal(signal.SIGWINCH, self._resize_default)
> +
> +class NonInteractiveProgress(object):
> +    fobj = sys.stdout
> +
> +    def __init__(self, msg, maxval):
> +        self.msg = msg
> +        self.maxval = maxval
> +
> +    def start(self):
> +        self.fobj.write("%s..." % self.msg)
> +        self.fobj.flush()
> +        return self
> +
> +    def update(self, value):
> +        pass
> +
> +    def finish(self):
> +        self.fobj.write("done.\n")
> +        self.fobj.flush()
> +
> +def new_progress(msg, maxval):
> +    if interactive:
> +        return BBProgress(msg, maxval)
> +    else:
> +        return NonInteractiveProgress(msg, maxval)
> +
> +def pluralise(singular, plural, qty):
> +    if(qty == 1):
> +        return singular % qty
> +    else:
> +        return plural % qty
> +
> +
> +class InteractConsoleLogFilter(logging.Filter):
> +    def __init__(self, tf, format):
> +        self.tf = tf
> +        self.format = format
> +
> +    def filter(self, record):
> +        if record.levelno == self.format.NOTE and (record.msg.startswith("Running") or record.msg.startswith("recipe ")):
> +            return False
> +        self.tf.clearFooter()
> +        return True
> +
> +class TerminalFilter(object):
> +    columns = 80
> +
> +    def sigwinch_handle(self, signum, frame):
> +        self.columns = self.getTerminalColumns()
> +        if self._sigwinch_default:
> +            self._sigwinch_default(signum, frame)
> +
> +    def getTerminalColumns(self):
> +        def ioctl_GWINSZ(fd):
> +            try:
> +                cr = struct.unpack('hh', fcntl.ioctl(fd, self.termios.TIOCGWINSZ, '1234'))
> +            except:
> +                return None
> +            return cr
> +        cr = ioctl_GWINSZ(sys.stdout.fileno())
> +        if not cr:
> +            try:
> +                fd = os.open(os.ctermid(), os.O_RDONLY)
> +                cr = ioctl_GWINSZ(fd)
> +                os.close(fd)
> +            except:
> +                pass
> +        if not cr:
> +            try:
> +                cr = (env['LINES'], env['COLUMNS'])
> +            except:
> +                cr = (25, 80)
> +        return cr[1]
> +
> +    def __init__(self, main, helper, console, format):
> +        self.main = main
> +        self.helper = helper
> +        self.cuu = None
> +        self.stdinbackup = None
> +        self.interactive = sys.stdout.isatty()
> +        self.footer_present = False
> +        self.lastpids = []
> +
> +        if not self.interactive:
> +            return
> +
> +        try:
> +            import curses
> +        except ImportError:
> +            sys.exit("FATAL: The knotty ui could not load the required
> + curses python module.")
> +
> +        import termios
> +        self.curses = curses
> +        self.termios = termios
> +        try:
> +            fd = sys.stdin.fileno()
> +            self.stdinbackup = termios.tcgetattr(fd)
> +            new = copy.deepcopy(self.stdinbackup)
> +            new[3] = new[3] & ~termios.ECHO
> +            termios.tcsetattr(fd, termios.TCSADRAIN, new)
> +            curses.setupterm()
> +            if curses.tigetnum("colors") > 2:
> +                format.enable_color()
> +            self.ed = curses.tigetstr("ed")
> +            if self.ed:
> +                self.cuu = curses.tigetstr("cuu")
> +            try:
> +                self._sigwinch_default = signal.getsignal(signal.SIGWINCH)
> +                signal.signal(signal.SIGWINCH, self.sigwinch_handle)
> +            except:
> +                pass
> +            self.columns = self.getTerminalColumns()
> +        except:
> +            self.cuu = None
> +        console.addFilter(InteractConsoleLogFilter(self, format))
> +
> +    def clearFooter(self):
> +        if self.footer_present:
> +            lines = self.footer_present
> +            sys.stdout.write(self.curses.tparm(self.cuu, lines))
> +            sys.stdout.write(self.curses.tparm(self.ed))
> +        self.footer_present = False
> +
> +    def updateFooter(self):
> +        if not self.cuu:
> +            return
> +        activetasks = self.helper.running_tasks
> +        failedtasks = self.helper.failed_tasks
> +        runningpids = self.helper.running_pids
> +        if self.footer_present and (self.lastcount == self.helper.tasknumber_current) and (self.lastpids == runningpids):
> +            return
> +        if self.footer_present:
> +            self.clearFooter()
> +        if (not self.helper.tasknumber_total or self.helper.tasknumber_current == self.helper.tasknumber_total) and not len(activetasks):
> +            return
> +        tasks = []
> +        for t in runningpids:
> +            tasks.append("%s (pid %s)" % (activetasks[t]["title"], t))
> +
> +        if self.main.shutdown:
> +            content = "Waiting for %s running tasks to finish:" % len(activetasks)
> +        elif not len(activetasks):
> +            content = "No currently running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
> +        else:
> +            content = "Currently %s running tasks (%s of %s):" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total)
> +        print(content)
> +        lines = 1 + int(len(content) / (self.columns + 1))
> +        for tasknum, task in enumerate(tasks):
> +            content = "%s: %s" % (tasknum, task)
> +            print(content)
> +            lines = lines + 1 + int(len(content) / (self.columns + 1))
> +        self.footer_present = lines
> +        self.lastpids = runningpids[:]
> +        self.lastcount = self.helper.tasknumber_current
> +
> +    def finish(self):
> +        if self.stdinbackup:
> +            fd = sys.stdin.fileno()
> +            self.termios.tcsetattr(fd, self.termios.TCSADRAIN,
> + self.stdinbackup)
> +
> +def main(server, eventHandler, params, tf = TerminalFilter):
> +    print "This is running in DSI mode"
> +
> +    # Generate an unique ID for this build
> +    # TODO: for multiple build commands this might not work as expected
> +    import uuid
> +    uuid = sessionid = str(uuid.uuid4())
> +
> +    # Get values of variables which control our output
> +    includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"])
> +    if error:
> +        logger.error("Unable to get the value of BBINCLUDELOGS variable: %s" % error)
> +        return 1
> +    loglines, error = server.runCommand(["getVariable", "BBINCLUDELOGS_LINES"])
> +    if error:
> +        logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s" % error)
> +        return 1
> +    consolelogfile, error = server.runCommand(["getVariable", "BB_CONSOLELOG"])
> +    if error:
> +        logger.error("Unable to get the value of BB_CONSOLELOG variable: %s" % error)
> +        return 1
> +
> +    if sys.stdin.isatty() and sys.stdout.isatty():
> +        log_exec_tty = True
> +    else:
> +        log_exec_tty = False
> +
> +    helper = uihelper.BBUIHelper()
> +
> +    console = logging.StreamHandler(sys.stdout)
> +    format_str = "%(levelname)s: %(message)s"
> +    format = bb.msg.BBLogFormatter(format_str)
> +    bb.msg.addDefaultlogFilter(console)
> +    console.setFormatter(format)
> +    logger.addHandler(console)
> +
> +    if consolelogfile and not params.options.show_environment:
> +        bb.utils.mkdirhier(os.path.dirname(consolelogfile))
> +        conlogformat = bb.msg.BBLogFormatter(format_str)
> +        consolelog = logging.FileHandler(consolelogfile)
> +        bb.msg.addDefaultlogFilter(consolelog)
> +        consolelog.setFormatter(conlogformat)
> +        logger.addHandler(consolelog)
> +
> +    try:
> +        params.updateFromServer(server)
> +        cmdline = params.parseActions()
> +        if not cmdline:
> +            print("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
> +            return 1
> +        if 'msg' in cmdline and cmdline['msg']:
> +            logger.error(cmdline['msg'])
> +            return 1
> +
> +        ret, error = server.runCommand(cmdline['action'])
> +        if error:
> +            logger.error("Command '%s' failed: %s" % (cmdline, error))
> +            return 1
> +        elif ret != True:
> +            logger.error("Command '%s' failed: returned %s" % (cmdline, ret))
> +            return 1
> +    except xmlrpclib.Fault as x:
> +        logger.error("XMLRPC Fault getting commandline:\n %s" % x)
> +        return 1
> +
> +    parseprogress = None
> +    cacheprogress = None
> +    main.shutdown = 0
> +    interrupted = False
> +    return_value = 0
> +    errors = 0
> +    warnings = 0
> +    taskfailures = []
> +    wbhbhelper = WebHOBHelper()
> +
> +    termfilter = tf(main, helper, console, format)
> +
> +    while True:
> +        try:
> +            termfilter.updateFooter()
> +            event = eventHandler.waitEvent(0.25)
> +
> +            if event is None:
> +                if main.shutdown > 1:
> +                    break
> +                continue
> +
> +            helper.eventHandler(event)
> +
> +            if isinstance(event, bb.runqueue.runQueueExitWait):
> +                if not main.shutdown:
> +                    main.shutdown = 1
> +
> +            if isinstance(event, bb.build.TaskStarted):
> +                wbhbhelper.store_started_task(event, uuid)
> +
> +            if isinstance(event, (bb.build.TaskSucceeded, bb.build.TaskFailedSilent, bb.build.TaskFailed)):
> +                wbhbhelper.update_stored_tasks(event, uuid)
> +
> +            if isinstance(event, bb.event.LogExecTTY):
> +                if log_exec_tty:
> +                    tries = event.retries
> +                    while tries:
> +                        print("Trying to run: %s" % event.prog)
> +                        if os.system(event.prog) == 0:
> +                            break
> +                        time.sleep(event.sleep_delay)
> +                        tries -= 1
> +                    if tries:
> +                        continue
> +                logger.warn(event.msg)
> +                continue
> +
> +            if isinstance(event, logging.LogRecord):
> +                if event.levelno >= format.ERROR:
> +                    errors = errors + 1
> +                    return_value = 1
> +                elif event.levelno == format.WARNING:
> +                    warnings = warnings + 1
> +                # For "normal" logging conditions, don't show note logs from tasks
> +                # but do show them if the user has changed the default log level to
> +                # include verbose/debug messages
> +                if event.taskpid != 0 and event.levelno <= format.NOTE:
> +                    continue
> +                logger.handle(event)
> +                continue
> +
> +            if isinstance(event, bb.build.TaskFailed):
> +                return_value = 1
> +                logfile = event.logfile
> +                if logfile and os.path.exists(logfile):
> +                    termfilter.clearFooter()
> +                    bb.error("Logfile of failure stored in: %s" % logfile)
> +                    if includelogs and not event.errprinted:
> +                        print("Log data follows:")
> +                        f = open(logfile, "r")
> +                        lines = []
> +                        while True:
> +                            l = f.readline()
> +                            if l == '':
> +                                break
> +                            l = l.rstrip()
> +                            if loglines:
> +                                lines.append(' | %s' % l)
> +                                if len(lines) > int(loglines):
> +                                    lines.pop(0)
> +                            else:
> +                                print('| %s' % l)
> +                        f.close()
> +                        if lines:
> +                            for line in lines:
> +                                print(line)
> +            if isinstance(event, bb.build.TaskBase):
> +                logger.info(event._message)
> +                continue
> +            if isinstance(event, bb.event.ParseStarted):
> +                if event.total == 0:
> +                    continue
> +                parseprogress = new_progress("Parsing recipes", event.total).start()
> +                continue
> +            if isinstance(event, bb.event.ParseProgress):
> +                parseprogress.update(event.current)
> +                continue
> +            if isinstance(event, bb.event.ParseCompleted):
> +                if not parseprogress:
> +                    continue
> +
> +                parseprogress.finish()
> +                print(("Parsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors."
> +                    % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors)))
> +                continue
> +
> +            if isinstance(event, bb.event.CacheLoadStarted):
> +                cacheprogress = new_progress("Loading cache", event.total).start()
> +                continue
> +            if isinstance(event, bb.event.CacheLoadProgress):
> +                cacheprogress.update(event.current)
> +                continue
> +            if isinstance(event, bb.event.CacheLoadCompleted):
> +                cacheprogress.finish()
> +                print("Loaded %d entries from dependency cache." % event.num_entries)
> +                continue
> +
> +            if isinstance(event, bb.command.CommandFailed):
> +                return_value = event.exitcode
> +                errors = errors + 1
> +                logger.error("Command execution failed: %s", event.error)
> +                main.shutdown = 2
> +                continue
> +            if isinstance(event, bb.command.CommandExit):
> +                if not return_value:
> +                    return_value = event.exitcode
> +                continue
> +            if isinstance(event, (bb.command.CommandCompleted, bb.cooker.CookerExit)):
> +                main.shutdown = 2
> +                continue
> +            if isinstance(event, bb.event.MultipleProviders):
> +                logger.info("multiple providers are available for %s%s (%s)", event._is_runtime and "runtime " or "",
> +                            event._item,
> +                            ", ".join(event._candidates))
> +                logger.info("consider defining a PREFERRED_PROVIDER entry to match %s", event._item)
> +                continue
> +            if isinstance(event, bb.event.NoProvider):
> +                return_value = 1
> +                errors = errors + 1
> +                if event._runtime:
> +                    r = "R"
> +                else:
> +                    r = ""
> +
> +                if event._dependees:
> +                    logger.error("Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)", r, event._item, ", ".join(event._dependees), r)
> +                else:
> +                    logger.error("Nothing %sPROVIDES '%s'", r, event._item)
> +                if event._reasons:
> +                    for reason in event._reasons:
> +                        logger.error("%s", reason)
> +                continue
> +
> +            if isinstance(event, bb.runqueue.sceneQueueTaskStarted):
> +                logger.info("Running setscene task %d of %d (%s)" % (event.stats.completed + event.stats.active + event.stats.failed + 1, event.stats.total, event.taskstring))
> +                continue
> +
> +            if isinstance(event, bb.runqueue.runQueueTaskStarted):
> +                if event.noexec:
> +                    tasktype = 'noexec task'
> +                else:
> +                    tasktype = 'task'
> +                logger.info("Running %s %s of %s (ID: %s, %s)",
> +                            tasktype,
> +                            event.stats.completed + event.stats.active +
> +                                event.stats.failed + 1,
> +                            event.stats.total, event.taskid, event.taskstring)
> +                continue
> +
> +            if isinstance(event, bb.runqueue.runQueueTaskFailed):
> +                taskfailures.append(event.taskstring)
> +                logger.error("Task %s (%s) failed with exit code '%s'",
> +                             event.taskid, event.taskstring, event.exitcode)
> +                continue
> +
> +            if isinstance(event, bb.runqueue.sceneQueueTaskFailed):
> +                logger.warn("Setscene task %s (%s) failed with exit code '%s' - real task will be run instead",
> +                             event.taskid, event.taskstring, event.exitcode)
> +                continue
> +
> +            if isinstance(event, bb.event.ConfigParsed):
> +                # timestamp should be added for this
> +                continue
> +
> +            if isinstance(event, bb.event.RecipeParsed):
> +                # timestamp should be added for this
> +                continue
> +
> +            if isinstance(event, bb.event.OperationStarted):
> +                # timestamp should be added for this
> +                continue
> +
> +            if isinstance(event, bb.event.OperationCompleted):
> +                # timestamp should be added for this
> +                # signal a complete operation
> +                # calculate timing
> +                continue
> +
> +            if isinstance(event, bb.event.DiskFull):
> +                # trigger an error
> +                continue
> +
> +            # ignore
> +            if isinstance(event, (bb.event.BuildBase,
> +                                  bb.event.StampUpdate,
> +                                  bb.event.RecipePreFinalise,
> +                                  bb.runqueue.runQueueEvent,
> +                                  bb.runqueue.runQueueExitWait,
> +                                  bb.event.OperationProgress)):
> +                continue
> +
> +            logger.error("Unknown event: %s", event)
> +
> +        except EnvironmentError as ioerror:
> +            termfilter.clearFooter()
> +            # ignore interrupted io
> +            if ioerror.args[0] == 4:
> +                pass
> +        except KeyboardInterrupt:
> +            termfilter.clearFooter()
> +            if main.shutdown == 1:
> +                print("\nSecond Keyboard Interrupt, stopping...\n")
> +                _, error = server.runCommand(["stateStop"])
> +                if error:
> +                    logger.error("Unable to cleanly stop: %s" % error)
> +            if main.shutdown == 0:
> +                print("\nKeyboard Interrupt, closing down...\n")
> +                interrupted = True
> +                _, error = server.runCommand(["stateShutdown"])
> +                if error:
> +                    logger.error("Unable to cleanly shutdown: %s" % error)
> +            main.shutdown = main.shutdown + 1
> +            pass
> +
> +    summary = ""
> +    if taskfailures:
> +        summary += pluralise("\nSummary: %s task failed:",
> +                             "\nSummary: %s tasks failed:", len(taskfailures))
> +        for failure in taskfailures:
> +            summary += "\n  %s" % failure
> +    if warnings:
> +        summary += pluralise("\nSummary: There was %s WARNING message shown.",
> +                             "\nSummary: There were %s WARNING messages shown.", warnings)
> +    if return_value:
> +        summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.",
> +                             "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors)
> +    if summary:
> +        print(summary)
> +
> +    if interrupted:
> +        print("Execution was interrupted, returning a non-zero exit code.")
> +        if return_value == 0:
> +            return_value = 1
> +
> +    termfilter.finish()
> +    wbhbhelper.write_in_database(wbhbhelper.tasks_information)
> +
> +    return return_value
> +
> --
> 1.8.1.2
>
> _______________________________________________
> webhob mailing list
> webhob@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/webhob