[PATCH] build/data: Write out more complete python run files

[PATCH] build/data: Write out more complete python run files

[Richard Purdie]

Currently the output in the python task/function run files is rather
incomplete and effectively useless. This enhances the code to take
advantage of the bitbake's dependency tracking and extend the output to
include dependencies. This makes the files more usable for debugging
purposes. Since this only happens at python function execution time, the
overhead is minimal in the grand scheme of things.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index f2922f3..65cc851 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -227,7 +227,7 @@ def exec_func_python(func, d, runfile, cwd=None):
     code = _functionfmt.format(function=func, body=d.getVar(func, True))
     bb.utils.mkdirhier(os.path.dirname(runfile))
     with open(runfile, 'w') as script:
-        script.write(code)
+        bb.data.emit_func_python(func, script, d)
 
     if cwd:
         try:
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py
index 3d776b3..91b1eb1 100644
--- a/bitbake/lib/bb/data.py
+++ b/bitbake/lib/bb/data.py
@@ -281,6 +281,41 @@ def emit_func(func, o=sys.__stdout__, d = init()):
                newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
         newdeps -= seen
 
+_functionfmt = """
+def {function}(d):
+{body}"""
+
+def emit_func_python(func, o=sys.__stdout__, d = init()):
+    """Emits all items in the data store in a format such that it can be sourced by a shell."""
+
+    def write_func(func, o, call = False):
+        body = d.getVar(func, True)
+        if not body.startswith("def"):
+            body = _functionfmt.format(function=func, body=body)
+
+        o.write(body.strip() + "\n\n")
+        if call:
+            o.write(func + "(d)" + "\n\n")
+
+    write_func(func, o, True)
+    pp = bb.codeparser.PythonParser(func, logger)
+    pp.parse_python(d.getVar(func, True))
+    newdeps = pp.execs
+    newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
+    seen = set()
+    while newdeps:
+        deps = newdeps
+        seen |= deps
+        newdeps = set()
+        for dep in deps:
+            if d.getVarFlag(dep, "func") and d.getVarFlag(dep, "python"):
+               write_func(dep, o)
+               pp = bb.codeparser.PythonParser(dep, logger)
+               pp.parse_python(d.getVar(dep, True))
+               newdeps |= pp.execs
+               newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
+        newdeps -= seen
+
 def update_data(d):
     """Performs final steps upon the datastore, including application of overrides"""
     d.finalize(parent = True)

Re: [PATCH] build/data: Write out more complete python run files

[Olof Johansson]

On 14-08-27 15:24 +0200, Richard Purdie wrote:
> Currently the output in the python task/function run files is rather
> incomplete and effectively useless. This enhances the code to take
> advantage of the bitbake's dependency tracking and extend the output to
> include dependencies. This makes the files more usable for debugging
> purposes. Since this only happens at python function execution time, the
> overhead is minimal in the grand scheme of things.
> 
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> 
....
> diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py
> index 3d776b3..91b1eb1 100644
> --- a/bitbake/lib/bb/data.py
> +++ b/bitbake/lib/bb/data.py
....
> +def emit_func_python(func, o=sys.__stdout__, d = init()):
> +    """Emits all items in the data store in a format such that it can be sourced by a shell."""
> +
> +    def write_func(func, o, call = False):
> +        body = d.getVar(func, True)
> +        if not body.startswith("def"):
> +            body = _functionfmt.format(function=func, body=body)
....

With this change, we are seeing problems in certain cases when a python
function (def foo():) is being resolved. d.getVar("foo") would return None
and then fail from not having a startswith() method:

ERROR: Build of do_unpack failed
ERROR: Traceback (most recent call last):
  File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 497, in exec_task
    return _exec_task(fn, task, d, quieterr)
  File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 438, in _exec_task
    exec_func(task, localdata)
  File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 212, in exec_func
    exec_func_python(func, d, runfile, cwd=adir)
  File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 230, in exec_func_python
    bb.data.emit_func_python(func, script, d)
  File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/data.py", line 313, in emit_func_python
    write_func(dep, o)
  File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/data.py", line 294, in write_func
    if not body.startswith("def"):
AttributeError: 'NoneType' object has no attribute 'startswith'


Our qemu_%.bbappend contains the following:

    def _remove_git_submodule_file(filename):
        import os
    
        if not os.path.isfile(filename):
            return
    
        os.unlink(filename)
    
    do_unpack_append() {
        _remove_git_submodule_file(d.getVar("S", True) + "/dtc/.git")
        _remove_git_submodule_file(d.getVar("S", True) + "/pixman/.git")
    }

Moving the function to within do_unpack_append would solve the problem.

-- 
olofjn

Re: [PATCH] build/data: Write out more complete python run files

[Richard Purdie]

On Tue, 2014-09-02 at 10:20 +0200, Olof Johansson wrote:
> On 14-08-27 15:24 +0200, Richard Purdie wrote:
> > Currently the output in the python task/function run files is rather
> > incomplete and effectively useless. This enhances the code to take
> > advantage of the bitbake's dependency tracking and extend the output to
> > include dependencies. This makes the files more usable for debugging
> > purposes. Since this only happens at python function execution time, the
> > overhead is minimal in the grand scheme of things.
> > 
> > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> > 
> ....
> > diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py
> > index 3d776b3..91b1eb1 100644
> > --- a/bitbake/lib/bb/data.py
> > +++ b/bitbake/lib/bb/data.py
> ....
> > +def emit_func_python(func, o=sys.__stdout__, d = init()):
> > +    """Emits all items in the data store in a format such that it can be sourced by a shell."""
> > +
> > +    def write_func(func, o, call = False):
> > +        body = d.getVar(func, True)
> > +        if not body.startswith("def"):
> > +            body = _functionfmt.format(function=func, body=body)
> ....
> 
> With this change, we are seeing problems in certain cases when a python
> function (def foo():) is being resolved. d.getVar("foo") would return None
> and then fail from not having a startswith() method:
> 
> ERROR: Build of do_unpack failed
> ERROR: Traceback (most recent call last):
>   File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 497, in exec_task
>     return _exec_task(fn, task, d, quieterr)
>   File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 438, in _exec_task
>     exec_func(task, localdata)
>   File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 212, in exec_func
>     exec_func_python(func, d, runfile, cwd=adir)
>   File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 230, in exec_func_python
>     bb.data.emit_func_python(func, script, d)
>   File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/data.py", line 313, in emit_func_python
>     write_func(dep, o)
>   File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/data.py", line 294, in write_func
>     if not body.startswith("def"):
> AttributeError: 'NoneType' object has no attribute 'startswith'
> 
> 
> Our qemu_%.bbappend contains the following:
> 
>     def _remove_git_submodule_file(filename):
>         import os
>     
>         if not os.path.isfile(filename):
>             return
>     
>         os.unlink(filename)
>     
>     do_unpack_append() {
>         _remove_git_submodule_file(d.getVar("S", True) + "/dtc/.git")
>         _remove_git_submodule_file(d.getVar("S", True) + "/pixman/.git")
>     }
> 
> Moving the function to within do_unpack_append would solve the problem.

We should probably put a safeguard in that code, if the function isn't
in the datastore, not to try and write it out, just pass over it. Its
either that, or pull it from the methodpool, which thinking about it,
may not be a bad idea since it is in there somewhere...

Cheers,

Richard