* [PATCH 0/1]
@ 2017-10-30 21:56 Amanda Brindle
2017-10-30 21:57 ` [PATCH 1/1] scripts/contrib/bbvars.py: Rewrite to use tinfoil Amanda Brindle
0 siblings, 1 reply; 6+ messages in thread
From: Amanda Brindle @ 2017-10-30 21:56 UTC (permalink / raw)
To: openembedded-core; +Cc: paul.eggleton
The following changes since commit 65d23bd7986615fdfb0f1717b615534a2a14ab80:
README.qemu: qemuppc64 is not supported (2017-10-16 23:54:31 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib abrindle/bbvars_tinfoil
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=abrindle/bbvars_tinfoil
Amanda Brindle (1):
scripts/contrib/bbvars.py: Rewrite to use tinfoil
scripts/contrib/bbvars.py | 90 ++++++++++++++++++++++++++++++++++-------------
1 file changed, 66 insertions(+), 24 deletions(-)
--
2.7.4
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH 1/1] scripts/contrib/bbvars.py: Rewrite to use tinfoil
2017-10-30 21:56 [PATCH 0/1] Amanda Brindle
@ 2017-10-30 21:57 ` Amanda Brindle
2017-11-13 1:27 ` Paul Eggleton
0 siblings, 1 reply; 6+ messages in thread
From: Amanda Brindle @ 2017-10-30 21:57 UTC (permalink / raw)
To: openembedded-core; +Cc: paul.eggleton
Use tinfoil to collect all variable names globally and in each recipe.
No longer show the count of variables if they are undocumented.
Fixes [YOCTO #2086]
Signed-off-by: Amanda Brindle <amanda.r.brindle@intel.com>
---
scripts/contrib/bbvars.py | 90 ++++++++++++++++++++++++++++++++++-------------
1 file changed, 66 insertions(+), 24 deletions(-)
diff --git a/scripts/contrib/bbvars.py b/scripts/contrib/bbvars.py
index d8d0594..556f652 100755
--- a/scripts/contrib/bbvars.py
+++ b/scripts/contrib/bbvars.py
@@ -23,6 +23,14 @@ import os
import os.path
import re
+# Set up sys.path to let us import tinfoil
+scripts_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+lib_path = scripts_path + '/lib'
+sys.path.insert(0, lib_path)
+import scriptpath
+scriptpath.add_bitbake_lib_path()
+import bb.tinfoil
+
def usage():
print('Usage: %s -d FILENAME [-d FILENAME]* -m METADIR [-m MATADIR]*' % os.path.basename(sys.argv[0]))
print(' -d FILENAME documentation file to search')
@@ -66,19 +74,23 @@ def collect_bbvars(metadir):
bbvars[key] = 1
return bbvars
-def bbvar_is_documented(var, docfiles):
- prog = re.compile(".*($|[^A-Z_])%s([^A-Z_]|$)" % (var))
- for doc in docfiles:
- try:
- f = open(doc)
- except IOError as err:
- print('WARNING: Failed to open doc ', doc)
- print(err.args[1])
- for line in f:
- if prog.match(line):
- return True
- f.close()
- return False
+def bbvar_is_documented(var, documented_vars):
+ ''' Check if variable (var) is in the list of documented variables(documented_vars) '''
+ if var in documented_vars:
+ return True
+ else:
+ return False
+
+def collect_documented_vars(docfiles):
+ ''' Walk the docfiles and collect the documented variables '''
+ documented_vars = []
+ prog = re.compile(".*($|[^A-Z_])<glossentry id=\'var-")
+ var_prog = re.compile('<glossentry id=\'var-(.*)\'>')
+ for d in docfiles:
+ with open(d) as f:
+ documented_vars += var_prog.findall(f.read())
+
+ return documented_vars
def bbvar_doctag(var, docconf):
prog = re.compile('^%s\[doc\] *= *"(.*)"' % (var))
@@ -101,7 +113,7 @@ def bbvar_doctag(var, docconf):
def main():
docfiles = []
metadirs = []
- bbvars = {}
+ bbvars = set()
undocumented = []
docconf = ""
onlydoctags = False
@@ -153,33 +165,63 @@ def main():
usage()
sys.exit(7)
- # Collect all the variable names from the recipes in the metadirs
- for m in metadirs:
- for key,cnt in collect_bbvars(m).items():
- if key in bbvars:
- bbvars[key] = bbvars[key] + cnt
+ prog = re.compile("^[^a-z]*$")
+ with bb.tinfoil.Tinfoil() as tinfoil:
+ tinfoil.prepare(config_only=False)
+ parser = bb.codeparser.PythonParser('parser', None)
+ datastore = tinfoil.config_data
+
+ def bbvars_update(data):
+ if prog.match(data):
+ bbvars.add(data)
+ if tinfoil.config_data.getVarFlag(data, 'python'):
+ try:
+ parser.parse_python(tinfoil.config_data.getVar(data))
+ except bb.data_smart.ExpansionError:
+ pass
+ for var in parser.references:
+ if prog.match(var):
+ bbvars.add(var)
else:
- bbvars[key] = cnt
+ try:
+ expandedVar = datastore.expandWithRefs(datastore.getVar(data, False), data)
+ for var in expandedVar.references:
+ if prog.match(var):
+ bbvars.add(var)
+ except bb.data_smart.ExpansionError:
+ pass
+
+ # Use tinfoil to collect all the variable names globally
+ for data in datastore:
+ bbvars_update(data)
+
+ # Collect variables from all recipes
+ for recipe in tinfoil.all_recipe_files():
+ print("Checking %s" % recipe)
+ for data in tinfoil.parse_recipe_file(recipe):
+ bbvars_update(data)
+
+ documented_vars = collect_documented_vars(docfiles)
# Check each var for documentation
varlen = 0
- for v in bbvars.keys():
+ for v in bbvars:
if len(v) > varlen:
varlen = len(v)
- if not bbvar_is_documented(v, docfiles):
+ if not bbvar_is_documented(v, documented_vars):
undocumented.append(v)
undocumented.sort()
varlen = varlen + 1
# Report all undocumented variables
print('Found %d undocumented bb variables (out of %d):' % (len(undocumented), len(bbvars)))
- header = '%s%s%s' % (str("VARIABLE").ljust(varlen), str("COUNT").ljust(6), str("DOCTAG").ljust(7))
+ header = '%s%s' % (str("VARIABLE").ljust(varlen), str("DOCTAG").ljust(7))
print(header)
print(str("").ljust(len(header), '='))
for v in undocumented:
doctag = bbvar_doctag(v, docconf)
if not onlydoctags or not doctag == "":
- print('%s%s%s' % (v.ljust(varlen), str(bbvars[v]).ljust(6), doctag))
+ print('%s%s' % (v.ljust(varlen), doctag))
if __name__ == "__main__":
--
2.7.4
^ permalink raw reply related [flat|nested] 6+ messages in thread* Re: [PATCH 1/1] scripts/contrib/bbvars.py: Rewrite to use tinfoil
2017-10-30 21:57 ` [PATCH 1/1] scripts/contrib/bbvars.py: Rewrite to use tinfoil Amanda Brindle
@ 2017-11-13 1:27 ` Paul Eggleton
0 siblings, 0 replies; 6+ messages in thread
From: Paul Eggleton @ 2017-11-13 1:27 UTC (permalink / raw)
To: Amanda Brindle; +Cc: openembedded-core
Hi Amanda,
Finally getting around to reviewing this, sorry for the delay.
On Tuesday, 31 October 2017 10:57:04 AM NZDT Amanda Brindle wrote:
> Use tinfoil to collect all variable names globally and in each recipe.
>
> No longer show the count of variables if they are undocumented.
Can you please describe why we're making these changes? i.e. we can capture
many more variable references using tinfoil than the manual parsing, and
we're not showing the counts anymore because we can't distinguish unique
references using this mechanism.
I also notice there's some dead code left after these changes that we
should remove at the same time - recipe_bbvars() and collect_bbvars()
at least. We also no longer need the -m option since we're searching through
all recipes in the configuration (the latter should be noted in the commit
message as well).
> Fixes [YOCTO #2086]
>
> Signed-off-by: Amanda Brindle <amanda.r.brindle@intel.com>
> ---
> scripts/contrib/bbvars.py | 90 ++++++++++++++++++++++++++++++++++-------------
> 1 file changed, 66 insertions(+), 24 deletions(-)
>
> diff --git a/scripts/contrib/bbvars.py b/scripts/contrib/bbvars.py
> index d8d0594..556f652 100755
> --- a/scripts/contrib/bbvars.py
> +++ b/scripts/contrib/bbvars.py
> @@ -23,6 +23,14 @@ import os
> import os.path
> import re
>
> +# Set up sys.path to let us import tinfoil
> +scripts_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
> +lib_path = scripts_path + '/lib'
> +sys.path.insert(0, lib_path)
> +import scriptpath
> +scriptpath.add_bitbake_lib_path()
> +import bb.tinfoil
> +
> def usage():
> print('Usage: %s -d FILENAME [-d FILENAME]* -m METADIR [-m MATADIR]*' % os.path.basename(sys.argv[0]))
> print(' -d FILENAME documentation file to search')
> @@ -66,19 +74,23 @@ def collect_bbvars(metadir):
> bbvars[key] = 1
> return bbvars
>
> -def bbvar_is_documented(var, docfiles):
> - prog = re.compile(".*($|[^A-Z_])%s([^A-Z_]|$)" % (var))
> - for doc in docfiles:
> - try:
> - f = open(doc)
> - except IOError as err:
> - print('WARNING: Failed to open doc ', doc)
> - print(err.args[1])
> - for line in f:
> - if prog.match(line):
> - return True
> - f.close()
> - return False
> +def bbvar_is_documented(var, documented_vars):
> + ''' Check if variable (var) is in the list of documented variables(documented_vars) '''
> + if var in documented_vars:
> + return True
> + else:
> + return False
> +
> +def collect_documented_vars(docfiles):
> + ''' Walk the docfiles and collect the documented variables '''
> + documented_vars = []
> + prog = re.compile(".*($|[^A-Z_])<glossentry id=\'var-")
> + var_prog = re.compile('<glossentry id=\'var-(.*)\'>')
> + for d in docfiles:
> + with open(d) as f:
> + documented_vars += var_prog.findall(f.read())
> +
> + return documented_vars
>
> def bbvar_doctag(var, docconf):
> prog = re.compile('^%s\[doc\] *= *"(.*)"' % (var))
> @@ -101,7 +113,7 @@ def bbvar_doctag(var, docconf):
> def main():
> docfiles = []
> metadirs = []
> - bbvars = {}
> + bbvars = set()
> undocumented = []
> docconf = ""
> onlydoctags = False
> @@ -153,33 +165,63 @@ def main():
> usage()
> sys.exit(7)
>
> - # Collect all the variable names from the recipes in the metadirs
> - for m in metadirs:
> - for key,cnt in collect_bbvars(m).items():
> - if key in bbvars:
> - bbvars[key] = bbvars[key] + cnt
> + prog = re.compile("^[^a-z]*$")
> + with bb.tinfoil.Tinfoil() as tinfoil:
> + tinfoil.prepare(config_only=False)
> + parser = bb.codeparser.PythonParser('parser', None)
> + datastore = tinfoil.config_data
> +
> + def bbvars_update(data):
> + if prog.match(data):
> + bbvars.add(data)
> + if tinfoil.config_data.getVarFlag(data, 'python'):
> + try:
> + parser.parse_python(tinfoil.config_data.getVar(data))
> + except bb.data_smart.ExpansionError:
> + pass
> + for var in parser.references:
> + if prog.match(var):
> + bbvars.add(var)
> else:
> - bbvars[key] = cnt
> + try:
> + expandedVar = datastore.expandWithRefs(datastore.getVar(data, False), data)
> + for var in expandedVar.references:
> + if prog.match(var):
> + bbvars.add(var)
> + except bb.data_smart.ExpansionError:
> + pass
> +
> + # Use tinfoil to collect all the variable names globally
> + for data in datastore:
> + bbvars_update(data)
> +
> + # Collect variables from all recipes
> + for recipe in tinfoil.all_recipe_files():
I think we need to pass variants=False here, there's not much percentage in
scanning through all of the variants as well.
One other thing - looking at the output there are a lot of variables that
simply shouldn't be documented - particularly those that are set and
used entirely within one recipe or class and aren't intended to be used
outside of that context (a bit like local variables). I think the script would
be more useful if we could filter these out by default, but I don't think
there's any effective way we can detect those programmatically, so we'd
have to have an exclusion list. We can do that in a separate patch though.
Cheers,
Paul
--
Paul Eggleton
Intel Open Source Technology Centre
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH 0/1]
@ 2017-11-21 0:21 Amanda Brindle
0 siblings, 0 replies; 6+ messages in thread
From: Amanda Brindle @ 2017-11-21 0:21 UTC (permalink / raw)
To: openembedded-core; +Cc: paul.eggleton
The following changes since commit 4ed19ac8c19afd56d445d84e02b622cb056b8359:
poky: Switch to post release name/version (2017-11-14 17:26:58 +0000)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib abrindle/bbvars_tinfoil2
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=abrindle/bbvars_tinfoil2
Amanda Brindle (1):
scripts/contrib/bbvars.py: Remove dead code
scripts/contrib/bbvars.py | 52 ++---------------------------------------------
1 file changed, 2 insertions(+), 50 deletions(-)
--
2.7.4
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH 0/1]
@ 2013-06-18 15:20 Bruce Ashfield
2013-06-18 15:22 ` Bruce Ashfield
0 siblings, 1 reply; 6+ messages in thread
From: Bruce Ashfield @ 2013-06-18 15:20 UTC (permalink / raw)
To: richard.purdie; +Cc: openembedded-core
Richard/Saul,
Here's the integration of a change from Khem to fix the gcc 4.8 ARM
boot issues.
I'm still seeing some quasi random issues, but this definitely fixes
the issue at hand, gets us up and running and is much better than
what we had before.
The patch pretty much says everything else:
linux-yocto/3.8: fix gcc 4.8 ARM boot issues
Updating the linux-yocto-3.8 SRCREVs to fix a boot issue with ARM boards
when gcc 4.8 is used.
Without the following mainline backports:
f200475 ARM: 7670/1: fix the memset fix
8215b0e ARM: 7668/1: fix memset-related crashes caused by recent GCC (4.7.2) optimizations
The following trap will be seen on boot:
[<c00fc3b8>] (kmem_cache_alloc_trace+0x54/0x210) from [<c039f074>] (con_insert_unipair+0xcc/0x11c)
[<c039f074>] (con_insert_unipair+0xcc/0x11c) from [<c039fec8>] (con_set_default_unimap+0xfc/0x198)
[<c039fec8>] (con_set_default_unimap+0xfc/0x198) from [<c07ee258>] (console_map_init+0x44/0x58)
[<c07ee258>] (console_map_init+0x44/0x58) from [<c07ee738>] (vty_init+0x16c/0x1b0)
[<c07ee738>] (vty_init+0x16c/0x1b0) from [<c07edb68>] (tty_init+0x108/0x148)
[<c07edb68>] (tty_init+0x108/0x148) from [<c07eead0>] (chr_dev_init+0xb4/0xd8)
[<c07eead0>] (chr_dev_init+0xb4/0xd8) from [<c0008a18>] (do_one_initcall+0x11c/0x18c)
[<c0008a18>] (do_one_initcall+0x11c/0x18c) from [<c07d89d0>] (kernel_init_freeable+0x16c/0x254)
[<c07d89d0>] (kernel_init_freeable+0x16c/0x254) from [<c05a3810>] (kernel_init+0x18/0x160)
[<c05a3810>] (kernel_init+0x18/0x160) from [<c000e530>] (ret_from_fork+0x14/0x20)
Code: e593a000 e35a0000 0a000020 e5943014 (e79a1003)
---[ end trace e6c62de166779f86 ]---
Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b
Moderate stress and board testing shows the fix to hold, and it is good for
broader testing.
[YOCTO #4549]
Signed-off-by: Khem Raj <raj.khem@gmail.com>
Signed-off-by: Bruce Ashfield <bruce.ashfield@windriver.com>
Cheers,
Bruce
The following changes since commit 1dd643b142c69ac9035e29bff11d02201638dc65:
licences: Add SGI license (2013-06-17 16:45:37 +0100)
are available in the git repository at:
git://git.pokylinux.org/poky-contrib zedd/kernel
http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=zedd/kernel
Bruce Ashfield (1):
linux-yocto/3.8: fix gcc 4.8 ARM boot issues
meta/recipes-kernel/linux/linux-yocto-rt_3.8.bb | 6 +++---
meta/recipes-kernel/linux/linux-yocto-tiny_3.8.bb | 4 ++--
meta/recipes-kernel/linux/linux-yocto_3.8.bb | 14 +++++++-------
3 files changed, 12 insertions(+), 12 deletions(-)
--
1.7.10.4
^ permalink raw reply [flat|nested] 6+ messages in thread* Re: [PATCH 0/1]
2013-06-18 15:20 Bruce Ashfield
@ 2013-06-18 15:22 ` Bruce Ashfield
0 siblings, 0 replies; 6+ messages in thread
From: Bruce Ashfield @ 2013-06-18 15:22 UTC (permalink / raw)
To: Richard Purdie; +Cc: Patches and discussions about the oe-core layer
Apologies for the "no subject" email.
The rest is fine, just the guy typing it up had an itchy trigger finger!
Cheers,
Bruce
On Tue, Jun 18, 2013 at 11:20 AM, Bruce Ashfield
<bruce.ashfield@windriver.com> wrote:
> Richard/Saul,
>
> Here's the integration of a change from Khem to fix the gcc 4.8 ARM
> boot issues.
>
> I'm still seeing some quasi random issues, but this definitely fixes
> the issue at hand, gets us up and running and is much better than
> what we had before.
>
> The patch pretty much says everything else:
>
> linux-yocto/3.8: fix gcc 4.8 ARM boot issues
>
> Updating the linux-yocto-3.8 SRCREVs to fix a boot issue with ARM boards
> when gcc 4.8 is used.
>
> Without the following mainline backports:
>
> f200475 ARM: 7670/1: fix the memset fix
> 8215b0e ARM: 7668/1: fix memset-related crashes caused by recent GCC (4.7.2) optimizations
>
> The following trap will be seen on boot:
>
> [<c00fc3b8>] (kmem_cache_alloc_trace+0x54/0x210) from [<c039f074>] (con_insert_unipair+0xcc/0x11c)
> [<c039f074>] (con_insert_unipair+0xcc/0x11c) from [<c039fec8>] (con_set_default_unimap+0xfc/0x198)
> [<c039fec8>] (con_set_default_unimap+0xfc/0x198) from [<c07ee258>] (console_map_init+0x44/0x58)
> [<c07ee258>] (console_map_init+0x44/0x58) from [<c07ee738>] (vty_init+0x16c/0x1b0)
> [<c07ee738>] (vty_init+0x16c/0x1b0) from [<c07edb68>] (tty_init+0x108/0x148)
> [<c07edb68>] (tty_init+0x108/0x148) from [<c07eead0>] (chr_dev_init+0xb4/0xd8)
> [<c07eead0>] (chr_dev_init+0xb4/0xd8) from [<c0008a18>] (do_one_initcall+0x11c/0x18c)
> [<c0008a18>] (do_one_initcall+0x11c/0x18c) from [<c07d89d0>] (kernel_init_freeable+0x16c/0x254)
> [<c07d89d0>] (kernel_init_freeable+0x16c/0x254) from [<c05a3810>] (kernel_init+0x18/0x160)
> [<c05a3810>] (kernel_init+0x18/0x160) from [<c000e530>] (ret_from_fork+0x14/0x20)
> Code: e593a000 e35a0000 0a000020 e5943014 (e79a1003)
> ---[ end trace e6c62de166779f86 ]---
> Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b
>
> Moderate stress and board testing shows the fix to hold, and it is good for
> broader testing.
>
> [YOCTO #4549]
>
> Signed-off-by: Khem Raj <raj.khem@gmail.com>
> Signed-off-by: Bruce Ashfield <bruce.ashfield@windriver.com>
>
> Cheers,
>
> Bruce
>
> The following changes since commit 1dd643b142c69ac9035e29bff11d02201638dc65:
>
> licences: Add SGI license (2013-06-17 16:45:37 +0100)
>
> are available in the git repository at:
>
> git://git.pokylinux.org/poky-contrib zedd/kernel
> http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=zedd/kernel
>
> Bruce Ashfield (1):
> linux-yocto/3.8: fix gcc 4.8 ARM boot issues
>
> meta/recipes-kernel/linux/linux-yocto-rt_3.8.bb | 6 +++---
> meta/recipes-kernel/linux/linux-yocto-tiny_3.8.bb | 4 ++--
> meta/recipes-kernel/linux/linux-yocto_3.8.bb | 14 +++++++-------
> 3 files changed, 12 insertions(+), 12 deletions(-)
>
> --
> 1.7.10.4
>
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/openembedded-core
--
"Thou shalt not follow the NULL pointer, for chaos and madness await
thee at its end"
^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2017-11-21 0:21 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2017-10-30 21:56 [PATCH 0/1] Amanda Brindle
2017-10-30 21:57 ` [PATCH 1/1] scripts/contrib/bbvars.py: Rewrite to use tinfoil Amanda Brindle
2017-11-13 1:27 ` Paul Eggleton
-- strict thread matches above, loose matches on Subject: below --
2017-11-21 0:21 [PATCH 0/1] Amanda Brindle
2013-06-18 15:20 Bruce Ashfield
2013-06-18 15:22 ` Bruce Ashfield
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox