* [PATCH v2 1/1] scripts/contrib/bbvars.py: Rewrite to use tinfoil
2017-11-16 0:20 [PATCH v2 0/1] Amanda Brindle
@ 2017-11-16 0:21 ` Amanda Brindle
2017-11-20 20:57 ` Paul Eggleton
2017-11-17 22:53 ` ✗ patchtest: failure for [v2] " Patchwork
2017-11-18 0:11 ` Patchwork
2 siblings, 1 reply; 5+ messages in thread
From: Amanda Brindle @ 2017-11-16 0:21 UTC (permalink / raw)
To: openembedded-core; +Cc: paul.eggleton
Use tinfoil to collect all variable names globally and in each recipe.
This will capture more variable references than parsing manually.
With the use of tinfoil, this script can no longer distinguish unique
references to each variable, so it's no longer showing the count of
undocumented variables.
Removed the -m option since this script now searches through all recipes
in the configuration.
Fixes [YOCTO #2086]
Signed-off-by: Amanda Brindle <amanda.r.brindle@intel.com>
---
scripts/contrib/bbvars.py | 140 ++++++++++++++++++++++------------------------
1 file changed, 67 insertions(+), 73 deletions(-)
diff --git a/scripts/contrib/bbvars.py b/scripts/contrib/bbvars.py
index d8d0594..286b5a9 100755
--- a/scripts/contrib/bbvars.py
+++ b/scripts/contrib/bbvars.py
@@ -23,62 +23,38 @@ import os
import os.path
import re
+# Set up sys.path to let us import tinfoil
+scripts_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+lib_path = scripts_path + '/lib'
+sys.path.insert(0, lib_path)
+import scriptpath
+scriptpath.add_bitbake_lib_path()
+import bb.tinfoil
+
def usage():
- print('Usage: %s -d FILENAME [-d FILENAME]* -m METADIR [-m MATADIR]*' % os.path.basename(sys.argv[0]))
+ print('Usage: %s -d FILENAME [-d FILENAME]*' % os.path.basename(sys.argv[0]))
print(' -d FILENAME documentation file to search')
print(' -h, --help display this help and exit')
- print(' -m METADIR meta directory to search for recipes')
print(' -t FILENAME documentation config file (for doc tags)')
print(' -T Only display variables with doc tags (requires -t)')
-def recipe_bbvars(recipe):
- ''' Return a unique set of every bbvar encountered in the recipe '''
- prog = re.compile("[A-Z_]+")
- vset = set()
- try:
- r = open(recipe)
- except IOError as err:
- print('WARNING: Failed to open recipe ', recipe)
- print(err.args[1])
-
- for line in r:
- # Strip any comments from the line
- line = line.rsplit('#')[0]
- vset = vset.union(set(prog.findall(line)))
- r.close()
-
- bbvars = {}
- for v in vset:
- bbvars[v] = 1
-
- return bbvars
-
-def collect_bbvars(metadir):
- ''' Walk the metadir and collect the bbvars from each recipe found '''
- bbvars = {}
- for root,dirs,files in os.walk(metadir):
- for name in files:
- if name.find(".bb") >= 0:
- for key in recipe_bbvars(os.path.join(root,name)).keys():
- if key in bbvars:
- bbvars[key] = bbvars[key] + 1
- else:
- bbvars[key] = 1
- return bbvars
-
-def bbvar_is_documented(var, docfiles):
- prog = re.compile(".*($|[^A-Z_])%s([^A-Z_]|$)" % (var))
- for doc in docfiles:
- try:
- f = open(doc)
- except IOError as err:
- print('WARNING: Failed to open doc ', doc)
- print(err.args[1])
- for line in f:
- if prog.match(line):
- return True
- f.close()
- return False
+def bbvar_is_documented(var, documented_vars):
+ ''' Check if variable (var) is in the list of documented variables(documented_vars) '''
+ if var in documented_vars:
+ return True
+ else:
+ return False
+
+def collect_documented_vars(docfiles):
+ ''' Walk the docfiles and collect the documented variables '''
+ documented_vars = []
+ prog = re.compile(".*($|[^A-Z_])<glossentry id=\'var-")
+ var_prog = re.compile('<glossentry id=\'var-(.*)\'>')
+ for d in docfiles:
+ with open(d) as f:
+ documented_vars += var_prog.findall(f.read())
+
+ return documented_vars
def bbvar_doctag(var, docconf):
prog = re.compile('^%s\[doc\] *= *"(.*)"' % (var))
@@ -100,8 +76,7 @@ def bbvar_doctag(var, docconf):
def main():
docfiles = []
- metadirs = []
- bbvars = {}
+ bbvars = set()
undocumented = []
docconf = ""
onlydoctags = False
@@ -124,12 +99,6 @@ def main():
else:
print('ERROR: documentation file %s is not a regular file' % a)
sys.exit(3)
- elif o == '-m':
- if os.path.isdir(a):
- metadirs.append(a)
- else:
- print('ERROR: meta directory %s is not a directory' % a)
- sys.exit(4)
elif o == "-t":
if os.path.isfile(a):
docconf = a
@@ -143,43 +112,68 @@ def main():
usage()
sys.exit(5)
- if len(metadirs) == 0:
- print('ERROR: no metadir specified')
- usage()
- sys.exit(6)
-
if onlydoctags and docconf == "":
print('ERROR: no docconf specified')
usage()
sys.exit(7)
- # Collect all the variable names from the recipes in the metadirs
- for m in metadirs:
- for key,cnt in collect_bbvars(m).items():
- if key in bbvars:
- bbvars[key] = bbvars[key] + cnt
+ prog = re.compile("^[^a-z]*$")
+ with bb.tinfoil.Tinfoil() as tinfoil:
+ tinfoil.prepare(config_only=False)
+ parser = bb.codeparser.PythonParser('parser', None)
+ datastore = tinfoil.config_data
+
+ def bbvars_update(data):
+ if prog.match(data):
+ bbvars.add(data)
+ if tinfoil.config_data.getVarFlag(data, 'python'):
+ try:
+ parser.parse_python(tinfoil.config_data.getVar(data))
+ except bb.data_smart.ExpansionError:
+ pass
+ for var in parser.references:
+ if prog.match(var):
+ bbvars.add(var)
else:
- bbvars[key] = cnt
+ try:
+ expandedVar = datastore.expandWithRefs(datastore.getVar(data, False), data)
+ for var in expandedVar.references:
+ if prog.match(var):
+ bbvars.add(var)
+ except bb.data_smart.ExpansionError:
+ pass
+
+ # Use tinfoil to collect all the variable names globally
+ for data in datastore:
+ bbvars_update(data)
+
+ # Collect variables from all recipes
+ for recipe in tinfoil.all_recipe_files(variants=False):
+ print("Checking %s" % recipe)
+ for data in tinfoil.parse_recipe_file(recipe):
+ bbvars_update(data)
+
+ documented_vars = collect_documented_vars(docfiles)
# Check each var for documentation
varlen = 0
- for v in bbvars.keys():
+ for v in bbvars:
if len(v) > varlen:
varlen = len(v)
- if not bbvar_is_documented(v, docfiles):
+ if not bbvar_is_documented(v, documented_vars):
undocumented.append(v)
undocumented.sort()
varlen = varlen + 1
# Report all undocumented variables
print('Found %d undocumented bb variables (out of %d):' % (len(undocumented), len(bbvars)))
- header = '%s%s%s' % (str("VARIABLE").ljust(varlen), str("COUNT").ljust(6), str("DOCTAG").ljust(7))
+ header = '%s%s' % (str("VARIABLE").ljust(varlen), str("DOCTAG").ljust(7))
print(header)
print(str("").ljust(len(header), '='))
for v in undocumented:
doctag = bbvar_doctag(v, docconf)
if not onlydoctags or not doctag == "":
- print('%s%s%s' % (v.ljust(varlen), str(bbvars[v]).ljust(6), doctag))
+ print('%s%s' % (v.ljust(varlen), doctag))
if __name__ == "__main__":
--
2.7.4
^ permalink raw reply related [flat|nested] 5+ messages in thread