[PATCH] cooker: Separate out collections handling code into its own class

[Richard Purdie <richard.purdie@linuxfoundation.org>]

The Cooker class is too large and needs to be split up into different
functional units. Splitting out the collections code into its own class
seems like a good place to start to try and disentangle things.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 5c52d85..da598c9 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -89,7 +89,6 @@ class BBCooker:
 
     def __init__(self, configuration, server_registration_cb, savedenv={}):
         self.status = None
-        self.appendlist = {}
         self.skiplist = {}
 
         self.server_registration_cb = server_registration_cb
@@ -419,7 +418,7 @@ class BBCooker:
 
         if fn:
             try:
-                envdata = bb.cache.Cache.loadDataFull(fn, self.get_file_appends(fn), self.configuration.data)
+                envdata = bb.cache.Cache.loadDataFull(fn, self.collection.get_file_appends(fn), self.configuration.data)
             except Exception as e:
                 parselog.exception("Unable to read %s", fn)
                 raise
@@ -698,22 +697,13 @@ class BBCooker:
         print("}", file=tdepends_file)
         logger.info("Task dependencies saved to 'task-depends.dot'")
 
-    def calc_bbfile_priority( self, filename, matched = None ):
-        for _, _, regex, pri in self.status.bbfile_config_priorities:
-            if regex.match(filename):
-                if matched != None:
-                    if not regex in matched:
-                        matched.add(regex)
-                return pri
-        return 0
-
     def show_appends_with_no_recipes( self ):
         recipes = set(os.path.basename(f)
                       for f in self.status.pkg_fn.iterkeys())
         recipes |= set(os.path.basename(f)
                       for f in self.skiplist.iterkeys())
-        appended_recipes = self.appendlist.iterkeys()
-        appends_without_recipes = [self.appendlist[recipe]
+        appended_recipes = self.collection.appendlist.iterkeys()
+        appends_without_recipes = [self.collection.appendlist[recipe]
                                    for recipe in appended_recipes
                                    if recipe not in recipes]
         if appends_without_recipes:
@@ -747,32 +737,8 @@ class BBCooker:
                 providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.status.preferred[providee])
             self.status.preferred[providee] = provider
 
-        # Calculate priorities for each file
-        matched = set()
-        for p in self.status.pkg_fn:
-            realfn, cls = bb.cache.Cache.virtualfn2realfn(p)
-            self.status.bbfile_priority[p] = self.calc_bbfile_priority(realfn, matched)
  
-        # Don't show the warning if the BBFILE_PATTERN did match .bbappend files
-        unmatched = set()
-        for _, _, regex, pri in self.status.bbfile_config_priorities:        
-            if not regex in matched:
-                unmatched.add(regex)
-
-        def findmatch(regex):
-            for bbfile in self.appendlist:
-                for append in self.appendlist[bbfile]:
-                    if regex.match(append):
-                        return True
-            return False
-
-        for unmatch in unmatched.copy():
-            if findmatch(unmatch):
-                unmatched.remove(unmatch)
-
-        for collection, pattern, regex, _ in self.status.bbfile_config_priorities:
-            if regex in unmatched:
-                collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
+        self.status.bbfile_priority = self.collection.collection_priorities(self.status.pkg_fn)
 
     def findCoreBaseFiles(self, subdir, configfile):
         corebase = self.configuration.data.getVar('COREBASE', True) or ""
@@ -1111,7 +1077,9 @@ class BBCooker:
         """
         if bf.startswith("/") or bf.startswith("../"):
             bf = os.path.abspath(bf)
-        filelist, masked = self.collect_bbfiles()
+
+        self.collection = CookerCollectFiles(self.status.bbfile_config_priorities)
+        filelist, masked = self.collection.collect_bbfiles(self.configuration.data, self.configuration.event_data)
         try:
             os.stat(bf)
             bf = os.path.abspath(bf)
@@ -1165,7 +1133,7 @@ class BBCooker:
         self.buildSetVars()
 
         self.status = bb.cache.CacheData(self.caches_array)
-        infos = bb.cache.Cache.parse(fn, self.get_file_appends(fn), \
+        infos = bb.cache.Cache.parse(fn, self.collection.get_file_appends(fn), \
                                      self.configuration.data,
                                      self.caches_array)
         infos = dict(infos)
@@ -1332,7 +1300,9 @@ class BBCooker:
             for dep in self.configuration.extra_assume_provided:
                 self.status.ignored_dependencies.add(dep)
 
-            (filelist, masked) = self.collect_bbfiles()
+            self.collection = CookerCollectFiles(self.status.bbfile_config_priorities)
+            (filelist, masked) = self.collection.collect_bbfiles(self.configuration.data, self.configuration.event_data)
+
             self.configuration.data.renameVar("__depends", "__base_depends")
 
             self.parser = CookerParser(self, filelist, masked)
@@ -1369,7 +1339,87 @@ class BBCooker:
 
         return pkgs_to_build
 
-    def get_bbfiles( self, path = os.getcwd() ):
+
+
+
+    def pre_serve(self):
+        # Empty the environment. The environment will be populated as
+        # necessary from the data store.
+        #bb.utils.empty_environment()
+        try:
+            prserv.serv.auto_start(self.configuration.data)
+        except prserv.serv.PRServiceConfigError:
+            bb.event.fire(CookerExit(), self.configuration.event_data)
+        return
+
+    def post_serve(self):
+        prserv.serv.auto_shutdown(self.configuration.data)
+        bb.event.fire(CookerExit(), self.configuration.event_data)
+
+    def shutdown(self):
+        self.state = state.shutdown
+
+    def stop(self):
+        self.state = state.stop
+
+    def reparseFiles(self):
+        return
+
+    def initialize(self):
+        self.state = state.initial
+        self.initConfigurationData()
+
+    def reset(self):
+        self.state = state.initial
+        self.loadConfigurationData()
+
+def server_main(cooker, func, *args):
+    cooker.pre_serve()
+
+    if cooker.configuration.profile:
+        try:
+            import cProfile as profile
+        except:
+            import profile
+        prof = profile.Profile()
+
+        ret = profile.Profile.runcall(prof, func, *args)
+
+        prof.dump_stats("profile.log")
+        bb.utils.process_profilelog("profile.log")
+        print("Raw profiling information saved to profile.log and processed statistics to profile.log.processed")
+
+    else:
+        ret = func(*args)
+
+    cooker.post_serve()
+
+    return ret
+
+class CookerExit(bb.event.Event):
+    """
+    Notify clients of the Cooker shutdown
+    """
+
+    def __init__(self):
+        bb.event.Event.__init__(self)
+
+
+class CookerCollectFiles(object):
+    def __init__(self, priorities):
+        self.appendlist = {}
+        self.bbfile_config_priorities = priorities
+
+    def calc_bbfile_priority( self, filename, matched = None ):
+        for _, _, regex, pri in self.bbfile_config_priorities:
+            if regex.match(filename):
+                if matched != None:
+                    if not regex in matched:
+                        matched.add(regex)
+                return pri
+        return 0
+
+    def get_bbfiles(self, path = os.getcwd()):
         """Get list of default .bb files by reading out the current directory"""
         contents = os.listdir(path)
         bbfiles = []
@@ -1379,7 +1429,7 @@ class BBCooker:
                 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(), f)))
         return bbfiles
 
-    def find_bbfiles( self, path ):
+    def find_bbfiles(self, path):
         """Find all the .bb and .bbappend files in a directory"""
         from os.path import join
 
@@ -1392,14 +1442,14 @@ class BBCooker:
 
         return found
 
-    def collect_bbfiles( self ):
+    def collect_bbfiles(self, config, eventdata):
         """Collect all available .bb build files"""
         masked = 0
 
         collectlog.debug(1, "collecting .bb files")
 
-        files = (data.getVar( "BBFILES", self.configuration.data, True) or "").split()
-        data.setVar("BBFILES", " ".join(files), self.configuration.data)
+        files = (config.getVar( "BBFILES", True) or "").split()
+        config.setVar("BBFILES", " ".join(files))
 
         # Sort files by priority
         files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem) )
@@ -1409,7 +1459,7 @@ class BBCooker:
 
         if not len(files):
             collectlog.error("no recipe files to build, check your BBPATH and BBFILES?")
-            bb.event.fire(CookerExit(), self.configuration.event_data)
+            bb.event.fire(CookerExit(), eventdata)
 
         # Can't use set here as order is important
         newfiles = []
@@ -1427,7 +1477,7 @@ class BBCooker:
                     if g not in newfiles:
                         newfiles.append(g)
 
-        bbmask = self.configuration.data.getVar('BBMASK', True)
+        bbmask = config.getVar('BBMASK', True)
 
         if bbmask:
             try:
@@ -1475,74 +1525,44 @@ class BBCooker:
     def get_file_appends(self, fn):
         """
         Returns a list of .bbappend files to apply to fn
-        NB: collect_bbfiles() must have been called prior to this
         """
         f = os.path.basename(fn)
         if f in self.appendlist:
             return self.appendlist[f]
         return []
 
-    def pre_serve(self):
-        # Empty the environment. The environment will be populated as
-        # necessary from the data store.
-        #bb.utils.empty_environment()
-        try:
-            prserv.serv.auto_start(self.configuration.data)
-        except prserv.serv.PRServiceConfigError:
-            bb.event.fire(CookerExit(), self.configuration.event_data)
-        return
-
-    def post_serve(self):
-        prserv.serv.auto_shutdown(self.configuration.data)
-        bb.event.fire(CookerExit(), self.configuration.event_data)
-
-    def shutdown(self):
-        self.state = state.shutdown
+    def collection_priorities(self, pkgfns):
 
-    def stop(self):
-        self.state = state.stop
-
-    def reparseFiles(self):
-        return
-
-    def initialize(self):
-        self.state = state.initial
-        self.initConfigurationData()
-
-    def reset(self):
-        self.state = state.initial
-        self.loadConfigurationData()
+        priorities = {}
 
-def server_main(cooker, func, *args):
-    cooker.pre_serve()
-
-    if cooker.configuration.profile:
-        try:
-            import cProfile as profile
-        except:
-            import profile
-        prof = profile.Profile()
-
-        ret = profile.Profile.runcall(prof, func, *args)
-
-        prof.dump_stats("profile.log")
-        bb.utils.process_profilelog("profile.log")
-        print("Raw profiling information saved to profile.log and processed statistics to profile.log.processed")
-
-    else:
-        ret = func(*args)
+        # Calculate priorities for each file
+        matched = set()
+        for p in pkgfns:
+            realfn, cls = bb.cache.Cache.virtualfn2realfn(p)
+            priorities[p] = self.calc_bbfile_priority(realfn, matched)
+ 
+        # Don't show the warning if the BBFILE_PATTERN did match .bbappend files
+        unmatched = set()
+        for _, _, regex, pri in self.bbfile_config_priorities:        
+            if not regex in matched:
+                unmatched.add(regex)
 
-    cooker.post_serve()
+        def findmatch(regex):
+            for bbfile in self.appendlist:
+                for append in self.appendlist[bbfile]:
+                    if regex.match(append):
+                        return True
+            return False
 
-    return ret
+        for unmatch in unmatched.copy():
+            if findmatch(unmatch):
+                unmatched.remove(unmatch)
 
-class CookerExit(bb.event.Event):
-    """
-    Notify clients of the Cooker shutdown
-    """
+        for collection, pattern, regex, _ in self.bbfile_config_priorities:
+            if regex in unmatched:
+                collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
 
-    def __init__(self):
-        bb.event.Event.__init__(self)
+        return priorities
 
 def catch_parse_error(func):
     """Exception handling bits for our parsing"""
@@ -1677,7 +1697,7 @@ class CookerParser(object):
         self.fromcache = []
         self.willparse = []
         for filename in self.filelist:
-            appends = self.cooker.get_file_appends(filename)
+            appends = self.cooker.collection.get_file_appends(filename)
             if not self.bb_cache.cacheValid(filename, appends):
                 self.willparse.append((filename, appends, cooker.caches_array))
             else:
@@ -1840,7 +1860,7 @@ class CookerParser(object):
 
     def reparse(self, filename):
         infos = self.bb_cache.parse(filename,
-                                    self.cooker.get_file_appends(filename),
+                                    self.cooker.collection.get_file_appends(filename),
                                     self.cfgdata, self.cooker.caches_array)
         for vfn, info_array in infos:
             self.cooker.status.add_from_recipeinfo(vfn, info_array)