[PATCH 0/3] More logging improvements

[PATCH 0/3] More logging improvements

[Joshua Watt]

A few more logging improvements for bitbake.

Joshua Watt (3):
  bitbake: msg: Return config object
  bitbake: knotty: Treat verbconsole as a console output
  bitbake: Use logging.shutdown() instead of bb.msg.cleanupLogging()

 bitbake/lib/bb/msg.py       | 14 +++----------
 bitbake/lib/bb/ui/knotty.py | 42 +++++++++++++++++--------------------
 2 files changed, 22 insertions(+), 34 deletions(-)

-- 
2.17.1

[PATCH 1/3] bitbake: msg: Return config object

[Joshua Watt]

Returns the configuration object from setLoggingConfig(). This object
has a config dictionary that contains all of the created handlers,
filters and loggers, which makes it much easier to pull out items with
specific names.

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
 bitbake/lib/bb/msg.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bitbake/lib/bb/msg.py b/bitbake/lib/bb/msg.py
index c22d0d3e15..3f95e21cc1 100644
--- a/bitbake/lib/bb/msg.py
+++ b/bitbake/lib/bb/msg.py
@@ -270,7 +270,8 @@ def setLoggingConfig(defaultconfig, userconfigfile=None):
         if "level" in l:
             l["level"] = bb.msg.stringToLevel(l["level"])
 
-    logging.config.dictConfig(logconfig)
+    conf = logging.config.dictConfigClass(logconfig)
+    conf.configure()
 
     # The user may have specified logging domains they want at a higher debug
     # level than the standard.
@@ -292,6 +293,8 @@ def setLoggingConfig(defaultconfig, userconfigfile=None):
         #if newlevel < bb.msg.loggerDefaultLogLevel:
         #    bb.msg.loggerDefaultLogLevel = newlevel
 
+    return conf
+
 def cleanupLogging():
     # Iterate through all the handlers and close them if possible. Fixes
     # 'Unclosed resource' warnings when bitbake exits, see
-- 
2.17.1

[PATCH 2/3] bitbake: knotty: Treat verbconsole as a console output

[Joshua Watt]

The BitBake.verbconsole needs to be treated like a console output logger
(meaning that the TerminalFilter attaches an InteractConsoleLogFilter to
it), even if it's not directly attached to the root 'BitBake' logger.
First, assign a special "is_console" property to the relevant handlers,
then look for the property in the handlers from the configuration object
return by bb.msg.setLoggingConfig(). Finally, pass the list of all
handlers to the TerminalFilter object; it doesn't care about the
difference between console and errconsole, so pass all the relevant
handlers as a list.

This fixes cases where the console output was corrupted when messages
were sent to the 'BitBake.verbconsole' handler.

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
 bitbake/lib/bb/ui/knotty.py | 40 +++++++++++++++++--------------------
 1 file changed, 18 insertions(+), 22 deletions(-)

diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index b4df5f6e88..db4511844a 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -149,7 +149,7 @@ class TerminalFilter(object):
                 cr = (25, 80)
         return cr
 
-    def __init__(self, main, helper, console, errconsole, quiet):
+    def __init__(self, main, helper, handlers, quiet):
         self.main = main
         self.helper = helper
         self.cuu = None
@@ -179,14 +179,9 @@ class TerminalFilter(object):
             termios.tcsetattr(fd, termios.TCSADRAIN, new)
             curses.setupterm()
             if curses.tigetnum("colors") > 2:
-                if console:
+                for h in handlers:
                     try:
-                        console.formatter.enable_color()
-                    except AttributeError:
-                        pass
-                if errconsole:
-                    try:
-                        errconsole.formatter.enable_color()
+                        h.formatter.enable_color()
                     except AttributeError:
                         pass
             self.ed = curses.tigetstr("ed")
@@ -204,10 +199,9 @@ class TerminalFilter(object):
             self.interactive = False
             bb.note("Unable to use interactive mode for this terminal, using fallback")
             return
-        if console:
-            console.addFilter(InteractConsoleLogFilter(self))
-        if errconsole:
-            errconsole.addFilter(InteractConsoleLogFilter(self))
+
+        for h in handlers:
+            h.addFilter(InteractConsoleLogFilter(self))
 
         self.main_progress = None
 
@@ -411,6 +405,9 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 "level": console_loglevel,
                 "stream": "ext://sys.stdout",
                 "filters": ["BitBake.stdoutFilter"],
+                ".": {
+                    "is_console": True,
+                },
             },
             "BitBake.errconsole": {
                 "class": "logging.StreamHandler",
@@ -418,6 +415,9 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 "level": loglevel,
                 "stream": "ext://sys.stderr",
                 "filters": ["BitBake.stderrFilter"],
+                ".": {
+                    "is_console": True,
+                },
             },
             # This handler can be used if specific loggers should print on
             # the console at a lower severity than the default. It will
@@ -430,6 +430,9 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 "level": 1,
                 "stream": "ext://sys.stdout",
                 "filters": ["BitBake.verbconsoleFilter"],
+                ".": {
+                    "is_console": True,
+                },
             },
         },
         "formatters": {
@@ -523,7 +526,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         except OSError:
            pass
 
-    bb.msg.setLoggingConfig(logconfig, logconfigfile)
+    conf = bb.msg.setLoggingConfig(logconfig, logconfigfile)
 
     if sys.stdin.isatty() and sys.stdout.isatty():
         log_exec_tty = True
@@ -534,14 +537,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
     # Look for the specially designated handlers which need to be passed to the
     # terminal handler
-    console = None
-    errconsole = None
-    for h in logger.handlers:
-        name = getattr(h, '_name', None)
-        if name == 'BitBake.console':
-            console = h
-        elif name == 'BitBake.errconsole':
-            errconsole = h
+    console_handlers = [h for h in conf.config['handlers'].values() if getattr(h, 'is_console', False)]
 
     bb.utils.set_process_name("KnottyUI")
 
@@ -591,7 +587,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     printinterval = 5000
     lastprint = time.time()
 
-    termfilter = tf(main, helper, console, errconsole, params.options.quiet)
+    termfilter = tf(main, helper, console_handlers, params.options.quiet)
     atexit.register(termfilter.finish)
 
     while True:
-- 
2.17.1

[PATCH 3/3] bitbake: Use logging.shutdown() instead of bb.msg.cleanupLogging()

[Joshua Watt]

The logging module provides a shutdown() function that does the same
thing in a much better way

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
 bitbake/lib/bb/msg.py       | 11 -----------
 bitbake/lib/bb/ui/knotty.py |  2 +-
 2 files changed, 1 insertion(+), 12 deletions(-)

diff --git a/bitbake/lib/bb/msg.py b/bitbake/lib/bb/msg.py
index 3f95e21cc1..29f0a3999e 100644
--- a/bitbake/lib/bb/msg.py
+++ b/bitbake/lib/bb/msg.py
@@ -294,14 +294,3 @@ def setLoggingConfig(defaultconfig, userconfigfile=None):
         #    bb.msg.loggerDefaultLogLevel = newlevel
 
     return conf
-
-def cleanupLogging():
-    # Iterate through all the handlers and close them if possible. Fixes
-    # 'Unclosed resource' warnings when bitbake exits, see
-    # https://bugs.python.org/issue23010
-    handlers = set()
-    for logger_iter in logging.Logger.manager.loggerDict.keys():
-        handlers.update(logging.getLogger(logger_iter).handlers)
-
-    for h in handlers:
-        h.close()
diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index db4511844a..33ee891256 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -871,6 +871,6 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         if e.errno == errno.EPIPE:
             pass
 
-    bb.msg.cleanupLogging()
+    logging.shutdown()
 
     return return_value
-- 
2.17.1