[PATCH v2 001/169] moveconfig: Update to detect / correct missing SPL Kconfigs

[Simon Glass <sjg@chromium.org>]

This adds quite a few more features, all designed to detect problems with
Kconfig options and their use. It can find options mentioned in the source
code which are not in the Kconfig and vice versa. It can convert SPL
usages of non-SPL Kconfig options (i.e. changing CONFIG_IS_ENABLED() to
IS_ENABLED() and CONFIG_$(SPL)_FOO to CONFIG_FOO, creating commits
automatically if requested.

These features are only useful for code clean-up, not for normal
development.

Signed-off-by: Simon Glass <sjg@chromium.org>
---

Changes in v2:
- Rebase to previous series

 tools/moveconfig.py | 599 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 456 insertions(+), 143 deletions(-)

diff --git a/tools/moveconfig.py b/tools/moveconfig.py
index c4d72ede368..b3ac2672737 100755
--- a/tools/moveconfig.py
+++ b/tools/moveconfig.py
@@ -20,6 +20,7 @@ import doctest
 import filecmp
 import fnmatch
 import glob
+import io
 import multiprocessing
 import os
 import queue
@@ -118,6 +119,16 @@ def check_clean_directory():
         if os.path.exists(fname):
             sys.exit("source tree is not clean, please run 'make mrproper'")
 
+def write_commit(msg):
+    """Create a new commit with all modified files
+
+    Args:
+        msg (str): Commit message
+    """
+    subprocess.call(['git', 'add', '-u'])
+    rc = subprocess.call(['git', 'commit', '-s', '-m', msg])
+    return rc == 0
+
 def get_make_cmd():
     """Get the command name of GNU Make.
 
@@ -1606,30 +1617,59 @@ def prefix_config(cfg):
     return op + cfg
 
 
-RE_MK_CONFIGS = re.compile('CONFIG_(\$\(SPL_(?:TPL_)?\))?([A-Za-z0-9_]*)')
-RE_IFDEF = re.compile('(ifdef|ifndef)')
-RE_C_CONFIGS = re.compile('CONFIG_([A-Za-z0-9_]*)')
-RE_CONFIG_IS = re.compile('CONFIG_IS_ENABLED\(([A-Za-z0-9_]*)\)')
+RE_MK_CONFIGS = re.compile(r'CONFIG_(\$\(SPL_(?:TPL_)?\))?([A-Za-z0-9_]*)')
+
+# Makefile ifdefs: this only handles 'else' on its own line, so not
+# 'else ifeq ...', for example
+RE_IF = re.compile(r'^(ifdef|ifndef|endif|ifeq|ifneq|else)([^,]*,([^,]*)\))?.*$')
+
+# Normal CONFIG options in C
+RE_C_CONFIGS = re.compile(r'CONFIG_([A-Za-z0-9_]*)')
+
+# CONFIG_IS_ENABLED() construct
+RE_CONFIG_IS = re.compile(r'CONFIG_IS_ENABLED\(([A-Za-z0-9_]*)\)')
+
+# Preprocessor #if/#ifdef directives, etc.
+RE_IFDEF = re.compile(r'^\s*#\s*(ifdef|ifndef|endif|if|elif|else)\s*(?:#.*)?(.*)$')
 
 class ConfigUse:
-    def __init__(self, cfg, is_spl, fname, rest):
+    """Holds information about a use of a CONFIG option"""
+    def __init__(self, cfg, is_spl, conds, fname, rest, is_mk):
+        """Set up a new use of a CONFIG option
+
+        Args:
+            cfg (str): Config option, without the CONFIG_
+            is_spl (bool): True if it indicates an SPL option, i.e. has a
+                $(SPL_) or similar, False if not
+            conds (list of str): List of conditions for this use, e.g.
+                ['SPL_BUILD']
+            fname (str): Filename contining the use
+            rest (str): Entire line from the file
+            is_mk (bool): True if this is in a Makefile, False if not
+        """
         self.cfg = cfg
         self.is_spl = is_spl
+        self.conds = list(c for c in conds if '(NONE)' not in c)
         self.fname = fname
         self.rest = rest
+        self.is_mk = is_mk
 
     def __hash__(self):
         return hash((self.cfg, self.is_spl))
 
-def scan_makefiles(fnames):
+    def __str__(self):
+        return (f'ConfigUse({self.cfg}, is_spl={self.is_spl}, '
+                f'is_mk={self.is_mk}, fname={self.fname}, rest={self.rest}')
+
+def scan_makefiles(fname_dict):
     """Scan Makefiles looking for Kconfig options
 
     Looks for uses of CONFIG options in Makefiles
 
     Args:
-        fnames (list of tuple):
-            str: Makefile filename where the option was found
-            str: Line of the Makefile
+        fname_dict (dict lines):
+            key: Makefile filename where the option was found
+            value: List of str lines of the Makefile
 
     Returns:
         tuple:
@@ -1646,46 +1686,81 @@ def scan_makefiles(fnames):
     ('$(SPL_)', 'MARY')
     >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_TPL_)MARY').groups()
     ('$(SPL_TPL_)', 'MARY')
+    >>> RE_IF.match('ifdef CONFIG_SPL_BUILD').groups()
+    ('ifdef', None, None)
+    >>> RE_IF.match('endif # CONFIG_SPL_BUILD').groups()
+    ('endif', None, None)
+    >>> RE_IF.match('endif').groups()
+    ('endif', None, None)
+    >>> RE_IF.match('else').groups()
+    ('else', None, None)
     """
     all_uses = collections.defaultdict(list)
     fname_uses = {}
-    for fname, rest in fnames:
-        m_iter = RE_MK_CONFIGS.finditer(rest)
-        found = False
-        for m in m_iter:
-            found = True
-            real_opt = m.group(2)
-            if real_opt == '':
-                continue
-            is_spl = False
-            if m.group(1):
-                is_spl = True
-            use = ConfigUse(real_opt, is_spl, fname, rest)
-            if fname not in fname_uses:
-                fname_uses[fname] = set()
-            fname_uses[fname].add(use)
-            all_uses[use].append(rest)
+    for fname, rest_list in fname_dict.items():
+        conds = []
+        for rest in rest_list:
+            m_iter = RE_MK_CONFIGS.finditer(rest)
+            m_cond = RE_IF.match(rest)
+            #print('line', conds, m_cond and m_cond.group(1) or '.', rest)
+            last_use = None
+            use = None
+            for m in m_iter:
+                real_opt = m.group(2)
+                if real_opt == '':
+                    continue
+                is_spl = False
+                if m.group(1):
+                    is_spl = True
+                use = ConfigUse(real_opt, is_spl, conds, fname, rest, True)
+                if fname not in fname_uses:
+                    fname_uses[fname] = set()
+                fname_uses[fname].add(use)
+                all_uses[use].append(rest)
+                last_use = use
+            if m_cond:
+                cfg = last_use.cfg if last_use else '(NONE)'
+                cond = m_cond.group(1)
+                if cond == 'ifdef':
+                    conds.append(cfg)
+                elif cond == 'ifndef':
+                    conds.append(f'~{cfg}')
+                elif cond == 'ifeq':
+                    #print('todo', fname, m_cond.group(3), fname, rest)
+                    conds.append(f'{m_cond.group(3)}={cfg}')
+                elif cond == 'ifneq':
+                    conds.append(f'{m_cond.group(3)}={cfg}')
+                    #print('todo', m_cond.group(3))
+                elif cond == 'endif':
+                    conds.pop()
+                elif cond == 'else':
+                    cond = conds.pop()
+                    if cond.startswith('~'):
+                        cond = cond[1:]
+                    else:
+                        cond = f'~{cond}'
+                    conds.append(cond)
+                else:
+                    print(f'unknown condition: {rest}')
+        if conds:
+            print(f'leftover {conds}')
     return all_uses, fname_uses
 
-
-def scan_src_files(fnames):
+def scan_src_files(fname_dict, all_uses, fname_uses):
     """Scan source files (other than Makefiles) looking for Kconfig options
 
     Looks for uses of CONFIG options
 
     Args:
-        fnames (list of tuple):
-            str: Makefile filename where the option was found
-            str: Line of the Makefile
-
-    Returns:
-        tuple:
-            dict: all_uses
-                key (ConfigUse): object
-                value (list of str): matching lines
-            dict: Uses by filename
-                key (str): filename
-                value (set of ConfigUse): uses in that filename
+        fname_dict (dict):
+            key (str): Filename
+            value (list of ConfigUse): List of uses in that filename
+        all_uses (dict): to add more uses to:
+            key (ConfigUse): object
+            value (list of str): matching lines
+        fname (dict): to add more uses-by-filename to
+            key (str): filename
+            value (set of ConfigUse): uses in that filename
 
     >>> RE_C_CONFIGS.search('CONFIG_FRED').groups()
     ('FRED',)
@@ -1693,39 +1768,78 @@ def scan_src_files(fnames):
     ('MARY',)
     >>> RE_CONFIG_IS.search('#if CONFIG_IS_ENABLED(OF_PLATDATA)').groups()
     ('OF_PLATDATA',)
+    >>> RE_IFDEF.match('#ifdef CONFIG_SPL_BUILD').groups()
+    ('ifdef', 'CONFIG_SPL_BUILD')
+    >>> RE_IFDEF.match('#endif # CONFIG_SPL_BUILD').groups()
+    ('endif', '')
+    >>> RE_IFDEF.match('#endif').groups()
+    ('endif', '')
+    >>> RE_IFDEF.match('#else').groups()
+    ('else', '')
+    >>> RE_IFDEF.match(' # if defined(CONFIG_CMD_NET) && !defined(CONFIG_DM_ETH)').groups()
+    ('if', 'defined(CONFIG_CMD_NET) && !defined(CONFIG_DM_ETH)')
     """
     def add_uses(m_iter, is_spl):
+        last_use = None
         for m in m_iter:
             found = True
             real_opt = m.group(1)
             if real_opt == '':
                 continue
-            use = ConfigUse(real_opt, is_spl, fname, rest)
+            use = ConfigUse(real_opt, is_spl, conds, fname, rest, False)
             if fname not in fname_uses:
                 fname_uses[fname] = set()
             fname_uses[fname].add(use)
             all_uses[use].append(rest)
-
-    all_uses = collections.defaultdict(list)
-    fname_uses = {}
-    for fname, rest in fnames:
-        m_iter = RE_C_CONFIGS.finditer(rest)
-        add_uses(m_iter, False)
-
-        m_iter2 = RE_CONFIG_IS.finditer(rest)
-        add_uses(m_iter2, True)
-
-    return all_uses, fname_uses
+            last_use = use
+        return last_use
+
+    for fname, rest_list in fname_dict.items():
+        conds = []
+        for rest in rest_list:
+            m_iter = RE_C_CONFIGS.finditer(rest)
+            m_cond = RE_IFDEF.match(rest)
+            last_use1 = add_uses(m_iter, False)
+
+            m_iter2 = RE_CONFIG_IS.finditer(rest)
+            last_use2 = add_uses(m_iter2, True)
+            last_use = last_use1 or last_use2
+            if m_cond:
+                cfg = last_use.cfg if last_use else '(NONE)'
+                cond = m_cond.group(1)
+                #print(fname, rest, cond, conds)
+                if cond == 'ifdef':
+                    conds.append(cfg)
+                elif cond == 'ifndef':
+                    conds.append(f'~{cfg}')
+                elif cond == 'if':
+                    #print('todo', fname, m_cond.group(3), fname, rest)
+                    conds.append(f'{m_cond.group(2)}={cfg}')
+                elif cond == 'endif':
+                    conds.pop()
+                elif cond == 'else':
+                    cond = conds.pop()
+                    if cond.startswith('~'):
+                        cond = cond[1:]
+                    else:
+                        cond = f'~{cond}'
+                    conds.append(cond)
+                elif cond == 'elif':
+                    cond = conds.pop()
+                    conds.append(cfg)
+                else:
+                    print(f'{fname}: unknown condition: {rest}')
 
 
 MODE_NORMAL, MODE_SPL, MODE_PROPER = range(3)
 
-def do_scan_source(path, do_update):
+def do_scan_source(path, do_update, show_conflicts, do_commit):
     """Scan the source tree for Kconfig inconsistencies
 
     Args:
         path (str): Path to source tree
-        do_update (bool) : True to write to scripts/kconf_... files
+        do_update (bool): True to write to scripts/kconf_... files
+        show_conflicts (bool): True to show conflicts
     """
     def is_not_proper(name):
         for prefix in SPL_PREFIXES:
@@ -1754,7 +1868,7 @@ def do_scan_source(path, do_update):
         """
         # Make sure we know about all the options
         not_found = collections.defaultdict(list)
-        for use, rest in all_uses.items():
+        for use, _ in all_uses.items():
             name = use.cfg
             if name in IGNORE_SYMS:
                 continue
@@ -1766,7 +1880,6 @@ def do_scan_source(path, do_update):
                 # If it is an SPL symbol, try prepending all SPL_ prefixes to
                 # find at least one SPL symbol
                 if use.is_spl:
-                    add_to_dict = False
                     for prefix in SPL_PREFIXES:
                         try_name = prefix + name
                         sym = kconf.syms.get(try_name)
@@ -1784,7 +1897,6 @@ def do_scan_source(path, do_update):
                 elif not use.is_spl:
                     check = False
             else: # MODE_NORMAL
-                debug = False
                 sym = kconf.syms.get(name)
                 if not sym:
                     proper_name = is_not_proper(name)
@@ -1819,105 +1931,304 @@ def do_scan_source(path, do_update):
             for i, use in enumerate(uses[name]):
                 print(f'{"   " if i else ""}{use.fname}: {use.rest.strip()}')
 
+    def do_scan():
+        """Scan the source tree
 
-    print('Scanning Kconfig')
-    kconf = KconfigScanner().conf
-    print(f'Scanning source in {path}')
-    args = ['git', 'grep', '-E', r'IS_ENABLED|\bCONFIG']
-    with subprocess.Popen(args, stdout=subprocess.PIPE) as proc:
-        out, err = proc.communicate()
-    lines = out.splitlines()
-    re_fname = re.compile('^([^:]*):(.*)')
-    src_list = []
-    mk_list = []
-    for line in lines:
-        linestr = line.decode('utf-8')
-        m_fname = re_fname.search(linestr)
-        if not m_fname:
-            continue
-        fname, rest = m_fname.groups()
-        dirname, leaf = os.path.split(fname)
-        root, ext = os.path.splitext(leaf)
-        if ext == '.autoconf':
-            pass
-        elif ext in ['.c', '.h', '.S', '.lds', '.dts', '.dtsi', '.asl', '.cfg',
-                     '.env', '.tmpl']:
-            src_list.append([fname, rest])
-        elif 'Makefile' in root or ext == '.mk':
-            mk_list.append([fname, rest])
-        elif ext in ['.yml', '.sh', '.py', '.awk', '.pl', '.rst', '', '.sed']:
-            pass
-        elif 'Kconfig' in root or 'Kbuild' in root:
-            pass
-        elif 'README' in root:
-            pass
-        elif dirname in ['configs']:
-            pass
-        elif dirname.startswith('doc') or dirname.startswith('scripts/kconfig'):
-            pass
+        This runs a 'git grep' and then picks out uses of CONFIG options from
+        the resulting output
+
+        Returns: tuple
+            dict of uses of CONFIG in Makefiles:
+                key (str): Filename
+                value (list of ConfigUse): List of uses in that filename
+            dict of uses of CONFIG in source files:
+                key (str): Filename
+                value (list of ConfigUse): List of uses in that filename
+        """
+        def finish_file(last_fname, rest_list):
+            if is_mkfile:
+                mk_dict[last_fname] = rest_list
+            else:
+                src_dict[last_fname] = rest_list
+            rest_list = []
+
+        print(f'Scanning source in {path}')
+        args = ['git', 'grep', '-E',
+                r'IS_ENABLED|\bCONFIG|CONFIG_SPL_BUILD|if|else|endif']
+        with subprocess.Popen(args, stdout=subprocess.PIPE) as proc:
+            out, _ = proc.communicate()
+        lines = out.splitlines()
+        re_fname = re.compile('^([^:]*):(.*)')
+        src_dict = collections.OrderedDict()
+        mk_dict = collections.OrderedDict()
+        last_fname = None
+        rest_list = []
+        is_mkfile = None
+        for line in lines:
+            linestr = line.decode('utf-8')
+            m_fname = re_fname.search(linestr)
+            if not m_fname:
+                continue
+            fname, rest = m_fname.groups()
+            if fname != last_fname:
+                finish_file(last_fname, rest_list)
+                rest_list = []
+                last_fname = fname
+
+            dirname, leaf = os.path.split(fname)
+            root, ext = os.path.splitext(leaf)
+            #if dirname != '' or root != 'Makefile':
+                #continue
+            if (dirname.startswith('test') or
+                dirname.startswith('lib/efi_selftest')):
+                # Ignore test code since it is mostly only for sandbox
+                pass
+            elif ext == '.autoconf':
+                pass
+            elif ext in ['.c', '.h', '.S', '.lds', '.dts', '.dtsi', '.asl',
+                         '.cfg', '.env', '.tmpl']:
+                rest_list.append(rest)
+                is_mkfile = False
+            elif 'Makefile' in root or ext == '.mk':
+                rest_list.append(rest)
+                is_mkfile = True
+            elif ext in ['.yml', '.sh', '.py', '.awk', '.pl', '.rst', '', '.sed',
+                         '.src', '.inc', '.l', '.i_shipped', '.txt', '.cmd',
+                         '.cfg', '.y', '.cocci', '.ini', '.asn1', '.base',
+                         '.cnf', '.patch', '.mak', '.its', '.svg', '.tcl',
+                         '.css', '.config', '.conf']:
+                pass
+            elif 'Kconfig' in root or 'Kbuild' in root:
+                pass
+            elif 'README' in root:
+                pass
+            elif dirname in ['configs']:
+                pass
+            elif dirname.startswith('doc') or dirname.startswith('scripts/kconfig'):
+                pass
+            else:
+                print(f'Not sure how to handle file {fname}')
+        finish_file(last_fname, rest_list)
+        return mk_dict, src_dict
+
+    def check_mk_missing(all_uses, spl_not_found, proper_not_found):
+        # Make sure we know about all the options in Makefiles
+        print('\nCONFIG options present in Makefiles but not Kconfig:')
+        not_found = check_not_found(all_uses, MODE_NORMAL)
+        show_uses(not_found)
+
+        print('\nCONFIG options present in Makefiles but not Kconfig (SPL):')
+        not_found = check_not_found(all_uses, MODE_SPL)
+        show_uses(not_found)
+        spl_not_found |= set(is_not_proper(key) or key for key in not_found.keys())
+
+        print('\nCONFIG options used as Proper in Makefiles but without a non-SPL_ variant:')
+        not_found = check_not_found(all_uses, MODE_PROPER)
+        show_uses(not_found)
+        proper_not_found |= set(key for key in not_found.keys())
+
+    def check_src_missing(all_uses, spl_not_found, proper_not_found):
+        # Make sure we know about all the options in source files
+        print('\nCONFIG options present in source but not Kconfig:')
+        not_found = check_not_found(all_uses, MODE_NORMAL)
+        show_uses(not_found)
+
+        print('\nCONFIG options present in source but not Kconfig (SPL):')
+        not_found = check_not_found(all_uses, MODE_SPL)
+        show_uses(not_found)
+        spl_not_found |= set(is_not_proper(key) or key
+                             for key in not_found.keys())
+
+        print('\nCONFIG options used as Proper in source but without a non-SPL_ variant:')
+        not_found = check_not_found(all_uses, MODE_PROPER)
+        show_uses(not_found)
+        proper_not_found |= set(key for key in not_found.keys())
+
+    def show_summary(spl_not_found, proper_not_found):
+        print('\nCONFIG options used as SPL but without an SPL_ variant:')
+        for item in sorted(spl_not_found):
+            print(f'   {item}')
+
+        print('\nCONFIG options used as Proper but without a non-SPL_ variant:')
+        for item in sorted(proper_not_found):
+            print(f'   {item}')
+
+    def write_update(spl_not_found, proper_not_found):
+        with open(os.path.join(path, 'scripts', 'conf_nospl'),
+                  encoding='utf-8') as out:
+            print('# These options should not be enabled in SPL builds\n',
+                  file=out)
+            for item in sorted(spl_not_found):
+                print(item, file=out)
+        with open(os.path.join(path, 'scripts', 'conf_noproper'), 'w',
+                  encoding='utf-8') as out:
+            print('# These options should not be enabled in Proper builds\n',
+                  file=out)
+            for item in sorted(proper_not_found):
+                print(item, file=out)
+
+    def check_conflict(kconf, all_uses, show):
+        """Check conflicts between uses of CONFIG options in source
+
+        Sometimes an option is used in an SPL context in once place and not in
+        another. For example if we see CONFIG_SPL_FOO and CONFIG_FOO then we
+        don't know whether FOO should be enabled in SPL if CONFIG_FOO is
+        enabled, or only if CONFIG_SPL_FOO is enabled.,
+
+        This function detects such situations
+
+        Args:
+            kconf (Kconfiglib.Kconfig): Kconfig tree read from source
+            all_uses (dict): dict to add more uses to:
+                key (ConfigUse): object
+                value (list of str): matching lines
+            show (bool): True to show the problems
+
+        Returns:
+            dict of conflicts:
+                key: filename
+                value: list of ConfigUse objects
+        """
+        conflict_dict = collections.defaultdict(list)
+        cfg_dict = collections.defaultdict(list)
+
+        uses = collections.defaultdict(list)
+        for use in all_uses:
+            uses[use.cfg].append(use)
+
+        bad = 0
+        for cfg in sorted(uses):
+            use_list = uses[cfg]
+
+            # Check if
+            #  - there is no SPL_ version of the symbol, and
+            #  - some uses are SPL and some are not
+            is_spl = list(set(u.is_spl for u in use_list))
+            if len(is_spl) > 1 and f'SPL_{cfg}' not in kconf.syms:
+                if show:
+                    print(f'{cfg}: {is_spl}')
+                for use in use_list:
+                    if show:
+                        print(f'   spl={use.is_spl}: {use.conds} {use.fname} {use.rest}')
+                    if use.is_spl:
+                        conflict_dict[use.fname].append(use)
+                        cfg_dict[use.cfg].append(use)
+                bad += 1
+        print(f'total bad: {bad}')
+        return conflict_dict, cfg_dict
+
+    def replace_in_file(fname, use_or_uses):
+        """Replace a CONFIG pattern in a file
+
+        Args:
+            fname (str): Filename to replace in
+            use_or_uses (ConfigUse or list of ConfigUse): Uses to replace
+        """
+        with open(fname, encoding='utf-8') as inf:
+            data = inf.read()
+        out = io.StringIO()
+
+        if isinstance(use_or_uses, list):
+            uses = use_or_uses
         else:
-            print(f'Not sure how to handle file {fname}')
+            uses = [use_or_uses]
 
-    # Scan the Makefiles
-    all_uses, fname_uses = scan_makefiles(mk_list)
+        re_cfgs = []
+        for use in uses:
+            if use.is_mk:
+                expr = r'CONFIG_(?:\$\(SPL_(?:TPL_)?\))%s\b' % use.cfg
+            else:
+                expr = r'CONFIG_IS_ENABLED\(%s\)' % use.cfg
+            re_cfgs.append(re.compile(expr))
+
+        for line in data.splitlines():
+            new_line = line
+            if 'CONFIG_' in line:
+                todo = []
+                for i, re_cfg in enumerate(re_cfgs):
+                    if not re_cfg.search(line):
+                        todo.append(re_cfg)
+                    use = uses[i]
+                    if use.is_mk:
+                        new_line = re_cfg.sub(f'CONFIG_{use.cfg}', new_line)
+                    else:
+                        new = f'IS_ENABLED(CONFIG_{use.cfg})'
+                        new_line = re_cfg.sub(new, new_line)
+                re_cfgs = todo
+            out.write(new_line)
+            out.write('\n')
+        out_data = out.getvalue()
+        if out_data == data:
+            print(f'\n\nfailed with {fname}\n\n')
+        with open(fname, 'w', encoding='utf-8') as outf:
+            outf.write(out_data)
+
+    def update_source(conflicts):
+        """Replace any SPL constructs with plain non-SPL ones
+
+        CONFIG_IS_ENABLED(FOO) becomes IS_ENABLED(CONFIG_FOO)
+        CONFIG$_(SPL_TPL_)FOO becomes CONFIG_FOO
 
-    spl_not_found = set()
-    proper_not_found = set()
+        Args:
+            conflicts (dict): dict of conflicts:
+                key (str): filename
+                value (list of ConfigUse): uses within that file
+        """
+        print(f'Updating {len(conflicts)} files')
+        for fname, uses in conflicts.items():
+            replace_in_file(fname, uses)
 
-    # Make sure we know about all the options
-    print('\nCONFIG options present in Makefiles but not Kconfig:')
-    not_found = check_not_found(all_uses, MODE_NORMAL)
-    show_uses(not_found)
+    def create_commits(cfg_dict):
+        """Create a set of commits to fix SPL conflicts
 
-    print('\nCONFIG options present in Makefiles but not Kconfig (SPL):')
-    not_found = check_not_found(all_uses, MODE_SPL)
-    show_uses(not_found)
-    spl_not_found |= set([is_not_proper(key) or key for key in not_found.keys()])
+        """
+        print(f'Creating {len(cfg_dict)} commits')
+        for cfg, uses in cfg_dict.items():
+            print(f'Processing {cfg}')
+            for use in uses:
+                #print(f'   {use.fname}: {use}')
+                replace_in_file(use.fname, use)
+            plural = 's' if len(uses) > 1 else ''
+            msg = f'Correct SPL use{plural} of {cfg}'
+            msg += f'\n\nThis converts {len(uses)} usage{plural} '
+            msg += 'of this option to the non-SPL form, since there is\n'
+            msg += f'no SPL_{cfg} defined in Kconfig'
+            if not write_commit(msg):
+                print('failed\n', uses)
+                break
 
-    print('\nCONFIG options used as Proper in Makefiles but without a non-SPL_ variant:')
-    not_found = check_not_found(all_uses, MODE_PROPER)
-    show_uses(not_found)
-    proper_not_found |= set([key for key in not_found.keys()])
+    print('Scanning Kconfig')
+    kconf = KconfigScanner().conf
 
-    # Scan the source code
-    all_uses, fname_uses = scan_src_files(src_list)
+    all_uses = collections.defaultdict(list)
+    fname_uses = {}
+    mk_dict, src_dict = do_scan()
+
+    # Scan the Makefiles
+    all_uses, fname_uses = scan_makefiles(mk_dict)
+    scan_src_files(src_dict, all_uses, fname_uses)
 
-    # Make sure we know about all the options
-    print('\nCONFIG options present in source but not Kconfig:')
-    not_found = check_not_found(all_uses, MODE_NORMAL)
-    show_uses(not_found)
+    conflicts, cfg_dict = check_conflict(kconf, all_uses, show_conflicts)
 
-    print('\nCONFIG options present in source but not Kconfig (SPL):')
-    not_found = check_not_found(all_uses, MODE_SPL)
-    show_uses(not_found)
-    spl_not_found |= set([is_not_proper(key) or key for key in not_found.keys()])
+    if do_update:
+        update_source(conflicts)
+    if do_commit:
+        create_commits(cfg_dict)
 
-    print('\nCONFIG options used as Proper in source but without a non-SPL_ variant:')
-    not_found = check_not_found(all_uses, MODE_PROPER)
-    show_uses(not_found)
-    proper_not_found |= set([key for key in not_found.keys()])
+    # Look for CONFIG options that are not found
+    spl_not_found = set()
+    proper_not_found = set()
+    check_mk_missing(all_uses, spl_not_found, proper_not_found)
 
-    print('\nCONFIG options used as SPL but without an SPL_ variant:')
-    for item in sorted(spl_not_found):
-        print(f'   {item}')
+    # Scan the source code
+    scan_src_files(src_dict, all_uses, fname_uses)
+    check_src_missing(all_uses, spl_not_found, proper_not_found)
 
-    print('\nCONFIG options used as Proper but without a non-SPL_ variant:')
-    for item in sorted(proper_not_found):
-        print(f'   {item}')
+    show_summary(spl_not_found, proper_not_found)
 
     # Write out the updated information
     if do_update:
-        with open(os.path.join(path, 'scripts', 'conf_nospl'), 'w') as out:
-            print('# These options should not be enabled in SPL builds\n',
-                  file=out)
-            for item in sorted(spl_not_found):
-                print(item, file=out)
-        with open(os.path.join(path, 'scripts', 'conf_noproper'), 'w') as out:
-            print('# These options should not be enabled in Proper builds\n',
-                  file=out)
-            for item in sorted(proper_not_found):
-                print(item, file=out)
-
+        write_update(spl_not_found, proper_not_found)
 
 def main():
     try:
@@ -1957,9 +2268,11 @@ doc/develop/moveconfig.rst for documentation.'''
     parser.add_argument('-i', '--imply', action='store_true', default=False,
                       help='find options which imply others')
     parser.add_argument('-I', '--imply-flags', type=str, default='',
-                      help="control the -i option ('help' for help")
+                      help="control the -i option ('help' for help)")
     parser.add_argument('-j', '--jobs', type=int, default=cpu_count,
                       help='the number of jobs to run simultaneously')
+    parser.add_argument('-l', '--list-problems', action='store_true',
+                        help='list problems found with --scan-source')
     parser.add_argument('-n', '--dry-run', action='store_true', default=False,
                       help='perform a trial run (show log with no changes)')
     parser.add_argument('-r', '--git-ref', type=str,
@@ -1991,7 +2304,8 @@ doc/develop/moveconfig.rst for documentation.'''
         unittest.main()
 
     if args.scan_source:
-        do_scan_source(os.getcwd(), args.update)
+        do_scan_source(os.getcwd(), args.update, args.list_problems,
+                       args.commit)
         return
 
     if not any((len(configs), args.force_sync, args.build_db, args.imply,
@@ -2049,7 +2363,6 @@ doc/develop/moveconfig.rst for documentation.'''
         cleanup_readme(configs, args)
 
     if args.commit:
-        subprocess.call(['git', 'add', '-u'])
         if configs:
             msg = 'Convert %s %sto Kconfig' % (configs[0],
                     'et al ' if len(configs) > 1 else '')
@@ -2058,7 +2371,7 @@ doc/develop/moveconfig.rst for documentation.'''
         else:
             msg = 'configs: Resync with savedefconfig'
             msg += '\n\nRsync all defconfig files using moveconfig.py'
-        subprocess.call(['git', 'commit', '-s', '-m', msg])
+        write_commit(msg)
 
     if args.build_db:
         with open(CONFIG_DATABASE, 'w', encoding='utf-8') as fd:
-- 
2.39.1.519.gcb327c4b5f-goog