[PATCH v4] binman: add support for creating dummy files for external blobs

[Heiko Thiery <heiko.thiery@gmail.com>]

While converting to binman for an imx8mq board, it has been found that
building in the u-boot CI fails. This is because an imx8mq requires an
external binary (signed_hdmi_imx8m.bin). If this file cannot be found
mkimage fails.
To be able to build this board in the u-boot CI a binman option
(--fake-ext-blobs) is introduced that can be switched on via the u-boot
makefile option BINMAN_FAKE_EXT_BLOBS. With that the needed dummy files are
created.

Signed-off-by: Heiko Thiery <heiko.thiery@gmail.com>
---
v4:
 - removed RFC from subject
 - fix to keep 80 cols limit
 - change CheckFakedBlobs() in entry.py
 - remove faked test blob when test is finished

v3:
 - add CheckFakedBlobs() and print a list of faked files at the end
 - add unittest

v2:
 - pass allow_fake_blobs to ProcessImage()
 - set AllowAllowFakeBlob() to images/entries
 - create fake blob in Entry_blot.ObtainContents() when file is missing and
   creation is allowed

 still missing:
  - unittest
  - option to set BINMAN_FAKE_EXT_BLOBS in Makefile via environment
	variable. With that we could simply set this env variable in the CI
	(gitlab-ci.yml) with adding support to buildman.

 Makefile                            |  1 +
 tools/binman/cmdline.py             |  2 ++
 tools/binman/control.py             | 26 +++++++++++++++++++-------
 tools/binman/entry.py               | 23 +++++++++++++++++++++++
 tools/binman/etype/blob.py          | 18 ++++++++++++++++++
 tools/binman/etype/blob_ext.py      |  8 ++++++++
 tools/binman/etype/mkimage.py       | 20 ++++++++++++++++++++
 tools/binman/etype/section.py       | 20 ++++++++++++++++++++
 tools/binman/ftest.py               | 15 ++++++++++++++-
 tools/binman/test/203_fake_blob.dts | 14 ++++++++++++++
 10 files changed, 139 insertions(+), 8 deletions(-)
 create mode 100644 tools/binman/test/203_fake_blob.dts

diff --git a/Makefile b/Makefile
index ae9bfab91a..63d286a4c1 100644
--- a/Makefile
+++ b/Makefile
@@ -1315,6 +1315,7 @@ cmd_binman = $(srctree)/tools/binman/binman $(if $(BINMAN_DEBUG),-D) \
 		-a tpl-bss-pad=$(if $(CONFIG_TPL_SEPARATE_BSS),,1) \
 		-a spl-dtb=$(CONFIG_SPL_OF_REAL) \
 		-a tpl-dtb=$(CONFIG_TPL_OF_REAL) \
+		$(if $(BINMAN_FAKE_EXT_BLOBS),--fake-ext-blobs) \
 		$(BINMAN_$(@F))
 
 OBJCOPYFLAGS_u-boot.ldr.hex := -I binary -O ihex
diff --git a/tools/binman/cmdline.py b/tools/binman/cmdline.py
index e73ff78095..700e5a29a5 100644
--- a/tools/binman/cmdline.py
+++ b/tools/binman/cmdline.py
@@ -52,6 +52,8 @@ controlled by a description in the board device tree.'''
             help='Configuration file (.dtb) to use')
     build_parser.add_argument('--fake-dtb', action='store_true',
             help='Use fake device tree contents (for testing only)')
+    build_parser.add_argument('--fake-ext-blobs', action='store_true',
+            help='Create fake ext blobs with dummy content (for testing only)')
     build_parser.add_argument('-i', '--image', type=str, action='append',
             help='Image filename to build (if not specified, build all)')
     build_parser.add_argument('-I', '--indir', action='append',
diff --git a/tools/binman/control.py b/tools/binman/control.py
index 304fc70f56..47aac207b4 100644
--- a/tools/binman/control.py
+++ b/tools/binman/control.py
@@ -479,7 +479,8 @@ def PrepareImagesAndDtbs(dtb_fname, select_images, update_fdt, use_expanded):
 
 
 def ProcessImage(image, update_fdt, write_map, get_contents=True,
-                 allow_resize=True, allow_missing=False):
+                 allow_resize=True, allow_missing=False,
+                 allow_fake_blobs=False):
     """Perform all steps for this image, including checking and # writing it.
 
     This means that errors found with a later image will be reported after
@@ -495,12 +496,15 @@ def ProcessImage(image, update_fdt, write_map, get_contents=True,
         allow_resize: True to allow entries to change size (this does a re-pack
             of the entries), False to raise an exception
         allow_missing: Allow blob_ext objects to be missing
+        allow_fake_blobs: Allow blob_ext objects to be faked with dummy files
 
     Returns:
-        True if one or more external blobs are missing, False if all are present
+        True if one or more external blobs are missing or faked,
+        False if all are present
     """
     if get_contents:
         image.SetAllowMissing(allow_missing)
+        image.SetAllowFakeBlob(allow_fake_blobs)
         image.GetEntryContents()
     image.GetEntryOffsets()
 
@@ -549,7 +553,13 @@ def ProcessImage(image, update_fdt, write_map, get_contents=True,
         tout.Warning("Image '%s' is missing external blobs and is non-functional: %s" %
                      (image.name, ' '.join([e.name for e in missing_list])))
         _ShowHelpForMissingBlobs(missing_list)
-    return bool(missing_list)
+    faked_list = []
+    image.CheckFakedBlobs(faked_list)
+    if faked_list:
+        tout.Warning("Image '%s:%s' has faked external blobs and is non-functional: %s" %
+                     (image.name, image.image_name,
+                      ' '.join([e.GetDefaultFilename() for e in faked_list])))
+    return bool(missing_list) or bool(faked_list)
 
 
 def Binman(args):
@@ -636,13 +646,15 @@ def Binman(args):
 
             images = PrepareImagesAndDtbs(dtb_fname, args.image,
                                           args.update_fdt, use_expanded)
+
             if args.test_section_timeout:
                 # Set the first image to timeout, used in testThreadTimeout()
                 images[list(images.keys())[0]].test_section_timeout = True
-            missing = False
+            invalid = False
             for image in images.values():
-                missing |= ProcessImage(image, args.update_fdt, args.map,
-                                        allow_missing=args.allow_missing)
+                invalid |= ProcessImage(image, args.update_fdt, args.map,
+                                       allow_missing=args.allow_missing,
+                                       allow_fake_blobs=args.fake_ext_blobs)
 
             # Write the updated FDTs to our output files
             for dtb_item in state.GetAllFdts():
@@ -652,7 +664,7 @@ def Binman(args):
                 data = state.GetFdtForEtype('u-boot-dtb').GetContents()
                 elf.UpdateFile(*elf_params, data)
 
-            if missing:
+            if invalid:
                 tout.Warning("\nSome images are invalid")
 
             # Use this to debug the time take to pack the image
diff --git a/tools/binman/entry.py b/tools/binman/entry.py
index 70222718ea..401476fe76 100644
--- a/tools/binman/entry.py
+++ b/tools/binman/entry.py
@@ -70,6 +70,8 @@ class Entry(object):
         missing: True if this entry is missing its contents
         allow_missing: Allow children of this entry to be missing (used by
             subclasses such as Entry_section)
+        allow_fake: Allow creating a dummy fake file if the blob file is not
+            available. This is mainly used for testing.
         external: True if this entry contains an external binary blob
     """
     def __init__(self, section, etype, node, name_prefix=''):
@@ -98,8 +100,10 @@ class Entry(object):
         self._expand_size = False
         self.compress = 'none'
         self.missing = False
+        self.faked = False
         self.external = False
         self.allow_missing = False
+        self.allow_fake = False
 
     @staticmethod
     def Lookup(node_path, etype, expanded):
@@ -898,6 +902,14 @@ features to produce new behaviours.
         # This is meaningless for anything other than sections
         pass
 
+    def SetAllowFakeBlob(self, allow_fake):
+        """Set whether a section allows to create a fake blob
+
+        Args:
+            allow_fake: True if allowed, False if not allowed
+        """
+        pass
+
     def CheckMissing(self, missing_list):
         """Check if any entries in this section have missing external blobs
 
@@ -909,6 +921,17 @@ features to produce new behaviours.
         if self.missing:
             missing_list.append(self)
 
+    def CheckFakedBlobs(self, faked_blobs_list):
+        """Check if any entries in this section have faked external blobs
+
+        If there are faked blobs, the entries are added to the list
+
+        Args:
+            fake_blobs_list: List of Entry objects to be added to
+        """
+        # This is meaningless for anything other than blobs
+        pass
+
     def GetAllowMissing(self):
         """Get whether a section allows missing external blobs
 
diff --git a/tools/binman/etype/blob.py b/tools/binman/etype/blob.py
index fae86ca3ec..6e63d777eb 100644
--- a/tools/binman/etype/blob.py
+++ b/tools/binman/etype/blob.py
@@ -5,6 +5,8 @@
 # Entry-type module for blobs, which are binary objects read from files
 #
 
+import pathlib
+
 from binman.entry import Entry
 from binman import state
 from dtoc import fdt_util
@@ -36,6 +38,11 @@ class Entry_blob(Entry):
         self._filename = fdt_util.GetString(self._node, 'filename', self.etype)
 
     def ObtainContents(self):
+        if self.allow_fake and not pathlib.Path(self._filename).is_file():
+            with open(self._filename, "wb") as out:
+                out.truncate(1024)
+            self.faked = True
+
         self._filename = self.GetDefaultFilename()
         self._pathname = tools.GetInputFilename(self._filename,
             self.external and self.section.GetAllowMissing())
@@ -75,3 +82,14 @@ class Entry_blob(Entry):
     def ProcessContents(self):
         # The blob may have changed due to WriteSymbols()
         return self.ProcessContentsUpdate(self.data)
+
+    def CheckFakedBlobs(self, faked_blobs_list):
+        """Check if any entries in this section have faked external blobs
+
+        If there are faked blobs, the entries are added to the list
+
+        Args:
+            fake_blobs_list: List of Entry objects to be added to
+        """
+        if self.faked:
+            faked_blobs_list.append(self)
diff --git a/tools/binman/etype/blob_ext.py b/tools/binman/etype/blob_ext.py
index d6b0ca17c3..fba6271de2 100644
--- a/tools/binman/etype/blob_ext.py
+++ b/tools/binman/etype/blob_ext.py
@@ -26,3 +26,11 @@ class Entry_blob_ext(Entry_blob):
     def __init__(self, section, etype, node):
         Entry_blob.__init__(self, section, etype, node)
         self.external = True
+
+    def SetAllowFakeBlob(self, allow_fake):
+        """Set whether the entry allows to create a fake blob
+
+        Args:
+            allow_fake_blob: True if allowed, False if not allowed
+        """
+        self.allow_fake = allow_fake
diff --git a/tools/binman/etype/mkimage.py b/tools/binman/etype/mkimage.py
index e49977522e..80fdce0b14 100644
--- a/tools/binman/etype/mkimage.py
+++ b/tools/binman/etype/mkimage.py
@@ -61,3 +61,23 @@ class Entry_mkimage(Entry):
             entry = Entry.Create(self, node)
             entry.ReadNode()
             self._mkimage_entries[entry.name] = entry
+
+    def SetAllowFakeBlob(self, allow_fake):
+        """Set whether the sub nodes allows to create a fake blob
+
+        Args:
+            allow_fake: True if allowed, False if not allowed
+        """
+        for entry in self._mkimage_entries.values():
+            entry.SetAllowFakeBlob(allow_fake)
+
+    def CheckFakedBlobs(self, faked_blobs_list):
+        """Check if any entries in this section have faked external blobs
+
+        If there are faked blobs, the entries are added to the list
+
+        Args:
+            faked_blobs_list: List of Entry objects to be added to
+        """
+        for entry in self._mkimage_entries.values():
+            entry.CheckFakedBlobs(faked_blobs_list)
diff --git a/tools/binman/etype/section.py b/tools/binman/etype/section.py
index e2949fc916..4e423855d9 100644
--- a/tools/binman/etype/section.py
+++ b/tools/binman/etype/section.py
@@ -689,6 +689,15 @@ class Entry_section(Entry):
         for entry in self._entries.values():
             entry.SetAllowMissing(allow_missing)
 
+    def SetAllowFakeBlob(self, allow_fake):
+        """Set whether a section allows to create a fake blob
+
+        Args:
+            allow_fake_blob: True if allowed, False if not allowed
+        """
+        for entry in self._entries.values():
+            entry.SetAllowFakeBlob(allow_fake)
+
     def CheckMissing(self, missing_list):
         """Check if any entries in this section have missing external blobs
 
@@ -700,6 +709,17 @@ class Entry_section(Entry):
         for entry in self._entries.values():
             entry.CheckMissing(missing_list)
 
+    def CheckFakedBlobs(self, faked_blobs_list):
+        """Check if any entries in this section have faked external blobs
+
+        If there are faked blobs, the entries are added to the list
+
+        Args:
+            fake_blobs_list: List of Entry objects to be added to
+        """
+        for entry in self._entries.values():
+            entry.CheckFakedBlobs(faked_blobs_list)
+
     def _CollectEntries(self, entries, entries_by_name, add_entry):
         """Collect all the entries in an section
 
diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py
index 6be003786e..2a98d81028 100644
--- a/tools/binman/ftest.py
+++ b/tools/binman/ftest.py
@@ -308,7 +308,7 @@ class TestFunctional(unittest.TestCase):
     def _DoTestFile(self, fname, debug=False, map=False, update_dtb=False,
                     entry_args=None, images=None, use_real_dtb=False,
                     use_expanded=False, verbosity=None, allow_missing=False,
-                    extra_indirs=None, threads=None,
+                    allow_fake_blobs=False, extra_indirs=None, threads=None,
                     test_section_timeout=False, update_fdt_in_elf=None):
         """Run binman with a given test file
 
@@ -331,6 +331,7 @@ class TestFunctional(unittest.TestCase):
             verbosity: Verbosity level to use (0-3, None=don't set it)
             allow_missing: Set the '--allow-missing' flag so that missing
                 external binaries just produce a warning instead of an error
+            allow_fake_blobs: Set the '--fake-ext-blobs' flag
             extra_indirs: Extra input directories to add using -I
             threads: Number of threads to use (None for default, 0 for
                 single-threaded)
@@ -369,6 +370,8 @@ class TestFunctional(unittest.TestCase):
                 args.append('-a%s=%s' % (arg, value))
         if allow_missing:
             args.append('-M')
+        if allow_fake_blobs:
+            args.append('--fake-ext-blobs')
         if update_fdt_in_elf:
             args += ['--update-fdt-in-elf', update_fdt_in_elf]
         if images:
@@ -4661,6 +4664,16 @@ class TestFunctional(unittest.TestCase):
             str(e.exception),
             "Not enough space in '.*u_boot_binman_embed_sm' for data length.*")
 
+    def testFakeBlob(self):
+        """Test handling of faking an external blob"""
+        with test_util.capture_sys_output() as (stdout, stderr):
+            self._DoTestFile('203_fake_blob.dts', allow_missing=True,
+                             allow_fake_blobs=True)
+        err = stderr.getvalue()
+        self.assertRegex(err,
+                         "Image '.*' has faked external blobs and is non-functional: .*")
+        os.remove('binman_faking_test_blob')
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tools/binman/test/203_fake_blob.dts b/tools/binman/test/203_fake_blob.dts
new file mode 100644
index 0000000000..22cf67f4f8
--- /dev/null
+++ b/tools/binman/test/203_fake_blob.dts
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	binman {
+		blob-ext {
+			filename = "binman_faking_test_blob";
+		};
+	};
+};
-- 
2.30.2