[Buildroot] [PATCH v3] Add support for Flattened Image Trees

[Buildroot] [PATCH v3] Add support for Flattened Image Trees

[Yegor Yefremov]

This patch provides an option to specify a FIT source
file (*.its), that will usually be stored in a board directory,
and the resulting blob file name (*.itb).

Signed-off-by: Yegor Yefremov <yegorslists@googlemail.com>
---
Changes:
	v3: fix syntax errors, remove itb name, add uboot-tools dependency
        v2: moved configuration to fs infra instead of linux

 fs/Config.in     |  1 +
 fs/fit/Config.in | 23 +++++++++++++++++++++++
 fs/fit/fit.mk    | 23 +++++++++++++++++++++++
 3 files changed, 47 insertions(+)
 create mode 100644 fs/fit/Config.in
 create mode 100644 fs/fit/fit.mk

diff --git a/fs/Config.in b/fs/Config.in
index 5853113..8d882d0 100644
--- a/fs/Config.in
+++ b/fs/Config.in
@@ -4,6 +4,7 @@ source "fs/cloop/Config.in"
 source "fs/cpio/Config.in"
 source "fs/cramfs/Config.in"
 source "fs/ext2/Config.in"
+source "fs/fit/Config.in"
 source "fs/initramfs/Config.in"
 source "fs/iso9660/Config.in"
 source "fs/jffs2/Config.in"
diff --git a/fs/fit/Config.in b/fs/fit/Config.in
new file mode 100644
index 0000000..12e977b
--- /dev/null
+++ b/fs/fit/Config.in
@@ -0,0 +1,23 @@
+config BR2_TARGET_ROOTFS_FIT
+	bool "Flattened Image Tree support"
+	select BR2_PACKAGE_HOST_UBOOT_TOOLS
+	help
+	  Compile a flattened image tree source into a flattened
+	  image tree blob. Specify the *.its file to compile in the
+	  options below.
+
+if BR2_TARGET_ROOTFS_FIT
+
+config BR2_FIT_PATH
+	string "Flattened Image Tree source file path"
+	help
+	  Path to the image tree source files.
+
+config BR2_FIT_INSTALL_TARGET
+	bool "Install Flattened Image Tree blob to /boot in target"
+	depends on !BR2_TARGET_ROOTFS_INITRAMFS
+	help
+	  Select this option to have the FIT image installed to
+	  /boot in the target root filesystem.
+
+endif
diff --git a/fs/fit/fit.mk b/fs/fit/fit.mk
new file mode 100644
index 0000000..2283cd7
--- /dev/null
+++ b/fs/fit/fit.mk
@@ -0,0 +1,23 @@
+################################################################################
+#
+# Build the FIT image
+#
+################################################################################
+
+ROOTFS_FIT_DEPENDENCIES = host-uboot-tools
+
+ifeq ($(BR2_FIT_INSTALL_TARGET),y)
+define ROOTFS_FIT_CMD
+	cp $(BR2_FIT_PATH) $(BINARIES_DIR) && \
+	$(HOST_DIR)/usr/bin/mkimage -f $(BINARIES_DIR)/$(notdir $(call qstrip,$(BR2_FIT_PATH))) $(BINARIES_DIR)/$(basename $(notdir $(call qstrip,$(BR2_FIT_PATH)))).itb && \
+	mkdir -p $(TARGET_DIR)/boot/ && \
+	cp $(BINARIES_DIR)/$(basename $(notdir $(call qstrip,$(BR2_FIT_PATH)))).itb $(TARGET_DIR)/boot/
+endef
+else
+define ROOTFS_FIT_CMD
+	cp $(BR2_FIT_PATH) $(BINARIES_DIR) && \
+	$(HOST_DIR)/usr/bin/mkimage -f $(BINARIES_DIR)/$(notdir $(call qstrip,$(BR2_FIT_PATH))) $(BINARIES_DIR)/$(basename $(notdir $(call qstrip,$(BR2_FIT_PATH)))).itb
+endef
+endif
+
+$(eval $(call ROOTFS_TARGET,fit))
-- 
1.8.3.2

[Buildroot] [PATCH v3] Add support for Flattened Image Trees

[Thomas Petazzoni]

Dear Yegor Yefremov,

On Thu, 18 Sep 2014 22:13:44 +0200, Yegor Yefremov wrote:

> +config BR2_FIT_PATH
> +	string "Flattened Image Tree source file path"
> +	help
> +	  Path to the image tree source files.

Which files will typically be listed here? Apparently your code assumes
those files must all be located in $(BINARIES_DIR), right? If that's
the case, it should be mentioned here.

Also, I'm worried about ordering: there is no ordering guarantee
between filesystem image types. So if your FIT filesystem image logic
depends on another filesystem image to be generated, this might be an
issue.

Also you say "files" in the description, but your code doesn't seem to
handle multiple "files". Can you explain?

> +ifeq ($(BR2_FIT_INSTALL_TARGET),y)
> +define ROOTFS_FIT_CMD
> +	cp $(BR2_FIT_PATH) $(BINARIES_DIR) && \
> +	$(HOST_DIR)/usr/bin/mkimage -f $(BINARIES_DIR)/$(notdir $(call qstrip,$(BR2_FIT_PATH))) $(BINARIES_DIR)/$(basename $(notdir $(call qstrip,$(BR2_FIT_PATH)))).itb && \
> +	mkdir -p $(TARGET_DIR)/boot/ && \
> +	cp $(BINARIES_DIR)/$(basename $(notdir $(call qstrip,$(BR2_FIT_PATH)))).itb $(TARGET_DIR)/boot/
> +endef
> +else
> +define ROOTFS_FIT_CMD
> +	cp $(BR2_FIT_PATH) $(BINARIES_DIR) && \
> +	$(HOST_DIR)/usr/bin/mkimage -f $(BINARIES_DIR)/$(notdir $(call qstrip,$(BR2_FIT_PATH))) $(BINARIES_DIR)/$(basename $(notdir $(call qstrip,$(BR2_FIT_PATH)))).itb
> +endef
> +endif

Avoid duplication:

ROOTFS_FIT_FILE = $(BINARIES_DIR)/$(notdir $(call qstrip,$(BR2_FIT_PATH)))
ROOTFS_FIT_ITB_PATH = $(ROOTFS_FIT_FILE).itb

ifeq ($(BR2_FIT_INSTALL_TARGET),y)
define ROOTFS_FIT_INSTALL_TARGET
	mkdir -p $(TARGET_DIR)/boot && cp $(ROOTFS_FIT_ITB_PATH) $(TARGET_DIR)/boot
endef
else
define ROOTFS_FIT_INSTALL_TARGET
	true
endef
endif

define ROOTFS_FIT_CMD
	cp $(BR2_FIT_PATH) $(BINARIES_DIR) && \
	$(HOST_DIR)/usr/bin/mkimage -f $(ROOTFS_FILE_FILE) $(ROOTFS_FIT_ITB_PATH) && \
	$(ROOTFS_FIT_INSTALL_TARGET)
endef

But maybe we should change the ROOTFS_TARGET infrastructure to support
multiple commands. There's also maybe the ROOTFS_xyz_POST_TARGETS
mechanism that could be used (I haven't looked whether it applies
properly here).

Can you give an example for the various Config.in options, so that
others could test this and understand better how it's typically used?

Best regards,

Thomas
-- 
Thomas Petazzoni, CTO, Free Electrons
Embedded Linux, Kernel and Android engineering
http://free-electrons.com

[Buildroot] [PATCH v3] Add support for Flattened Image Trees

[Baruch Siach]

Hi Yegor,

On Thu, Sep 18, 2014 at 10:13:44PM +0200, Yegor Yefremov wrote:
> This patch provides an option to specify a FIT source
> file (*.its), that will usually be stored in a board directory,
> and the resulting blob file name (*.itb).
> 
> Signed-off-by: Yegor Yefremov <yegorslists@googlemail.com>
> ---

[snip]

> +config BR2_FIT_INSTALL_TARGET
> +	bool "Install Flattened Image Tree blob to /boot in target"
> +	depends on !BR2_TARGET_ROOTFS_INITRAMFS

I don't think it is reasonable to disable target FIT image install just 
because the user has enabled initramfs target generation. Wouldn't it be nicer 
to remove the /boot directory from the generated .cpio image? If this is too 
hard to do, I suggest to add a comment explaining this limitation to be 
displayed when BR2_TARGET_ROOTFS_INITRAMFS is enabled.

Thanks for working on this,
baruch

> +	help
> +	  Select this option to have the FIT image installed to
> +	  /boot in the target root filesystem.
> +
> +endif

-- 
     http://baruch.siach.name/blog/                  ~. .~   Tk Open Systems
=}------------------------------------------------ooO--U--Ooo------------{=
   - baruch at tkos.co.il - tel: +972.2.679.5364, http://www.tkos.co.il -

[Buildroot] [PATCH v3] Add support for Flattened Image Trees

[Yegor Yefremov]

On Sun, Sep 21, 2014 at 8:46 AM, Baruch Siach <baruch@tkos.co.il> wrote:
> Hi Yegor,
>
> On Thu, Sep 18, 2014 at 10:13:44PM +0200, Yegor Yefremov wrote:
>> This patch provides an option to specify a FIT source
>> file (*.its), that will usually be stored in a board directory,
>> and the resulting blob file name (*.itb).
>>
>> Signed-off-by: Yegor Yefremov <yegorslists@googlemail.com>
>> ---
>
> [snip]
>
>> +config BR2_FIT_INSTALL_TARGET
>> +     bool "Install Flattened Image Tree blob to /boot in target"
>> +     depends on !BR2_TARGET_ROOTFS_INITRAMFS
>
> I don't think it is reasonable to disable target FIT image install just
> because the user has enabled initramfs target generation. Wouldn't it be nicer
> to remove the /boot directory from the generated .cpio image? If this is too
> hard to do, I suggest to add a comment explaining this limitation to be
> displayed when BR2_TARGET_ROOTFS_INITRAMFS is enabled.

I've copied this dependency/restriction from kernel (linux.mk),
because it has the same meaning. At least for me. So I'll try to add
related warning.

Yegor

[Buildroot] [PATCH v4] Add support for Flattened Image Trees

[Yegor Yefremov]

This patch provides an option to specify a FIT source
file (*.its), that will usually be stored in a board directory,
and the resulting blob file name (*.itb).

Signed-off-by: Yegor Yefremov <yegorslists@googlemail.com>
---
Changes:
        v4: add support for multiple *.its files, reduce code duplication
        v3: fix syntax errors, remove itb name, add uboot-tools dependency
        v2: moved configuration to fs infra instead of linux

 fs/Config.in     |  1 +
 fs/fit/Config.in | 31 +++++++++++++++++++++++++++++++
 fs/fit/fit.mk    | 29 +++++++++++++++++++++++++++++
 3 files changed, 61 insertions(+)
 create mode 100644 fs/fit/Config.in
 create mode 100644 fs/fit/fit.mk

diff --git a/fs/Config.in b/fs/Config.in
index 5853113..8d882d0 100644
--- a/fs/Config.in
+++ b/fs/Config.in
@@ -4,6 +4,7 @@ source "fs/cloop/Config.in"
 source "fs/cpio/Config.in"
 source "fs/cramfs/Config.in"
 source "fs/ext2/Config.in"
+source "fs/fit/Config.in"
 source "fs/initramfs/Config.in"
 source "fs/iso9660/Config.in"
 source "fs/jffs2/Config.in"
diff --git a/fs/fit/Config.in b/fs/fit/Config.in
new file mode 100644
index 0000000..f05b2a7
--- /dev/null
+++ b/fs/fit/Config.in
@@ -0,0 +1,31 @@
+config BR2_TARGET_ROOTFS_FIT
+       bool "Flattened Image Tree support"
+       select BR2_PACKAGE_HOST_UBOOT_TOOLS
+       help
+         Compile a flattened image tree sources into a flattened
+         image tree blobs. Specify the *.its files to compile in
+         the options below.
+
+         Please note, that all files referenced in *.its files,
+         should be available in output/images folder.
+
+if BR2_TARGET_ROOTFS_FIT
+
+config BR2_FIT_PATH
+       string "Flattened Image Tree source file paths"
+       help
+         Paths to the image tree source files. You can
+         provide a list of *.its paths to copy and build,
+         separated by spaces.
+
+         Please note, that all *.its files must be named uniquely,
+         otherwise the files with the same name will be overwritten.
+
+config BR2_FIT_INSTALL_TARGET
+       bool "Install Flattened Image Tree blobs to /boot in target"
+       depends on !BR2_TARGET_ROOTFS_INITRAMFS
+       help
+         Select this option to have the FIT images installed to
+         /boot in the target root filesystem.
+
+endif
diff --git a/fs/fit/fit.mk b/fs/fit/fit.mk
new file mode 100644
index 0000000..68a9557
--- /dev/null
+++ b/fs/fit/fit.mk
@@ -0,0 +1,29 @@
+################################################################################
+#
+# Build the FIT image
+#
+################################################################################
+
+ROOTFS_FIT_DEPENDENCIES = host-uboot-tools
+
+FIT_SRCS = $(addprefix $(BINARIES_DIR)/,$(notdir $(call
qstrip,$(BR2_FIT_PATH))))
+FIT_BLOBS = $(addsuffix .itb, $(basename $(FIT_SRCS)))
+
+ifeq ($(BR2_FIT_INSTALL_TARGET),y)
+define FIT_INSTALL_ITB_TARGET
+       mkdir -p $(TARGET_DIR)/boot/ && \
+       cp $(FIT_BLOBS) $(TARGET_DIR)/boot/
+endef
+else
+define FIT_INSTALL_ITB_TARGET
+       true
+endef
+endif
+
+define ROOTFS_FIT_CMD
+       cp $(BR2_FIT_PATH) $(BINARIES_DIR) && \
+       $(foreach its,$(FIT_SRCS),$(HOST_DIR)/usr/bin/mkimage -f
$(its) $(basename $(its)).itb &&) \
+       $(FIT_INSTALL_ITB_TARGET)
+endef
+
+$(eval $(call ROOTFS_TARGET,fit))
--
1.9.1

[Buildroot] [PATCH v3] Add support for Flattened Image Trees

[Yegor Yefremov]

On Thu, Sep 18, 2014 at 10:44 PM, Thomas Petazzoni
<thomas.petazzoni@free-electrons.com> wrote:
> Dear Yegor Yefremov,
>
> On Thu, 18 Sep 2014 22:13:44 +0200, Yegor Yefremov wrote:
>
>> +config BR2_FIT_PATH
>> +     string "Flattened Image Tree source file path"
>> +     help
>> +       Path to the image tree source files.
>
> Which files will typically be listed here? Apparently your code assumes
> those files must all be located in $(BINARIES_DIR), right? If that's
> the case, it should be mentioned here.

*.its files can be anywhere, but will be copied to $(BINARIES_DIR) to
overcome mkimage/dtb compiler path issues (mkimage invokes dtb
compiler at *.its location, so working directory changes).

> Also, I'm worried about ordering: there is no ordering guarantee
> between filesystem image types. So if your FIT filesystem image logic
> depends on another filesystem image to be generated, this might be an
> issue.

This is the crucial moment. For me there are basically following cases:

1.  create an image consisting of zImage + DTB and install it to boot
- in this case FIT must be invoked before file system packaging
2. create an image having cpio inside - in this case FIT must be the
last action taken in fs stuff or at least after cpio archive is
packaged
3. create an image with appended cpio as initramfs - in this case FIT
must be invoked after cpio is embedded into kernel

What would be the best strategy to implement these cases?

> Also you say "files" in the description, but your code doesn't seem to
> handle multiple "files". Can you explain?

See v4

>> +ifeq ($(BR2_FIT_INSTALL_TARGET),y)
>> +define ROOTFS_FIT_CMD
>> +     cp $(BR2_FIT_PATH) $(BINARIES_DIR) && \
>> +     $(HOST_DIR)/usr/bin/mkimage -f $(BINARIES_DIR)/$(notdir $(call qstrip,$(BR2_FIT_PATH))) $(BINARIES_DIR)/$(basename $(notdir $(call qstrip,$(BR2_FIT_PATH)))).itb && \
>> +     mkdir -p $(TARGET_DIR)/boot/ && \
>> +     cp $(BINARIES_DIR)/$(basename $(notdir $(call qstrip,$(BR2_FIT_PATH)))).itb $(TARGET_DIR)/boot/
>> +endef
>> +else
>> +define ROOTFS_FIT_CMD
>> +     cp $(BR2_FIT_PATH) $(BINARIES_DIR) && \
>> +     $(HOST_DIR)/usr/bin/mkimage -f $(BINARIES_DIR)/$(notdir $(call qstrip,$(BR2_FIT_PATH))) $(BINARIES_DIR)/$(basename $(notdir $(call qstrip,$(BR2_FIT_PATH)))).itb
>> +endef
>> +endif
>
> Avoid duplication:
>
> ROOTFS_FIT_FILE = $(BINARIES_DIR)/$(notdir $(call qstrip,$(BR2_FIT_PATH)))
> ROOTFS_FIT_ITB_PATH = $(ROOTFS_FIT_FILE).itb
>
> ifeq ($(BR2_FIT_INSTALL_TARGET),y)
> define ROOTFS_FIT_INSTALL_TARGET
>         mkdir -p $(TARGET_DIR)/boot && cp $(ROOTFS_FIT_ITB_PATH) $(TARGET_DIR)/boot
> endef
> else
> define ROOTFS_FIT_INSTALL_TARGET
>         true
> endef
> endif
>
> define ROOTFS_FIT_CMD
>         cp $(BR2_FIT_PATH) $(BINARIES_DIR) && \
>         $(HOST_DIR)/usr/bin/mkimage -f $(ROOTFS_FILE_FILE) $(ROOTFS_FIT_ITB_PATH) && \
>         $(ROOTFS_FIT_INSTALL_TARGET)
> endef

Fixed so far in v4.

> But maybe we should change the ROOTFS_TARGET infrastructure to support
> multiple commands. There's also maybe the ROOTFS_xyz_POST_TARGETS
> mechanism that could be used (I haven't looked whether it applies
> properly here).
>
> Can you give an example for the various Config.in options, so that
> others could test this and understand better how it's typically used?

I think now the approach is straight forward: you specify one or more
*.its files and you get *.itb files. The only option so far is to
install *.itbs into /boot/ or not.

Yegor

[Buildroot] [PATCH v4] Add support for Flattened Image Trees

[Yegor Yefremov]

On Mon, Sep 22, 2014 at 2:45 PM, Yegor Yefremov
<yegorslists@googlemail.com> wrote:
> This patch provides an option to specify a FIT source
> file (*.its), that will usually be stored in a board directory,
> and the resulting blob file name (*.itb).
>
> Signed-off-by: Yegor Yefremov <yegorslists@googlemail.com>
> ---
> Changes:
>         v4: add support for multiple *.its files, reduce code duplication
>         v3: fix syntax errors, remove itb name, add uboot-tools dependency
>         v2: moved configuration to fs infra instead of linux
>
>  fs/Config.in     |  1 +
>  fs/fit/Config.in | 31 +++++++++++++++++++++++++++++++
>  fs/fit/fit.mk    | 29 +++++++++++++++++++++++++++++
>  3 files changed, 61 insertions(+)
>  create mode 100644 fs/fit/Config.in
>  create mode 100644 fs/fit/fit.mk
>
> diff --git a/fs/Config.in b/fs/Config.in
> index 5853113..8d882d0 100644
> --- a/fs/Config.in
> +++ b/fs/Config.in
> @@ -4,6 +4,7 @@ source "fs/cloop/Config.in"
>  source "fs/cpio/Config.in"
>  source "fs/cramfs/Config.in"
>  source "fs/ext2/Config.in"
> +source "fs/fit/Config.in"
>  source "fs/initramfs/Config.in"
>  source "fs/iso9660/Config.in"
>  source "fs/jffs2/Config.in"
> diff --git a/fs/fit/Config.in b/fs/fit/Config.in
> new file mode 100644
> index 0000000..f05b2a7
> --- /dev/null
> +++ b/fs/fit/Config.in
> @@ -0,0 +1,31 @@
> +config BR2_TARGET_ROOTFS_FIT
> +       bool "Flattened Image Tree support"
> +       select BR2_PACKAGE_HOST_UBOOT_TOOLS
> +       help
> +         Compile a flattened image tree sources into a flattened
> +         image tree blobs. Specify the *.its files to compile in
> +         the options below.
> +
> +         Please note, that all files referenced in *.its files,
> +         should be available in output/images folder.
> +
> +if BR2_TARGET_ROOTFS_FIT
> +
> +config BR2_FIT_PATH
> +       string "Flattened Image Tree source file paths"
> +       help
> +         Paths to the image tree source files. You can
> +         provide a list of *.its paths to copy and build,
> +         separated by spaces.
> +
> +         Please note, that all *.its files must be named uniquely,
> +         otherwise the files with the same name will be overwritten.
> +
> +config BR2_FIT_INSTALL_TARGET
> +       bool "Install Flattened Image Tree blobs to /boot in target"
> +       depends on !BR2_TARGET_ROOTFS_INITRAMFS
> +       help
> +         Select this option to have the FIT images installed to
> +         /boot in the target root filesystem.
> +
> +endif
> diff --git a/fs/fit/fit.mk b/fs/fit/fit.mk
> new file mode 100644
> index 0000000..68a9557
> --- /dev/null
> +++ b/fs/fit/fit.mk
> @@ -0,0 +1,29 @@
> +################################################################################
> +#
> +# Build the FIT image
> +#
> +################################################################################
> +
> +ROOTFS_FIT_DEPENDENCIES = host-uboot-tools
> +
> +FIT_SRCS = $(addprefix $(BINARIES_DIR)/,$(notdir $(call
> qstrip,$(BR2_FIT_PATH))))
> +FIT_BLOBS = $(addsuffix .itb, $(basename $(FIT_SRCS)))
> +
> +ifeq ($(BR2_FIT_INSTALL_TARGET),y)
> +define FIT_INSTALL_ITB_TARGET
> +       mkdir -p $(TARGET_DIR)/boot/ && \
> +       cp $(FIT_BLOBS) $(TARGET_DIR)/boot/
> +endef
> +else
> +define FIT_INSTALL_ITB_TARGET
> +       true
> +endef
> +endif
> +
> +define ROOTFS_FIT_CMD
> +       cp $(BR2_FIT_PATH) $(BINARIES_DIR) && \
> +       $(foreach its,$(FIT_SRCS),$(HOST_DIR)/usr/bin/mkimage -f
> $(its) $(basename $(its)).itb &&) \
> +       $(FIT_INSTALL_ITB_TARGET)
> +endef
> +
> +$(eval $(call ROOTFS_TARGET,fit))
> --
> 1.9.1

I'm still working on the problem with sending e-mails with corporate
account, so the patch was just copypasted into GMail web interface and
is intended for reviewing only.

Yegor