[U-Boot] [PATCH 1/2] cmd/imxtract: Move decompression part to separate function

[U-Boot] [PATCH 1/2] cmd/imxtract: Move decompression part to separate function

[Alexey Ignatov]

Signed-off-by: Alexey Ignatov <lexszero@gmail.com>
---
 cmd/ximg.c | 142 +++++++++++++++++++++++++++++++++----------------------------
 1 file changed, 76 insertions(+), 66 deletions(-)

diff --git a/cmd/ximg.c b/cmd/ximg.c
index d033c15b62..73a571b52b 100644
--- a/cmd/ximg.c
+++ b/cmd/ximg.c
@@ -28,6 +28,81 @@
 #define CONFIG_SYS_XIMG_LEN	0x800000
 #endif
 
+static int decompress_data(ulong dest, ulong data, ulong len,
+		uint8_t comp, int part)
+{
+#ifdef CONFIG_GZIP
+	uint		unc_len = CONFIG_SYS_XIMG_LEN;
+#endif
+
+	switch (comp) {
+	case IH_COMP_NONE:
+#if defined(CONFIG_HW_WATCHDOG) || defined(CONFIG_WATCHDOG)
+		{
+			size_t l = len;
+			size_t tail;
+			void *to = (void *) dest;
+			void *from = (void *)data;
+
+			printf("   Loading part %d ... ", part);
+
+			while (l > 0) {
+				tail = (l > CHUNKSZ) ? CHUNKSZ : l;
+				WATCHDOG_RESET();
+				memmove(to, from, tail);
+				to += tail;
+				from += tail;
+				l -= tail;
+			}
+		}
+#else	/* !(CONFIG_HW_WATCHDOG || CONFIG_WATCHDOG) */
+		printf("   Loading part %d ... ", part);
+		memmove((char *) dest, (char *)data, len);
+#endif	/* CONFIG_HW_WATCHDOG || CONFIG_WATCHDOG */
+		break;
+#ifdef CONFIG_GZIP
+	case IH_COMP_GZIP:
+		printf("   Uncompressing part %d ... ", part);
+		if (gunzip((void *) dest, unc_len,
+			   (uchar *) data, &len) != 0) {
+			puts("GUNZIP ERROR - image not loaded\n");
+			return 1;
+		}
+		break;
+#endif
+#if defined(CONFIG_BZIP2) && defined(CONFIG_IMAGE_FORMAT_LEGACY)
+	case IH_COMP_BZIP2:
+		{
+			int i;
+
+			printf("   Uncompressing part %d ... ", part);
+			/*
+			 * If we've got less than 4 MB of malloc()
+			 * space, use slower decompression algorithm
+			 * which requires at most 2300 KB of memory.
+			 */
+			i = BZ2_bzBuffToBuffDecompress(
+				map_sysmem(ntohl(hdr->ih_load), 0),
+				&unc_len, (char *)data, len,
+				CONFIG_SYS_MALLOC_LEN < (4096 * 1024),
+				0);
+			if (i != BZ_OK) {
+				printf("BUNZIP2 ERROR %d - "
+					"image not loaded\n", i);
+				return 1;
+			}
+		}
+		break;
+#endif /* CONFIG_BZIP2 */
+	default:
+		printf("Unimplemented compression type %d\n", comp);
+		return 1;
+	}
+	puts("OK\n");
+
+	return 0;
+}
+
 static int
 do_imgextract(cmd_tbl_t * cmdtp, int flag, int argc, char * const argv[])
 {
@@ -47,9 +122,6 @@ do_imgextract(cmd_tbl_t * cmdtp, int flag, int argc, char * const argv[])
 	const void	*fit_data;
 	size_t		fit_len;
 #endif
-#ifdef CONFIG_GZIP
-	uint		unc_len = CONFIG_SYS_XIMG_LEN;
-#endif
 	uint8_t		comp;
 
 	verify = getenv_yesno("verify");
@@ -183,70 +255,8 @@ do_imgextract(cmd_tbl_t * cmdtp, int flag, int argc, char * const argv[])
 	}
 
 	if (argc > 3) {
-		switch (comp) {
-		case IH_COMP_NONE:
-#if defined(CONFIG_HW_WATCHDOG) || defined(CONFIG_WATCHDOG)
-			{
-				size_t l = len;
-				size_t tail;
-				void *to = (void *) dest;
-				void *from = (void *)data;
-
-				printf("   Loading part %d ... ", part);
-
-				while (l > 0) {
-					tail = (l > CHUNKSZ) ? CHUNKSZ : l;
-					WATCHDOG_RESET();
-					memmove(to, from, tail);
-					to += tail;
-					from += tail;
-					l -= tail;
-				}
-			}
-#else	/* !(CONFIG_HW_WATCHDOG || CONFIG_WATCHDOG) */
-			printf("   Loading part %d ... ", part);
-			memmove((char *) dest, (char *)data, len);
-#endif	/* CONFIG_HW_WATCHDOG || CONFIG_WATCHDOG */
-			break;
-#ifdef CONFIG_GZIP
-		case IH_COMP_GZIP:
-			printf("   Uncompressing part %d ... ", part);
-			if (gunzip((void *) dest, unc_len,
-				   (uchar *) data, &len) != 0) {
-				puts("GUNZIP ERROR - image not loaded\n");
-				return 1;
-			}
-			break;
-#endif
-#if defined(CONFIG_BZIP2) && defined(CONFIG_IMAGE_FORMAT_LEGACY)
-		case IH_COMP_BZIP2:
-			{
-				int i;
-
-				printf("   Uncompressing part %d ... ", part);
-				/*
-				 * If we've got less than 4 MB of malloc()
-				 * space, use slower decompression algorithm
-				 * which requires at most 2300 KB of memory.
-				 */
-				i = BZ2_bzBuffToBuffDecompress(
-					map_sysmem(ntohl(hdr->ih_load), 0),
-					&unc_len, (char *)data, len,
-					CONFIG_SYS_MALLOC_LEN < (4096 * 1024),
-					0);
-				if (i != BZ_OK) {
-					printf("BUNZIP2 ERROR %d - "
-						"image not loaded\n", i);
-					return 1;
-				}
-			}
-			break;
-#endif /* CONFIG_BZIP2 */
-		default:
-			printf("Unimplemented compression type %d\n", comp);
+		if (!decompress_data(dest, data, len, comp, part))
 			return 1;
-		}
-		puts("OK\n");
 	}
 
 	flush_cache(dest, len);
-- 
2.12.2

[U-Boot] [PATCH 2/2] cmd: fsfitxtract: Extract a part of a FIT multi-image stored on a filesystem

[Alexey Ignatov]

Sometimes we interested only in one single small subimage from big multipart
FIT. This command tries to avoid reading out the whole FIT because of memory
or time considerations.
Since we don't have mmap() in U-Boot, this is done by reading the file in
small chunks and trying to parse FIT until requested image is found or
total read size limit exceeded.

Signed-off-by: Alexey Ignatov <lexszero@gmail.com>
---
 cmd/Kconfig |   7 ++
 cmd/ximg.c  | 226 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 233 insertions(+)

diff --git a/cmd/Kconfig b/cmd/Kconfig
index 2e4d2334a6..02b8357224 100644
--- a/cmd/Kconfig
+++ b/cmd/Kconfig
@@ -265,6 +265,13 @@ config CMD_XIMG
 	help
 	  Extract a part of a multi-image.
 
+config CMD_FSFITXIMG
+	bool "fsfitxtract"
+	default y
+	depends on FIT
+	help
+	  Extract a part of a FIT multi-image stored on a filesystem
+
 config CMD_POWEROFF
 	bool
 
diff --git a/cmd/ximg.c b/cmd/ximg.c
index 73a571b52b..17d961d5b6 100644
--- a/cmd/ximg.c
+++ b/cmd/ximg.c
@@ -22,6 +22,11 @@
 #endif
 #include <asm/byteorder.h>
 #include <asm/io.h>
+#ifdef CONFIG_CMD_FSFITXIMG
+#include <fs.h>
+#include <div64.h>
+#include <libfdt.h>
+#endif
 
 #ifndef CONFIG_SYS_XIMG_LEN
 /* use 8MByte as default max gunzip size */
@@ -283,3 +288,224 @@ U_BOOT_CMD(
 	imxtract, 4, 1, do_imgextract,
 	"extract a part of a multi-image", imgextract_help_text
 );
+
+#if defined(CONFIG_FIT) && defined(CONFIG_CMD_FSFITXIMG)
+
+/* TODO: maybe make this configurable */
+#define CHUNK_SIZE		(1024*1024)
+#define MAX_LOAD_SIZE	(16*1024*1024)
+
+struct fs_file {
+	const char *dev;
+	const char *part;
+	const char *name;
+};
+
+static int fs_load_chunk(struct fs_file *file, ulong addr, loff_t offset, loff_t len)
+{
+	int ret = 0;
+	loff_t len_read;
+
+	debug("Reading %lld bytes from offset 0x%llx to 0x%lx\n", len, offset, addr);
+
+	if (fs_set_blk_dev(file->dev, file->part, FS_TYPE_ANY))
+		return -1;
+
+	ret = fs_read(file->name, addr, offset, len, &len_read);
+
+	if (ret < 0) {
+		printf("Read failed: %d", ret);
+		return ret;
+	}
+
+	return len_read;
+}
+
+static int fdt_err_retry(int err)
+{
+	return
+		(err == -FDT_ERR_TRUNCATED) ||
+		(err == -FDT_ERR_BADSTRUCTURE);
+}
+
+static int
+do_fsfitextract(cmd_tbl_t * cmdtp, int flag, int argc, char * const argv[])
+{
+	ulong		dest;
+	int		ret;
+	int		verify;
+	int		part = 0;
+	struct fs_file file;
+	const char	*uname;
+	void	*fit_hdr;
+	const void	*fit_data;
+	size_t	fit_len;
+	int		noffset = 0,
+			src_off, fit_off = 0;
+	uint8_t		comp;
+	unsigned long time;
+
+	if (argc != 6)
+		return CMD_RET_USAGE;
+
+	verify = getenv_yesno("verify");
+
+	file.dev = argv[1];
+	file.part = argv[2];
+	file.name = argv[3];
+	uname = argv[4];
+	dest = simple_strtoul(argv[5], NULL, 16);
+	
+	time = get_timer(0);
+
+	/* Load FIT from the filesystem incrementally to avoid reading data we
+	 * won't need. If requested subimage contained in the beginning of file,
+	 * this can make a huge speedup.
+	 *
+	 * First, read just the header
+	 */
+	printf("## Loading FIT header to 0x%08lx ...\n", dest);
+	ret = fs_load_chunk(&file, dest, 0, sizeof(struct fdt_header));
+	if (ret < 0)
+		return -1;
+
+	fit_hdr = map_sysmem(dest, sizeof(struct fdt_header));
+	ret = fdt_check_header(fit_hdr);
+	if (ret) {
+		printf("Bad FIT header: %d\n", ret);
+		return 1;
+	}
+	
+	fit_off += sizeof(struct fdt_header);
+
+	/* Read "strings" block to resolve all the node names, put it after the
+	 * header and adjust offset appropriately
+	 */
+	printf("## Loading FIT strings to 0x%08lx ...\n", dest + fit_off);
+	ret = fs_load_chunk(&file, dest + fit_off,
+			fdt_off_dt_strings(fit_hdr),
+			fdt_size_dt_strings(fit_hdr));
+	if (ret < 0)
+		return 1;
+
+	fdt_set_off_dt_strings(fit_hdr, fit_off);
+	fit_off += fdt_size_dt_strings(fit_hdr);
+
+	/* Align to 4 bytes */
+	fit_off &= ~0x3;
+	fit_off += 4;
+
+	/* Save offset of "struct" block in file and set the real in-memory offset */
+	src_off = fdt_off_dt_struct(fit_hdr);
+	fdt_set_off_dt_struct(fit_hdr, fit_off);
+
+	/* Now, start to load struct in CHUNK_SIZE chunks, and parse it@the same
+	 * time. When libfdt notices that the data is truncated or broken, read next
+	 * chunk from the filesystem.
+	 */
+	printf("## Loading FIT struct to 0x%08lx ...\n", dest + fit_off);
+	do {
+		ret = fs_load_chunk(&file, dest + fit_off,
+				src_off, CHUNK_SIZE);
+		if (ret < 0)
+			return 1;
+		fit_off += ret;
+		src_off += ret;
+
+		ret = fit_image_get_node(fit_hdr, uname);
+		if (ret > 0) {
+			noffset = ret;
+			break;
+		}
+
+		debug("fit_image_get_node returned %d\n", ret);
+		if (!fdt_err_retry(ret)) {
+			printf("Error while processing FIT: %d\n", ret);
+			return 1;
+		}
+	} while (src_off < MAX_LOAD_SIZE);
+
+	if (src_off > MAX_LOAD_SIZE) {
+		printf("Requested subimage not found in first 0x%x bytes of FIT\n",
+				MAX_LOAD_SIZE);
+		return 1;
+	}
+
+	debug("found image node at %08x\n", noffset);
+
+	do {
+		/* get subimage fit_data address and length */
+		ret = fit_image_get_data(fit_hdr, noffset,
+				&fit_data, &fit_len);
+		if (!ret)
+			break;
+
+		if (!fdt_err_retry(fit_len)) {
+			puts("Could not find subimage fit_data\n");
+			return 1;
+		}
+
+		ret = fs_load_chunk(&file, dest + fit_off,
+				src_off, CHUNK_SIZE);
+		if (ret < 0)
+			return 1;
+		fit_off += ret;
+		src_off += ret;
+	} while (src_off < MAX_LOAD_SIZE);
+
+	debug("fit_data@%p, fit_len 0x%lx\n", fit_data, fit_len);
+
+	/* Now we know the length of data, so read it out */
+	while (fit_data + fit_len > fit_hdr + fit_off) {
+		ret = fs_load_chunk(&file, dest + fit_off,
+				src_off, CHUNK_SIZE);
+		if (ret < 0)
+			return 1;
+		fit_off += ret;
+		src_off += ret;
+	}
+
+	time = get_timer(time);
+
+	printf("0x%x bytes read in %lu ms", fit_off, time);
+	if (time > 0) {
+		puts(" (");
+		print_size(lldiv(fit_off, time) * 1000, "/s");
+		puts(")");
+	}
+	puts("\n");
+
+	fit_image_print(fit_hdr, noffset, "   ");
+
+	if (fit_image_get_comp(fit_hdr, noffset, &comp)) {
+		puts("Could not find FIT subimage compression type\n");
+		return 1;
+	}
+
+	/* verify integrity */
+	if (verify) {
+		if (!fit_image_verify(fit_hdr, noffset)) {
+			puts("Bad fit_data Hash\n");
+			return 1;
+		}
+	}
+
+	if (!decompress_data(dest, (ulong)fit_data, (ulong)fit_len, comp, part))
+		return 1;
+
+	flush_cache(dest, fit_len);
+
+	setenv_hex("filesize", fit_len);
+
+	return 0;
+}
+
+U_BOOT_CMD(
+	fsfitxtract, 7, 1, do_fsfitextract,
+	"extract a part of a FIT stored on a filesystem",
+	"<interface> [<dev[:part]> <filename> <uname> <addr>\n"
+	"    - Extract subimage <uname> from FIT file <filename> from\n"
+	"       partition <part> on device type <interface> instance <dev>\n"
+	"       to address <addr> in memory"
+);
+#endif
-- 
2.12.2

[U-Boot] [PATCH 2/2] cmd: fsfitxtract: Extract a part of a FIT multi-image stored on a filesystem

[Tom Rini]

On Sat, May 20, 2017 at 09:50:20AM +0300, Alexey Ignatov wrote:

> Sometimes we interested only in one single small subimage from big multipart
> FIT. This command tries to avoid reading out the whole FIT because of memory
> or time considerations.
> Since we don't have mmap() in U-Boot, this is done by reading the file in
> small chunks and trying to parse FIT until requested image is found or
> total read size limit exceeded.
> 
> Signed-off-by: Alexey Ignatov <lexszero@gmail.com>
> ---
>  cmd/Kconfig |   7 ++
>  cmd/ximg.c  | 226 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 233 insertions(+)

We need to add an entry for cmd/Makefile as CMD_XIMG and CMD_FSFITXIMG
do not depend on eachother.  I also think you need to guard the CMD_XIMG
specific code in cmd/ximg.c when adding this for when the old command is
disabled and the new one is enabled, thanks!

-- 
Tom
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 819 bytes
Desc: Digital signature
URL: <http://lists.denx.de/pipermail/u-boot/attachments/20170522/ab68d619/attachment.sig>

[U-Boot] [PATCH 1/2] cmd/imxtract: Move decompression part to separate function

[Tom Rini]

On Sat, May 20, 2017 at 09:50:19AM +0300, Alexey Ignatov wrote:

> Signed-off-by: Alexey Ignatov <lexszero@gmail.com>

Reviewed-by: Tom Rini <trini@konsulko.com>

-- 
Tom
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 819 bytes
Desc: Digital signature
URL: <http://lists.denx.de/pipermail/u-boot/attachments/20170522/1cf95c25/attachment.sig>