From: Richard Purdie <richard.purdie@linuxfoundation.org>
To: openembedded-core <openembedded-core@lists.openembedded.org>
Subject: [PATCH] image_types: Refactor compression code into a generic compression solution
Date: Mon, 27 Feb 2012 12:43:49 +0000 [thread overview]
Message-ID: <1330346629.4593.20.camel@ted> (raw)
The current approach of adding each different compressed image type doesn't
scale. This patch changes the code so compressed images for each form are
automatically available using the form <type>.<compression type> in
IMAGE_FSTYPES. This doesn't change any existing externally visible behaviour
and the image generation process becomes more efficient as a result too.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 54ab157..4db55ff 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -75,9 +75,7 @@ inherit image-${IMAGE_TYPE}
python () {
deps = d.getVarFlag('do_rootfs', 'depends') or ""
- for type in (d.getVar('IMAGE_FSTYPES', True) or "").split():
- for dep in ((d.getVar('IMAGE_DEPENDS_%s' % type) or "").split() or []):
- deps += " %s:do_populate_sysroot" % dep
+ deps += imagetypes_getdepends(d)
for dep in (d.getVar('EXTRA_IMAGEDEPENDS', True) or "").split():
deps += " %s:do_populate_sysroot" % dep
d.setVarFlag('do_rootfs', 'depends', deps)
diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass
index 3b26162..0fb68e7 100644
--- a/meta/classes/image_types.bbclass
+++ b/meta/classes/image_types.bbclass
@@ -2,7 +2,21 @@ def get_imagecmds(d):
cmds = "\n"
old_overrides = d.getVar('OVERRIDES', 0)
+ alltypes = d.getVar('IMAGE_FSTYPES', True).split()
types = d.getVar('IMAGE_FSTYPES', True).split()
+ ctypes = d.getVar('COMPRESSIONTYPES', True).split()
+ cimages = {}
+
+ # Filter out all the compressed images from types
+ for type in types:
+ for ctype in ctypes:
+ if type.endswith("." + ctype):
+ basetype = type.rsplit(".", 1)[0]
+ types[types.index(type)] = basetype
+ if type not in cimages:
+ cimages[basetype] = []
+ cimages[basetype].append(ctype)
+
# Live images will be processed via inheriting bbclass and
# does not get processed here.
# live images also depend on ext3 so ensure its present
@@ -11,11 +25,24 @@ def get_imagecmds(d):
types.append("ext3")
types.remove("live")
+ cmds += " rm -f ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.*"
for type in types:
+ ccmd = []
+ subimages = []
localdata = bb.data.createCopy(d)
localdata.setVar('OVERRIDES', '%s:%s' % (type, old_overrides))
bb.data.update_data(localdata)
localdata.setVar('type', type)
+ if type in cimages:
+ for ctype in cimages[type]:
+ ccmd.append("\t" + localdata.getVar("COMPRESS_CMD_" + ctype, True))
+ subimages.append(type + "." + ctype)
+ if type not in alltypes:
+ ccmd.append(localdata.expand("\trm ${IMAGE_NAME}.rootfs.${type}"))
+ else:
+ subimages.append(type)
+ localdata.setVar('ccmd', "\n".join(ccmd))
+ localdata.setVar('subimages', " ".join(subimages))
cmd = localdata.getVar("IMAGE_CMD", True)
localdata.setVar('cmd', cmd)
cmds += localdata.getVar("runimagecmd", True)
@@ -25,10 +52,36 @@ runimagecmd () {
# Image generation code for image type ${type}
ROOTFS_SIZE=`du -ks ${IMAGE_ROOTFS}|awk '{base_size = ($1 * ${IMAGE_OVERHEAD_FACTOR}); OFMT = "%.0f" ; print ((base_size > ${IMAGE_ROOTFS_SIZE} ? base_size : ${IMAGE_ROOTFS_SIZE}) + ${IMAGE_ROOTFS_EXTRA_SPACE}) }'`
${cmd}
+ # Now create the needed compressed versions
cd ${DEPLOY_DIR_IMAGE}/
- rm -f ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.${type}
- ln -s ${IMAGE_NAME}.rootfs.${type} ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.${type}
-}
+ ${ccmd}
+ # And create the symlinks
+ for type in ${subimages}; do
+ ln -s ${IMAGE_NAME}.rootfs.$type ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.$type
+ done
+}
+
+def imagetypes_getdepends(d):
+ def adddep(depstr, deps):
+ for i in (depstr or "").split():
+ if i not in deps:
+ deps.append(i)
+
+ deps = []
+ ctypes = d.getVar('COMPRESSIONTYPES', True).split()
+ for type in (d.getVar('IMAGE_FSTYPES', True) or "").split():
+ basetype = type
+ for ctype in ctypes:
+ if type.endswith("." + ctype):
+ basetype = type.rsplit(".", 1)[0]
+ adddep(d.getVar("COMPRESS_DEPENDS_%s" % ctype, True), deps)
+ break
+ adddep(d.getVar('IMAGE_DEPENDS_%s' % basetype, True) , deps)
+
+ depstr = ""
+ for dep in deps:
+ depstr += " " + dep + ":do_populate_sysroot"
+ return depstr
XZ_COMPRESSION_LEVEL ?= "-e -9"
@@ -46,40 +99,11 @@ IMAGE_CMD_ext2 () {
mv ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}/${IMAGE_NAME}.rootfs.ext2 ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.ext2
rmdir ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}
}
-IMAGE_CMD_ext2.gz () {
- rm -rf ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN} && mkdir ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}
- genext2fs -b $ROOTFS_SIZE -i 4096 -d ${IMAGE_ROOTFS} ${EXTRA_IMAGECMD} ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}/${IMAGE_NAME}.rootfs.ext2
- gzip -f -9 ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}/${IMAGE_NAME}.rootfs.ext2
- mv ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}/${IMAGE_NAME}.rootfs.ext2.gz ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.ext2.gz
- rmdir ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}
-}
-IMAGE_CMD_ext2.bz2 () {
- rm -rf ${DEPLOY_DIR_IMAGE}/tmp.gz && mkdir ${DEPLOY_DIR_IMAGE}/tmp.gz
- genext2fs -b $ROOTFS_SIZE -i 4096 -d ${IMAGE_ROOTFS} ${EXTRA_IMAGECMD} ${DEPLOY_DIR_IMAGE}/tmp.gz/${IMAGE_NAME}.rootfs.ext2
- bzip2 -f -9 ${DEPLOY_DIR_IMAGE}/tmp.gz/${IMAGE_NAME}.rootfs.ext2
- mv ${DEPLOY_DIR_IMAGE}/tmp.gz/${IMAGE_NAME}.rootfs.ext2.bz2 ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.ext2.bz2
- rmdir ${DEPLOY_DIR_IMAGE}/tmp.gz
-}
-IMAGE_CMD_ext2.lzma () {
- rm -rf ${DEPLOY_DIR_IMAGE}/tmp.gz && mkdir ${DEPLOY_DIR_IMAGE}/tmp.gz
- genext2fs -b $ROOTFS_SIZE -i 4096 -d ${IMAGE_ROOTFS} ${EXTRA_IMAGECMD} ${DEPLOY_DIR_IMAGE}/tmp.gz/${IMAGE_NAME}.rootfs.ext2
- lzma -f -7 ${DEPLOY_DIR_IMAGE}/tmp.gz/${IMAGE_NAME}.rootfs.ext2
- mv ${DEPLOY_DIR_IMAGE}/tmp.gz/${IMAGE_NAME}.rootfs.ext2.lzma ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.ext2.lzma
- rmdir ${DEPLOY_DIR_IMAGE}/tmp.gz
-}
IMAGE_CMD_ext3 () {
genext2fs -b $ROOTFS_SIZE -i 4096 -d ${IMAGE_ROOTFS} ${EXTRA_IMAGECMD} ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.ext3
tune2fs -j ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.ext3
}
-IMAGE_CMD_ext3.gz () {
- rm -rf ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN} && mkdir ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}
- genext2fs -b $ROOTFS_SIZE -i 4096 -d ${IMAGE_ROOTFS} ${EXTRA_IMAGECMD} ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}/${IMAGE_NAME}.rootfs.ext3
- tune2fs -j ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}/${IMAGE_NAME}.rootfs.ext3
- gzip -f -9 ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}/${IMAGE_NAME}.rootfs.ext3
- mv ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}/${IMAGE_NAME}.rootfs.ext3.gz ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.ext3.gz
- rmdir ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}
-}
oe_mkext4fs () {
genext2fs -b $ROOTFS_SIZE -i 4096 -d ${IMAGE_ROOTFS} ${EXTRA_IMAGECMD} $1
@@ -97,13 +121,6 @@ oe_mkext4fs () {
IMAGE_CMD_ext4 () {
oe_mkext4fs ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.ext4
}
-IMAGE_CMD_ext4.gz () {
- rm -rf ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN} && mkdir ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}
- oe_mkext4fs ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}/${IMAGE_NAME}.rootfs.ext4
- gzip -f -9 ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}/${IMAGE_NAME}.rootfs.ext4
- mv ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}/${IMAGE_NAME}.rootfs.ext4.gz ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.ext4.gz
- rmdir ${DEPLOY_DIR_IMAGE}/tmp.gz-${PN}
-}
IMAGE_CMD_btrfs () {
mkfs.btrfs -b `expr ${ROOTFS_SIZE} \* 1024` ${EXTRA_IMAGECMD} -r ${IMAGE_ROOTFS} ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.btrfs
@@ -112,9 +129,6 @@ IMAGE_CMD_btrfs () {
IMAGE_CMD_squashfs = "mksquashfs ${IMAGE_ROOTFS} ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.squashfs ${EXTRA_IMAGECMD} -noappend"
IMAGE_CMD_squashfs-lzma = "mksquashfs-lzma ${IMAGE_ROOTFS} ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.squashfs-lzma ${EXTRA_IMAGECMD} -noappend"
IMAGE_CMD_tar = "cd ${IMAGE_ROOTFS} && tar -cvf ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.tar ."
-IMAGE_CMD_tar.gz = "cd ${IMAGE_ROOTFS} && tar -zcvf ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.tar.gz ."
-IMAGE_CMD_tar.bz2 = "cd ${IMAGE_ROOTFS} && tar -jcvf ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.tar.bz2 ."
-IMAGE_CMD_tar.xz = "cd ${IMAGE_ROOTFS} && tar --xz -cvf ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.tar.xz ."
CPIO_TOUCH_INIT () {
if [ ! -L ${IMAGE_ROOTFS}/init ]
@@ -126,18 +140,6 @@ IMAGE_CMD_cpio () {
${CPIO_TOUCH_INIT}
cd ${IMAGE_ROOTFS} && (find . | cpio -o -H newc >${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.cpio)
}
-IMAGE_CMD_cpio.gz () {
- ${CPIO_TOUCH_INIT}
- cd ${IMAGE_ROOTFS} && (find . | cpio -o -H newc | gzip -c -9 >${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.cpio.gz)
-}
-IMAGE_CMD_cpio.xz () {
- ${CPIO_TOUCH_INIT}
- cd ${IMAGE_ROOTFS} && (find . | cpio -o -H newc | xz -c ${XZ_COMPRESSION_LEVEL} --check=${XZ_INTEGRITY_CHECK} > ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.cpio.xz) ${EXTRA_IMAGECMD}
-}
-IMAGE_CMD_cpio.lzma () {
- ${CPIO_TOUCH_INIT}
- cd ${IMAGE_ROOTFS} && (find . | cpio -o -H newc | xz --format=lzma -c ${XZ_COMPRESSION_LEVEL} --check=${XZ_INTEGRITY_CHECK} >${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.cpio.lzma) ${EXTRA_IMAGECMD}
-}
UBI_VOLNAME ?= "${MACHINE}-rootfs"
@@ -159,9 +161,7 @@ EXTRA_IMAGECMD = ""
EXTRA_IMAGECMD_jffs2 ?= "--pad --little-endian --eraseblock=0x40000"
# Change these if you want default genext2fs behavior (i.e. create minimal inode number)
EXTRA_IMAGECMD_ext2 ?= "-i 8192"
-EXTRA_IMAGECMD_ext2.gz ?= "-i 8192"
EXTRA_IMAGECMD_ext3 ?= "-i 8192"
-EXTRA_IMAGECMD_ext3.gz ?= "-i 8192"
EXTRA_IMAGECMD_btrfs ?= ""
IMAGE_DEPENDS = ""
@@ -169,22 +169,25 @@ IMAGE_DEPENDS_jffs2 = "mtd-utils-native"
IMAGE_DEPENDS_sum.jffs2 = "mtd-utils-native"
IMAGE_DEPENDS_cramfs = "cramfs-native"
IMAGE_DEPENDS_ext2 = "genext2fs-native"
-IMAGE_DEPENDS_ext2.gz = "genext2fs-native"
-IMAGE_DEPENDS_ext2.bz2 = "genext2fs-native"
-IMAGE_DEPENDS_ext2.lzma = "genext2fs-native xz-native"
IMAGE_DEPENDS_ext3 = "genext2fs-native e2fsprogs-native"
-IMAGE_DEPENDS_ext3.gz = "genext2fs-native e2fsprogs-native"
IMAGE_DEPENDS_ext4 = "genext2fs-native e2fsprogs-native"
-IMAGE_DEPENDS_ext4.gz = "genext2fs-native e2fsprogs-native"
IMAGE_DEPENDS_btrfs = "btrfs-tools-native"
IMAGE_DEPENDS_squashfs = "squashfs-tools-native"
IMAGE_DEPENDS_squashfs-lzma = "squashfs-lzma-tools-native"
-IMAGE_DEPENDS_tar.xz = "tar-native xz-native"
-IMAGE_DEPENDS_cpio.lzma = "xz-native"
-IMAGE_DEPENDS_cpio.xz = "xz-native"
IMAGE_DEPENDS_ubi = "mtd-utils-native"
IMAGE_DEPENDS_ubifs = "mtd-utils-native"
IMAGE_DEPENDS_vmdk = "qemu-native"
# This variable is available to request which values are suitable for IMAGE_FSTYPES
IMAGE_TYPES = "jffs2 sum.jffs2 cramfs ext2 ext2.gz ext2.bz2 ext3 ext3.gz ext2.lzma btrfs live squashfs squashfs-lzma ubi tar tar.gz tar.bz2 tar.xz cpio cpio.gz cpio.xz cpio.lzma vmdk"
+
+COMPRESSIONTYPES = "gz bz2 lzma xy"
+COMPRESS_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}.rootfs.${type}"
+COMPRESS_CMD_gz = "gzip -f -9 -c ${IMAGE_NAME}.rootfs.${type} > ${IMAGE_NAME}.rootfs.${type}.gz"
+COMPRESS_CMD_bz2 = "bzip2 -k ${IMAGE_NAME}.rootfs.${type}"
+COMPRESS_CMD_xy = "xz -k -c ${XZ_COMPRESSION_LEVEL} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}.rootfs.${type}"
+COMPRESS_DEPENDS_lzma = "xz-native"
+COMPRESS_DEPENDS_gz = ""
+COMPRESS_DEPENDS_bz2 = ""
+COMPRESS_DEPENDS_xy = "xz-native"
+
next reply other threads:[~2012-02-27 12:52 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2012-02-27 12:43 Richard Purdie [this message]
2012-02-27 14:01 ` [PATCH] image_types: Refactor compression code into a generic compression solution Koen Kooi
2012-02-27 14:20 ` Andrea Adami
2012-02-27 14:22 ` Andrea Adami
2012-02-28 20:19 ` Koen Kooi
2012-02-28 23:41 ` Andrea Adami
2012-02-28 23:48 ` Otavio Salvador
2012-02-29 1:12 ` Andrea Adami
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1330346629.4593.20.camel@ted \
--to=richard.purdie@linuxfoundation.org \
--cc=openembedded-core@lists.openembedded.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox