[RFC - WIP PATCH 0/1] image.bbclass: create binary pkg for image recipe

[RFC - WIP PATCH 0/1] image.bbclass: create binary pkg for image recipe

[Robert Yang]

The following changes since commit 0ed5a13f5f9478e2c22f517fb3504b738422f85c:

  python3: fix builtins imports (2014-06-30 16:36:44 +0100)

are available in the git repository at:

  git://git.pokylinux.org/poky-contrib rbt/image_pkg
  http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=rbt/image_pkg

Robert Yang (1):
  image.bbclass: create binary pkg for image recipe

 meta/classes/image.bbclass                   |   91 +++++++++++++++++++++++---
 meta/recipes-core/meta/buildtools-tarball.bb |    2 +-
 2 files changed, 83 insertions(+), 10 deletions(-)

-- 
1.7.9.5

[RFC - WIP PATCH 1/1] image.bbclass: create binary pkg for image recipe

[Robert Yang]

* Benefits
  We can known the image's RDEPENDS outside the build environment, and
  can use a third part installer such as anaconda to install the packages.
  It's hard to get the ROOTFS_POSTPROCESS_COMMAND outside the build
  environment, now the shell function can be added to postinst, and will
  show a warning for the python function. We have two python functions
  atm: write_package_manifest and write_image_manifest, we don't need them
  in the binary pkg.

* Brief design:
  - Set the RDEPENDS and create the package as other regular recipes.

  - Translate the ROOTFS_POSTPROCESS_COMMAND to postinst
    do_install[postfuncs] += "get_rootfs_postprocess_command"

    The get_rootfs_postprocess_command() will emit the shell script to
    /usr/share/${PN}, and the pkg_postinst_${PN} will run it when do the
    install.

* Fixed:
  RDEPENDS -> RDEPENDS_${PN}, the similar to RRECOMMENDS

* Tested on rpm, dep and ipk

* TODO:
  - Create the binary pkg optionally rather than default ?

[YOCTO #6463]

Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
 meta/classes/image.bbclass                   |   91 +++++++++++++++++++++++---
 meta/recipes-core/meta/buildtools-tarball.bb |    2 +-
 2 files changed, 83 insertions(+), 10 deletions(-)

diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index a03b880..ad2bd2d 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -9,10 +9,15 @@ POPULATE_SDK_POST_TARGET_COMMAND += "rootfs_sysroot_relativelinks; "
 inherit gzipnative
 
 LICENSE = "MIT"
-PACKAGES = ""
+
+# Only build one package for the image recipe, for example:
+# core-image-minimal -> core-image-minimal.rpm
+# core-image-minimal-dev -> core-image-minimal-dev.rpm
+PACKAGES = "${PN}"
+
 DEPENDS += "${MLPREFIX}qemuwrapper-cross ${MLPREFIX}depmodwrapper-cross"
-RDEPENDS += "${PACKAGE_INSTALL} ${LINGUAS_INSTALL}"
-RRECOMMENDS += "${PACKAGE_INSTALL_ATTEMPTONLY}"
+RDEPENDS_${PN} += "${PACKAGE_INSTALL} ${LINGUAS_INSTALL}"
+RRECOMMENDS_${PN} += "${PACKAGE_INSTALL_ATTEMPTONLY}"
 
 INHIBIT_DEFAULT_DEPS = "1"
 
@@ -385,18 +390,86 @@ rootfs_sysroot_relativelinks () {
 	sysroot-relativelinks.py ${SDK_OUTPUT}/${SDKTARGETSYSROOT}
 }
 
+do_install[postfuncs] += "get_rootfs_postprocess_command"
+
+# The target package doesn't need these commands
+SKIPPED_POSTCOMMAND_FROM_POSTINSTS ?= "write_package_manifest write_image_manifest \
+                                    license_create_manifest"
+
+python get_rootfs_postprocess_command() {
+    import subprocess
+
+    skip_funcs = d.getVar('SKIPPED_POSTCOMMAND_FROM_POSTINSTS', True)
+
+    installdir = os.path.join(d.getVar('D', True), \
+            d.getVar('datadir', True).lstrip('/'), d.getVar('PN', True), 'postinsts')
+    bb.utils.mkdirhier(installdir)
+
+    # The postinst list
+    postinst_list = os.path.join(installdir, 'postinst_list')
+
+    for f in d.getVar('ROOTFS_POSTPROCESS_COMMAND', True).split(';'):
+        func = f.strip()
+        if not func or func in skip_funcs:
+            continue
+
+        flags = d.getVarFlags(func)
+        ispython = flags.get('python')
+        if ispython:
+            bb.warn('%s is a python func, can not add it to postinst' % func)
+            continue
+
+        postinst = os.path.join(installdir, func)
+        with open(postinst, 'w') as script:
+            script.write("#!/bin/sh\n\n")
+            bb.data.emit_func(func, script, d)
+            # Run the function
+            script.write("%s\n" % func)
+
+        bb.note('Fix build path in %s' % postinst)
+        # Remove the IMAGE_ROOTFS prefix
+        fix_cmd = "sed -i -e 's:%s::g'" % d.getVar('IMAGE_ROOTFS', True)
+        # Remove the STAGING_DIR_TARGET prefix
+        fix_cmd += " -e 's:%s::g'" % d.getVar('STAGING_DIR_TARGET', True)
+        # Comment out the STAGING_DIR_NATIVE related lines
+        fix_cmd += " -e 's:\(^.*%s\):#\\1:'" % d.getVar('STAGING_DIR_NATIVE', True)
+        # Comment out the PSEUDO related lines
+        fix_cmd += " -e 's:\(^export .*PSEUDO\):#\\1:'"
+        # Comment out the username related lines
+        fix_cmd += " -e 's:\(^export .*%s\):#\\1:'" % d.getVar('USER', True)
+
+        fix_cmd += " %s" % postinst
+        subprocess.call(fix_cmd, shell=True)
+
+        os.chmod(postinst, 0755)
+
+        with open(postinst_list, 'a') as plist:
+            plist.write("%s\n" % postinst.replace(d.getVar('D', True), ''))
+}
+
+pkg_postinst_${PN} () {
+    listfile = "${datadir}/${PN}/postinst_list"
+    if [ -s $listfile ]; then
+        for script in `cat $listfile`; do
+            echo "Running $script..."
+            /bin/sh $script
+            if [ $? -ne 0 ]; then
+                # Allow the failure rather than re-install the package
+                # since the script can be manualy run
+                echo "ERROR: failed to run $script" >&2
+                echo "ERROR: please fix $script and manually run it" >&2
+                true
+            fi
+        done
+    fi
+}
+
 do_fetch[noexec] = "1"
 do_unpack[noexec] = "1"
 do_patch[noexec] = "1"
 do_configure[noexec] = "1"
 do_compile[noexec] = "1"
-do_install[noexec] = "1"
 do_populate_sysroot[noexec] = "1"
-do_package[noexec] = "1"
-do_packagedata[noexec] = "1"
-do_package_write_ipk[noexec] = "1"
-do_package_write_deb[noexec] = "1"
-do_package_write_rpm[noexec] = "1"
 
 addtask rootfs before do_build
 # Allow the kernel to be repacked with the initramfs and boot image file as a single file
diff --git a/meta/recipes-core/meta/buildtools-tarball.bb b/meta/recipes-core/meta/buildtools-tarball.bb
index 62e1e0b..7501525 100644
--- a/meta/recipes-core/meta/buildtools-tarball.bb
+++ b/meta/recipes-core/meta/buildtools-tarball.bb
@@ -45,7 +45,7 @@ TOOLCHAIN_HOST_TASK ?= "\
 
 TOOLCHAIN_OUTPUTNAME ?= "${SDK_NAME}-buildtools-nativesdk-standalone-${DISTRO_VERSION}"
 
-RDEPENDS = "${TOOLCHAIN_HOST_TASK}"
+RDEPENDS_${PN} = "${TOOLCHAIN_HOST_TASK}"
 
 EXCLUDE_FROM_WORLD = "1"
 
-- 
1.7.9.5

Re: [RFC - WIP PATCH 1/1] image.bbclass: create binary pkg for image recipe

[David Nyström]

On 2014-07-02 11:25, Robert Yang wrote:
> * Benefits
>    We can known the image's RDEPENDS outside the build environment, and
>    can use a third part installer such as anaconda to install the packages.
>    It's hard to get the ROOTFS_POSTPROCESS_COMMAND outside the build
>    environment, now the shell function can be added to postinst, and will
>    show a warning for the python function. We have two python functions
>    atm: write_package_manifest and write_image_manifest, we don't need them
>    in the binary pkg.
>
> * Brief design:
>    - Set the RDEPENDS and create the package as other regular recipes.
>
>    - Translate the ROOTFS_POSTPROCESS_COMMAND to postinst
>      do_install[postfuncs] += "get_rootfs_postprocess_command"
>
>      The get_rootfs_postprocess_command() will emit the shell script to
>      /usr/share/${PN}, and the pkg_postinst_${PN} will run it when do the
>      install.
>
> * Fixed:
>    RDEPENDS -> RDEPENDS_${PN}, the similar to RRECOMMENDS
>
> * Tested on rpm, dep and ipk
>
> * TODO:
>    - Create the binary pkg optionally rather than default ?
> [YOCTO #6463]

Interesting approach.

Image binaries were created before 
poky-commit:6706c7bdd2de6e0e447d90062e74a718a8d31778,
but this feature was removed.

A similiar approach has been rejected before:
http://lists.openembedded.org/pipermail/openembedded-core/2013-December/087474.html

I ended up with a fork of image.bbclass for images I needed available as
meta-packages.

> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
> ---
>   meta/classes/image.bbclass                   |   91 +++++++++++++++++++++++---
>   meta/recipes-core/meta/buildtools-tarball.bb |    2 +-
>   2 files changed, 83 insertions(+), 10 deletions(-)

<SNIP>

> +
> +        bb.note('Fix build path in %s' % postinst)
> +        # Remove the IMAGE_ROOTFS prefix
> +        fix_cmd = "sed -i -e 's:%s::g'" % d.getVar('IMAGE_ROOTFS', True)
> +        # Remove the STAGING_DIR_TARGET prefix
> +        fix_cmd += " -e 's:%s::g'" % d.getVar('STAGING_DIR_TARGET', True)
> +        # Comment out the STAGING_DIR_NATIVE related lines
> +        fix_cmd += " -e 's:\(^.*%s\):#\\1:'" % d.getVar('STAGING_DIR_NATIVE', True)
> +        # Comment out the PSEUDO related lines
> +        fix_cmd += " -e 's:\(^export .*PSEUDO\):#\\1:'"
> +        # Comment out the username related lines
> +        fix_cmd += " -e 's:\(^export .*%s\):#\\1:'" % d.getVar('USER', True)
> +

Why are you commenting out these vars ?
Perhaps it would be better to fix the ROOTFS_POSTPROCESS_COMMANDs, and 
rely on a sane environment ?


> +        fix_cmd += " %s" % postinst
> +        subprocess.call(fix_cmd, shell=True)
> +
> +        os.chmod(postinst, 0755)
> +
> +        with open(postinst_list, 'a') as plist:
> +            plist.write("%s\n" % postinst.replace(d.getVar('D', True), ''))
> +}
> +
> +pkg_postinst_${PN} () {

You need to check if your running on target here.

> +    listfile = "${datadir}/${PN}/postinst_list"
> +    if [ -s $listfile ]; then
> +        for script in `cat $listfile`; do
> +            echo "Running $script..."
> +            /bin/sh $script
> +            if [ $? -ne 0 ]; then
> +                # Allow the failure rather than re-install the package
> +                # since the script can be manualy run
> +                echo "ERROR: failed to run $script" >&2
> +                echo "ERROR: please fix $script and manually run it" >&2

If the ROOTFS_POSTPROCESS_COMMANDs do follow the same rules as other 
postinstalls, you could potentially defer executing to first boot when 
errors occur. (run-postinst)

> +                true
> +            fi
> +        done
> +    fi
> +}
> +
>   do_fetch[noexec] = "1"
>   do_unpack[noexec] = "1"
>   do_patch[noexec] = "1"
>   do_configure[noexec] = "1"
>   do_compile[noexec] = "1"
> -do_install[noexec] = "1"
>   do_populate_sysroot[noexec] = "1"
> -do_package[noexec] = "1"
> -do_packagedata[noexec] = "1"
> -do_package_write_ipk[noexec] = "1"
> -do_package_write_deb[noexec] = "1"
> -do_package_write_rpm[noexec] = "1"
>
>   addtask rootfs before do_build
>   # Allow the kernel to be repacked with the initramfs and boot image file as a single file
> diff --git a/meta/recipes-core/meta/buildtools-tarball.bb b/meta/recipes-core/meta/buildtools-tarball.bb
> index 62e1e0b..7501525 100644
> --- a/meta/recipes-core/meta/buildtools-tarball.bb
> +++ b/meta/recipes-core/meta/buildtools-tarball.bb
> @@ -45,7 +45,7 @@ TOOLCHAIN_HOST_TASK ?= "\
>
>   TOOLCHAIN_OUTPUTNAME ?= "${SDK_NAME}-buildtools-nativesdk-standalone-${DISTRO_VERSION}"
>
> -RDEPENDS = "${TOOLCHAIN_HOST_TASK}"
> +RDEPENDS_${PN} = "${TOOLCHAIN_HOST_TASK}"
>
>   EXCLUDE_FROM_WORLD = "1"
>
>

Re: [RFC - WIP PATCH 1/1] image.bbclass: create binary pkg for image recipe

[Robert Yang]

On 07/03/2014 04:19 PM, David Nyström wrote:
> On 2014-07-02 11:25, Robert Yang wrote:
>> * Benefits
>>    We can known the image's RDEPENDS outside the build environment, and
>>    can use a third part installer such as anaconda to install the packages.
>>    It's hard to get the ROOTFS_POSTPROCESS_COMMAND outside the build
>>    environment, now the shell function can be added to postinst, and will
>>    show a warning for the python function. We have two python functions
>>    atm: write_package_manifest and write_image_manifest, we don't need them
>>    in the binary pkg.
>>
>> * Brief design:
>>    - Set the RDEPENDS and create the package as other regular recipes.
>>
>>    - Translate the ROOTFS_POSTPROCESS_COMMAND to postinst
>>      do_install[postfuncs] += "get_rootfs_postprocess_command"
>>
>>      The get_rootfs_postprocess_command() will emit the shell script to
>>      /usr/share/${PN}, and the pkg_postinst_${PN} will run it when do the
>>      install.
>>
>> * Fixed:
>>    RDEPENDS -> RDEPENDS_${PN}, the similar to RRECOMMENDS
>>
>> * Tested on rpm, dep and ipk
>>
>> * TODO:
>>    - Create the binary pkg optionally rather than default ?
>> [YOCTO #6463]
>
> Interesting approach.
>
> Image binaries were created before
> poky-commit:6706c7bdd2de6e0e447d90062e74a718a8d31778,
> but this feature was removed.
>
> A similiar approach has been rejected before:
> http://lists.openembedded.org/pipermail/openembedded-core/2013-December/087474.html
>
> I ended up with a fork of image.bbclass for images I needed available as
> meta-packages.
>

Thanks for the comment, I didn't know that you had done such things before.
I'd like to add another bbclass to do this if this isn't accepted.

>> Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
>> ---
>>   meta/classes/image.bbclass                   |   91 +++++++++++++++++++++++---
>>   meta/recipes-core/meta/buildtools-tarball.bb |    2 +-
>>   2 files changed, 83 insertions(+), 10 deletions(-)
>
> <SNIP>
>
>> +
>> +        bb.note('Fix build path in %s' % postinst)
>> +        # Remove the IMAGE_ROOTFS prefix
>> +        fix_cmd = "sed -i -e 's:%s::g'" % d.getVar('IMAGE_ROOTFS', True)
>> +        # Remove the STAGING_DIR_TARGET prefix
>> +        fix_cmd += " -e 's:%s::g'" % d.getVar('STAGING_DIR_TARGET', True)
>> +        # Comment out the STAGING_DIR_NATIVE related lines
>> +        fix_cmd += " -e 's:\(^.*%s\):#\\1:'" % d.getVar('STAGING_DIR_NATIVE',
>> True)
>> +        # Comment out the PSEUDO related lines
>> +        fix_cmd += " -e 's:\(^export .*PSEUDO\):#\\1:'"
>> +        # Comment out the username related lines
>> +        fix_cmd += " -e 's:\(^export .*%s\):#\\1:'" % d.getVar('USER', True)
>> +
>
> Why are you commenting out these vars ?
> Perhaps it would be better to fix the ROOTFS_POSTPROCESS_COMMANDs, and rely on a
> sane environment ?

They are not available when outside the build env, for example, when you run the
smart install core-image-minimal.rpm on the target.

>
>
>> +        fix_cmd += " %s" % postinst
>> +        subprocess.call(fix_cmd, shell=True)
>> +
>> +        os.chmod(postinst, 0755)
>> +
>> +        with open(postinst_list, 'a') as plist:
>> +            plist.write("%s\n" % postinst.replace(d.getVar('D', True), ''))
>> +}
>> +
>> +pkg_postinst_${PN} () {
>
> You need to check if your running on target here.
>
>> +    listfile = "${datadir}/${PN}/postinst_list"
>> +    if [ -s $listfile ]; then
>> +        for script in `cat $listfile`; do
>> +            echo "Running $script..."
>> +            /bin/sh $script
>> +            if [ $? -ne 0 ]; then
>> +                # Allow the failure rather than re-install the package
>> +                # since the script can be manualy run
>> +                echo "ERROR: failed to run $script" >&2
>> +                echo "ERROR: please fix $script and manually run it" >&2
>
> If the ROOTFS_POSTPROCESS_COMMANDs do follow the same rules as other
> postinstalls, you could potentially defer executing to first boot when errors
> occur. (run-postinst)

Yes, we need fix here a little.

// Robert

>
>> +                true
>> +            fi
>> +        done
>> +    fi
>> +}
>> +
>>   do_fetch[noexec] = "1"
>>   do_unpack[noexec] = "1"
>>   do_patch[noexec] = "1"
>>   do_configure[noexec] = "1"
>>   do_compile[noexec] = "1"
>> -do_install[noexec] = "1"
>>   do_populate_sysroot[noexec] = "1"
>> -do_package[noexec] = "1"
>> -do_packagedata[noexec] = "1"
>> -do_package_write_ipk[noexec] = "1"
>> -do_package_write_deb[noexec] = "1"
>> -do_package_write_rpm[noexec] = "1"
>>
>>   addtask rootfs before do_build
>>   # Allow the kernel to be repacked with the initramfs and boot image file as
>> a single file
>> diff --git a/meta/recipes-core/meta/buildtools-tarball.bb
>> b/meta/recipes-core/meta/buildtools-tarball.bb
>> index 62e1e0b..7501525 100644
>> --- a/meta/recipes-core/meta/buildtools-tarball.bb
>> +++ b/meta/recipes-core/meta/buildtools-tarball.bb
>> @@ -45,7 +45,7 @@ TOOLCHAIN_HOST_TASK ?= "\
>>
>>   TOOLCHAIN_OUTPUTNAME ?=
>> "${SDK_NAME}-buildtools-nativesdk-standalone-${DISTRO_VERSION}"
>>
>> -RDEPENDS = "${TOOLCHAIN_HOST_TASK}"
>> +RDEPENDS_${PN} = "${TOOLCHAIN_HOST_TASK}"
>>
>>   EXCLUDE_FROM_WORLD = "1"
>>
>>
>
>
>