* [PATCH 1/5] manifest.py: add create_full for OpkgManifest class
2014-02-20 7:06 [PATCH V2 0/5] manifest.py/package_manager.py/rootfs.py: support ipk incremental image generation Hongxu Jia
@ 2014-02-20 7:06 ` Hongxu Jia
2014-02-20 15:21 ` Laurentiu Palcu
2014-02-20 7:06 ` [PATCH 2/5] package_manager.py: support ipk incremental image generation Hongxu Jia
` (3 subsequent siblings)
4 siblings, 1 reply; 10+ messages in thread
From: Hongxu Jia @ 2014-02-20 7:06 UTC (permalink / raw)
To: openembedded-core; +Cc: saul.wold
The function create_full creates the manifest after the package in
initial manifest has been dummy installed. It lists all *to be
installed* packages. There is no real installation, just a test.
[YOCTO #1894]
Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
---
meta/lib/oe/manifest.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 64 insertions(+), 1 deletion(-)
diff --git a/meta/lib/oe/manifest.py b/meta/lib/oe/manifest.py
index a4bc04b..6e40a9f 100644
--- a/meta/lib/oe/manifest.py
+++ b/meta/lib/oe/manifest.py
@@ -1,6 +1,8 @@
from abc import ABCMeta, abstractmethod
+from oe.package_manager import *
import os
import re
+import bb
class Manifest(object):
@@ -69,6 +71,7 @@ class Manifest(object):
self.initial_manifest = os.path.join(self.manifest_dir, "%s_initial_manifest" % manifest_type)
self.final_manifest = os.path.join(self.manifest_dir, "%s_final_manifest" % manifest_type)
+ self.full_manifest = os.path.join(self.manifest_dir, "%s_full_manifest" % manifest_type)
# packages in the following vars will be split in 'must install' and
# 'multilib'
@@ -128,6 +131,15 @@ class Manifest(object):
pass
"""
+ This creates the manifest after the package in initial manifest has been
+ dummy installed. It lists all *to be installed* packages. There is no real
+ installation, just a test.
+ """
+ @abstractmethod
+ def create_full(self):
+ pass
+
+ """
The following function parses an initial manifest and returns a dictionary
object with the must install, attempt only, multilib and language packages.
"""
@@ -158,6 +170,22 @@ class Manifest(object):
return pkgs
+ '''
+ This following function parses a full manifest and return a list
+ object with packages.
+ '''
+ def parse_full_manifest(self):
+ installed_pkgs = list()
+ if not os.path.exists(self.full_manifest):
+ bb.note('full manifest not exist')
+ return installed_pkgs
+
+ with open(self.full_manifest, 'r') as manifest:
+ for pkg in manifest.read().split('\n'):
+ installed_pkgs.append(pkg.strip())
+
+ return installed_pkgs
+
class RpmManifest(Manifest):
"""
@@ -202,10 +230,12 @@ class RpmManifest(Manifest):
for pkg in pkgs[pkg_type].split():
manifest.write("%s,%s\n" % (pkg_type, pkg))
-
def create_final(self):
pass
+ def create_full(self):
+ pass
+
class OpkgManifest(Manifest):
"""
@@ -253,6 +283,36 @@ class OpkgManifest(Manifest):
def create_final(self):
pass
+ def create_full(self):
+ if not os.path.exists(self.initial_manifest):
+ self.create_initial()
+
+ initial_manifest = self.parse_initial_manifest()
+ pkgs_to_install = list()
+ for pkg_type in initial_manifest:
+ pkgs_to_install += initial_manifest[pkg_type]
+ if len(pkgs_to_install) == 0:
+ return
+
+ image_rootfs = self.d.expand("${T}/opkg")
+ opkg_conf = self.d.getVar("IPKGCONF_TARGET", True)
+ pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True)
+ pm = OpkgPM(self.d, image_rootfs, opkg_conf, pkg_archs)
+
+ pm.update()
+ pm.handle_bad_recommendations()
+ output = pm.dummy_install(pkgs_to_install)
+
+ with open(self.full_manifest, 'w+') as manifest:
+ pkg_re = re.compile('^Installing ([^ ]+) [^ ].*')
+ for line in set(output.split('\n')):
+ m = pkg_re.match(line)
+ if m:
+ manifest.write(m.group(1) + '\n')
+
+ bb.utils.remove(image_rootfs, True)
+ return
+
class DpkgManifest(Manifest):
def create_initial(self):
@@ -272,6 +332,9 @@ class DpkgManifest(Manifest):
def create_final(self):
pass
+ def create_full(self):
+ pass
+
def create_manifest(d, final_manifest=False, manifest_dir=None,
manifest_type=Manifest.MANIFEST_TYPE_IMAGE):
--
1.8.1.2
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [PATCH 1/5] manifest.py: add create_full for OpkgManifest class
2014-02-20 7:06 ` [PATCH 1/5] manifest.py: add create_full for OpkgManifest class Hongxu Jia
@ 2014-02-20 15:21 ` Laurentiu Palcu
0 siblings, 0 replies; 10+ messages in thread
From: Laurentiu Palcu @ 2014-02-20 15:21 UTC (permalink / raw)
To: Hongxu Jia; +Cc: saul.wold, openembedded-core
On Thu, Feb 20, 2014 at 03:06:51PM +0800, Hongxu Jia wrote:
> The function create_full creates the manifest after the package in
> initial manifest has been dummy installed. It lists all *to be
> installed* packages. There is no real installation, just a test.
>
> [YOCTO #1894]
> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
> ---
> meta/lib/oe/manifest.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++-
> 1 file changed, 64 insertions(+), 1 deletion(-)
>
> diff --git a/meta/lib/oe/manifest.py b/meta/lib/oe/manifest.py
> index a4bc04b..6e40a9f 100644
> --- a/meta/lib/oe/manifest.py
> +++ b/meta/lib/oe/manifest.py
> @@ -1,6 +1,8 @@
> from abc import ABCMeta, abstractmethod
> +from oe.package_manager import *
> import os
> import re
> +import bb
>
>
> class Manifest(object):
> @@ -69,6 +71,7 @@ class Manifest(object):
>
> self.initial_manifest = os.path.join(self.manifest_dir, "%s_initial_manifest" % manifest_type)
> self.final_manifest = os.path.join(self.manifest_dir, "%s_final_manifest" % manifest_type)
> + self.full_manifest = os.path.join(self.manifest_dir, "%s_full_manifest" % manifest_type)
>
> # packages in the following vars will be split in 'must install' and
> # 'multilib'
> @@ -128,6 +131,15 @@ class Manifest(object):
> pass
>
> """
> + This creates the manifest after the package in initial manifest has been
> + dummy installed. It lists all *to be installed* packages. There is no real
> + installation, just a test.
> + """
> + @abstractmethod
> + def create_full(self):
> + pass
> +
> + """
> The following function parses an initial manifest and returns a dictionary
> object with the must install, attempt only, multilib and language packages.
> """
> @@ -158,6 +170,22 @@ class Manifest(object):
>
> return pkgs
>
> + '''
> + This following function parses a full manifest and return a list
> + object with packages.
> + '''
> + def parse_full_manifest(self):
> + installed_pkgs = list()
> + if not os.path.exists(self.full_manifest):
> + bb.note('full manifest not exist')
> + return installed_pkgs
> +
> + with open(self.full_manifest, 'r') as manifest:
> + for pkg in manifest.read().split('\n'):
> + installed_pkgs.append(pkg.strip())
> +
> + return installed_pkgs
> +
>
> class RpmManifest(Manifest):
> """
> @@ -202,10 +230,12 @@ class RpmManifest(Manifest):
> for pkg in pkgs[pkg_type].split():
> manifest.write("%s,%s\n" % (pkg_type, pkg))
>
> -
> def create_final(self):
> pass
>
> + def create_full(self):
> + pass
> +
>
> class OpkgManifest(Manifest):
> """
> @@ -253,6 +283,36 @@ class OpkgManifest(Manifest):
> def create_final(self):
> pass
>
> + def create_full(self):
> + if not os.path.exists(self.initial_manifest):
> + self.create_initial()
> +
> + initial_manifest = self.parse_initial_manifest()
> + pkgs_to_install = list()
> + for pkg_type in initial_manifest:
> + pkgs_to_install += initial_manifest[pkg_type]
> + if len(pkgs_to_install) == 0:
> + return
> +
> + image_rootfs = self.d.expand("${T}/opkg")
> + opkg_conf = self.d.getVar("IPKGCONF_TARGET", True)
> + pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True)
> + pm = OpkgPM(self.d, image_rootfs, opkg_conf, pkg_archs)
Do we really need to instantiate another PM object here? Can't the
caller pass an instance to it? After all, dummy_install() is not
supposed to alter the metadata, am I wrong? So, in this case, I believe
the caller can safely pass an instance to PM as an argument to the
function.
> +
> + pm.update()
> + pm.handle_bad_recommendations()
> + output = pm.dummy_install(pkgs_to_install)
> +
> + with open(self.full_manifest, 'w+') as manifest:
> + pkg_re = re.compile('^Installing ([^ ]+) [^ ].*')
> + for line in set(output.split('\n')):
> + m = pkg_re.match(line)
> + if m:
> + manifest.write(m.group(1) + '\n')
> +
> + bb.utils.remove(image_rootfs, True)
> + return
> +
>
> class DpkgManifest(Manifest):
> def create_initial(self):
> @@ -272,6 +332,9 @@ class DpkgManifest(Manifest):
> def create_final(self):
> pass
>
> + def create_full(self):
> + pass
> +
>
> def create_manifest(d, final_manifest=False, manifest_dir=None,
> manifest_type=Manifest.MANIFEST_TYPE_IMAGE):
> --
> 1.8.1.2
>
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH 2/5] package_manager.py: support ipk incremental image generation
2014-02-20 7:06 [PATCH V2 0/5] manifest.py/package_manager.py/rootfs.py: support ipk incremental image generation Hongxu Jia
2014-02-20 7:06 ` [PATCH 1/5] manifest.py: add create_full for OpkgManifest class Hongxu Jia
@ 2014-02-20 7:06 ` Hongxu Jia
2014-02-20 15:22 ` Laurentiu Palcu
2014-02-20 7:06 ` [PATCH 3/5] rootfs.py: " Hongxu Jia
` (2 subsequent siblings)
4 siblings, 1 reply; 10+ messages in thread
From: Hongxu Jia @ 2014-02-20 7:06 UTC (permalink / raw)
To: openembedded-core; +Cc: saul.wold
Add the following three functions to OpkgPM class:
- The 'dummy_install' is used to dummy install pkgs, and returns the log
of output;
- The 'backup_packaging_data' is used to back up the current opkg
database;
- The 'recover_packaging_data' is used to recover the opkg database
which backed up by the previous image creation;
Tweak 'remove' function in OpkgPM class, which the options for remove
with dependencies was incorrect.
Tweak 'handle_bad_recommendations' function in OpkgPM class:
- Fix none value check;
- Add the existance check of opkg status file;
- Fix the log format typo;
[YOCTO #1894]
Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
---
meta/lib/oe/package_manager.py | 66 +++++++++++++++++++++++++++++++++++++-----
1 file changed, 59 insertions(+), 7 deletions(-)
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 6dc8fbd..eb96727 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -934,12 +934,13 @@ class RpmPM(PackageManager):
class OpkgPM(PackageManager):
- def __init__(self, d, target_rootfs, config_file, archs):
+ def __init__(self, d, target_rootfs, config_file, archs, task_name='target'):
super(OpkgPM, self).__init__(d)
self.target_rootfs = target_rootfs
self.config_file = config_file
self.pkg_archs = archs
+ self.task_name = task_name
self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True)
self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
@@ -956,6 +957,10 @@ class OpkgPM(PackageManager):
bb.utils.mkdirhier(self.opkg_dir)
+ self.saved_opkg_dir = self.d.expand('${T}/saved/%s' % self.task_name)
+ if not os.path.exists(self.d.expand('${T}/saved')):
+ bb.utils.mkdirhier(self.d.expand('${T}/saved'))
+
if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1":
self._create_config()
else:
@@ -1075,7 +1080,9 @@ class OpkgPM(PackageManager):
try:
bb.note("Installing the following packages: %s" % ' '.join(pkgs))
- subprocess.check_output(cmd.split())
+ bb.note(cmd)
+ output = subprocess.check_output(cmd.split())
+ bb.note(output)
except subprocess.CalledProcessError as e:
(bb.fatal, bb.note)[attempt_only]("Unable to install packages. "
"Command '%s' returned %d:\n%s" %
@@ -1083,14 +1090,16 @@ class OpkgPM(PackageManager):
def remove(self, pkgs, with_dependencies=True):
if with_dependencies:
- cmd = "%s %s remove %s" % \
+ cmd = "%s %s --force-depends --force-remove --force-removal-of-dependent-packages remove %s" % \
(self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
else:
cmd = "%s %s --force-depends remove %s" % \
(self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
try:
- subprocess.check_output(cmd.split())
+ bb.note(cmd)
+ output = subprocess.check_output(cmd.split())
+ bb.note(output)
except subprocess.CalledProcessError as e:
bb.fatal("Unable to remove packages. Command '%s' "
"returned %d:\n%s" % (e.cmd, e.returncode, e.output))
@@ -1146,12 +1155,17 @@ class OpkgPM(PackageManager):
return output
def handle_bad_recommendations(self):
- bad_recommendations = self.d.getVar("BAD_RECOMMENDATIONS", True)
- if bad_recommendations is None:
+ bad_recommendations = self.d.getVar("BAD_RECOMMENDATIONS", True) or ""
+ if bad_recommendations.strip() == "":
return
status_file = os.path.join(self.opkg_dir, "status")
+ # If status file existed, it means the bad recommendations has already
+ # been handled
+ if os.path.exists(status_file):
+ return
+
cmd = "%s %s info " % (self.opkg_cmd, self.opkg_args)
with open(status_file, "w+") as status:
@@ -1165,7 +1179,7 @@ class OpkgPM(PackageManager):
"returned %d:\n%s" % (pkg_info, e.returncode, e.output))
if output == "":
- bb.note("Requested ignored recommendation $i is "
+ bb.note("Requested ignored recommendation %s is "
"not a package" % pkg)
continue
@@ -1175,6 +1189,44 @@ class OpkgPM(PackageManager):
else:
status.write(line + "\n")
+ '''
+ The following function dummy installs pkgs and returns the log of output.
+ '''
+ def dummy_install(self, pkgs):
+ if len(pkgs) == 0:
+ return
+
+ # Simulate installation
+ cmd = "%s %s --noaction install %s " % (self.opkg_cmd,
+ self.opkg_args,
+ ' '.join(pkgs))
+ try:
+ output = subprocess.check_output(cmd, shell=True)
+ except subprocess.CalledProcessError as e:
+ bb.note("Unable to dummy install packages. Command '%s' "
+ "returned %d:\n%s" % (cmd, e.returncode, e.output))
+
+ return output
+
+ def backup_packaging_data(self):
+ # Save the opkglib for increment ipk image generation
+ if os.path.exists(self.saved_opkg_dir):
+ bb.utils.remove(self.saved_opkg_dir, True)
+ shutil.copytree(self.opkg_dir,
+ self.saved_opkg_dir,
+ symlinks=True)
+
+ def recover_packaging_data(self):
+ # Move the opkglib back
+ if os.path.exists(self.saved_opkg_dir):
+ if os.path.exists(self.opkg_dir):
+ bb.utils.remove(self.opkg_dir, True)
+
+ bb.note('Recover packaging data')
+ shutil.copytree(self.saved_opkg_dir,
+ self.opkg_dir,
+ symlinks=True)
+
class DpkgPM(PackageManager):
def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None):
--
1.8.1.2
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [PATCH 2/5] package_manager.py: support ipk incremental image generation
2014-02-20 7:06 ` [PATCH 2/5] package_manager.py: support ipk incremental image generation Hongxu Jia
@ 2014-02-20 15:22 ` Laurentiu Palcu
0 siblings, 0 replies; 10+ messages in thread
From: Laurentiu Palcu @ 2014-02-20 15:22 UTC (permalink / raw)
To: Hongxu Jia; +Cc: saul.wold, openembedded-core
On Thu, Feb 20, 2014 at 03:06:52PM +0800, Hongxu Jia wrote:
> Add the following three functions to OpkgPM class:
> - The 'dummy_install' is used to dummy install pkgs, and returns the log
> of output;
> - The 'backup_packaging_data' is used to back up the current opkg
> database;
> - The 'recover_packaging_data' is used to recover the opkg database
> which backed up by the previous image creation;
>
> Tweak 'remove' function in OpkgPM class, which the options for remove
> with dependencies was incorrect.
>
> Tweak 'handle_bad_recommendations' function in OpkgPM class:
> - Fix none value check;
> - Add the existance check of opkg status file;
> - Fix the log format typo;
>
> [YOCTO #1894]
> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
> ---
> meta/lib/oe/package_manager.py | 66 +++++++++++++++++++++++++++++++++++++-----
> 1 file changed, 59 insertions(+), 7 deletions(-)
>
> diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
> index 6dc8fbd..eb96727 100644
> --- a/meta/lib/oe/package_manager.py
> +++ b/meta/lib/oe/package_manager.py
> @@ -934,12 +934,13 @@ class RpmPM(PackageManager):
>
>
> class OpkgPM(PackageManager):
> - def __init__(self, d, target_rootfs, config_file, archs):
> + def __init__(self, d, target_rootfs, config_file, archs, task_name='target'):
> super(OpkgPM, self).__init__(d)
>
> self.target_rootfs = target_rootfs
> self.config_file = config_file
> self.pkg_archs = archs
> + self.task_name = task_name
>
> self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True)
> self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
> @@ -956,6 +957,10 @@ class OpkgPM(PackageManager):
>
> bb.utils.mkdirhier(self.opkg_dir)
>
> + self.saved_opkg_dir = self.d.expand('${T}/saved/%s' % self.task_name)
> + if not os.path.exists(self.d.expand('${T}/saved')):
> + bb.utils.mkdirhier(self.d.expand('${T}/saved'))
> +
> if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1":
> self._create_config()
> else:
> @@ -1075,7 +1080,9 @@ class OpkgPM(PackageManager):
>
> try:
> bb.note("Installing the following packages: %s" % ' '.join(pkgs))
> - subprocess.check_output(cmd.split())
> + bb.note(cmd)
> + output = subprocess.check_output(cmd.split())
> + bb.note(output)
> except subprocess.CalledProcessError as e:
> (bb.fatal, bb.note)[attempt_only]("Unable to install packages. "
> "Command '%s' returned %d:\n%s" %
> @@ -1083,14 +1090,16 @@ class OpkgPM(PackageManager):
>
> def remove(self, pkgs, with_dependencies=True):
> if with_dependencies:
> - cmd = "%s %s remove %s" % \
> + cmd = "%s %s --force-depends --force-remove --force-removal-of-dependent-packages remove %s" % \
> (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
> else:
> cmd = "%s %s --force-depends remove %s" % \
> (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
>
> try:
> - subprocess.check_output(cmd.split())
> + bb.note(cmd)
> + output = subprocess.check_output(cmd.split())
> + bb.note(output)
> except subprocess.CalledProcessError as e:
> bb.fatal("Unable to remove packages. Command '%s' "
> "returned %d:\n%s" % (e.cmd, e.returncode, e.output))
> @@ -1146,12 +1155,17 @@ class OpkgPM(PackageManager):
> return output
>
> def handle_bad_recommendations(self):
> - bad_recommendations = self.d.getVar("BAD_RECOMMENDATIONS", True)
> - if bad_recommendations is None:
> + bad_recommendations = self.d.getVar("BAD_RECOMMENDATIONS", True) or ""
> + if bad_recommendations.strip() == "":
> return
>
> status_file = os.path.join(self.opkg_dir, "status")
>
> + # If status file existed, it means the bad recommendations has already
> + # been handled
> + if os.path.exists(status_file):
> + return
> +
> cmd = "%s %s info " % (self.opkg_cmd, self.opkg_args)
>
> with open(status_file, "w+") as status:
> @@ -1165,7 +1179,7 @@ class OpkgPM(PackageManager):
> "returned %d:\n%s" % (pkg_info, e.returncode, e.output))
>
> if output == "":
> - bb.note("Requested ignored recommendation $i is "
> + bb.note("Requested ignored recommendation %s is "
I already saw a patch fixing this on the mailing list. You might need to
rebase.
> "not a package" % pkg)
> continue
>
> @@ -1175,6 +1189,44 @@ class OpkgPM(PackageManager):
> else:
> status.write(line + "\n")
>
> + '''
> + The following function dummy installs pkgs and returns the log of output.
> + '''
> + def dummy_install(self, pkgs):
> + if len(pkgs) == 0:
> + return
> +
> + # Simulate installation
> + cmd = "%s %s --noaction install %s " % (self.opkg_cmd,
> + self.opkg_args,
> + ' '.join(pkgs))
> + try:
> + output = subprocess.check_output(cmd, shell=True)
> + except subprocess.CalledProcessError as e:
> + bb.note("Unable to dummy install packages. Command '%s' "
> + "returned %d:\n%s" % (cmd, e.returncode, e.output))
> +
> + return output
> +
> + def backup_packaging_data(self):
> + # Save the opkglib for increment ipk image generation
> + if os.path.exists(self.saved_opkg_dir):
> + bb.utils.remove(self.saved_opkg_dir, True)
> + shutil.copytree(self.opkg_dir,
> + self.saved_opkg_dir,
> + symlinks=True)
> +
> + def recover_packaging_data(self):
> + # Move the opkglib back
> + if os.path.exists(self.saved_opkg_dir):
> + if os.path.exists(self.opkg_dir):
> + bb.utils.remove(self.opkg_dir, True)
> +
> + bb.note('Recover packaging data')
> + shutil.copytree(self.saved_opkg_dir,
> + self.opkg_dir,
> + symlinks=True)
> +
>
> class DpkgPM(PackageManager):
> def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None):
> --
> 1.8.1.2
>
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH 3/5] rootfs.py: support ipk incremental image generation
2014-02-20 7:06 [PATCH V2 0/5] manifest.py/package_manager.py/rootfs.py: support ipk incremental image generation Hongxu Jia
2014-02-20 7:06 ` [PATCH 1/5] manifest.py: add create_full for OpkgManifest class Hongxu Jia
2014-02-20 7:06 ` [PATCH 2/5] package_manager.py: support ipk incremental image generation Hongxu Jia
@ 2014-02-20 7:06 ` Hongxu Jia
2014-02-20 15:22 ` Laurentiu Palcu
2014-02-20 7:06 ` [PATCH 4/5] rootfs.py: tweak _multilib_sanity_test for " Hongxu Jia
2014-02-20 7:06 ` [PATCH 5/5] rootfs.py: support BAD_RECOMMENDATIONS " Hongxu Jia
4 siblings, 1 reply; 10+ messages in thread
From: Hongxu Jia @ 2014-02-20 7:06 UTC (permalink / raw)
To: openembedded-core; +Cc: saul.wold
The incremental image generation is based on the previous existing
image, adds new packages, upgrades existing packages, and removes unused
packages.
[YOCTO #1894]
Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
---
meta/lib/oe/rootfs.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 49 insertions(+), 3 deletions(-)
diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
index 3bcb812..93106c2 100644
--- a/meta/lib/oe/rootfs.py
+++ b/meta/lib/oe/rootfs.py
@@ -438,13 +438,25 @@ class OpkgRootfs(Rootfs):
def __init__(self, d, manifest_dir):
super(OpkgRootfs, self).__init__(d)
- bb.utils.remove(self.image_rootfs, True)
- bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True)
self.manifest = OpkgManifest(d, manifest_dir)
self.opkg_conf = self.d.getVar("IPKGCONF_TARGET", True)
self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True)
- self.pm = OpkgPM(d, self.image_rootfs, self.opkg_conf, self.pkg_archs)
+ self.inc_opkg_image_gen = self.d.getVar('INC_IPK_IMAGE_GEN', True) or ""
+ if self.inc_opkg_image_gen != '1':
+ bb.utils.remove(self.image_rootfs, True)
+ self.pm = OpkgPM(d,
+ self.image_rootfs,
+ self.opkg_conf,
+ self.pkg_archs)
+ else:
+ self.pm = OpkgPM(d,
+ self.image_rootfs,
+ self.opkg_conf,
+ self.pkg_archs)
+ self.pm.recover_packaging_data()
+
+ bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True)
"""
This function was reused from the old implementation.
@@ -508,6 +520,34 @@ class OpkgRootfs(Rootfs):
self._multilib_sanity_test(dirs)
+ '''
+ While ipk incremental image generation is enabled, it will remove the
+ unneeded pkgs by comparing the old full manifest in previous existing
+ image and the new full manifest in the current image.
+ '''
+ def _create_incremental(self, pkgs_initial_install):
+ if self.inc_opkg_image_gen == "1":
+ # Parse full manifest in previous existing image creation session
+ old_full_manifest = self.manifest.parse_full_manifest()
+
+ # Create full manifest for the current image session, the old one
+ # will be replaced by the new one.
+ self.manifest.create_full()
+
+ # Parse full manifest in current image creation session
+ new_full_manifest = self.manifest.parse_full_manifest()
+
+ pkg_to_remove = list()
+ for pkg in old_full_manifest:
+ if pkg not in new_full_manifest:
+ pkg_to_remove.append(pkg)
+
+ self.pm.update()
+
+ if pkg_to_remove != []:
+ bb.note('decremental removed: %s' % ' '.join(pkg_to_remove))
+ self.pm.remove(pkg_to_remove)
+
def _create(self):
pkgs_to_install = self.manifest.parse_initial_manifest()
opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS', True)
@@ -518,6 +558,9 @@ class OpkgRootfs(Rootfs):
if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1":
self.pm.write_index()
+ if self.inc_opkg_image_gen == "1":
+ self._create_incremental(pkgs_to_install)
+
execute_pre_post_process(self.d, opkg_pre_process_cmds)
self.pm.update()
@@ -540,6 +583,9 @@ class OpkgRootfs(Rootfs):
execute_pre_post_process(self.d, opkg_post_process_cmds)
execute_pre_post_process(self.d, rootfs_post_install_cmds)
+ if self.inc_opkg_image_gen == "1":
+ self.pm.backup_packaging_data()
+
def _get_delayed_postinsts(self):
pkg_list = []
status_file = os.path.join(self.image_rootfs,
--
1.8.1.2
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [PATCH 3/5] rootfs.py: support ipk incremental image generation
2014-02-20 7:06 ` [PATCH 3/5] rootfs.py: " Hongxu Jia
@ 2014-02-20 15:22 ` Laurentiu Palcu
0 siblings, 0 replies; 10+ messages in thread
From: Laurentiu Palcu @ 2014-02-20 15:22 UTC (permalink / raw)
To: Hongxu Jia; +Cc: saul.wold, openembedded-core
On Thu, Feb 20, 2014 at 03:06:53PM +0800, Hongxu Jia wrote:
> The incremental image generation is based on the previous existing
> image, adds new packages, upgrades existing packages, and removes unused
> packages.
>
> [YOCTO #1894]
> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
> ---
> meta/lib/oe/rootfs.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++---
> 1 file changed, 49 insertions(+), 3 deletions(-)
>
> diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
> index 3bcb812..93106c2 100644
> --- a/meta/lib/oe/rootfs.py
> +++ b/meta/lib/oe/rootfs.py
> @@ -438,13 +438,25 @@ class OpkgRootfs(Rootfs):
> def __init__(self, d, manifest_dir):
> super(OpkgRootfs, self).__init__(d)
>
> - bb.utils.remove(self.image_rootfs, True)
> - bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True)
> self.manifest = OpkgManifest(d, manifest_dir)
> self.opkg_conf = self.d.getVar("IPKGCONF_TARGET", True)
> self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True)
>
> - self.pm = OpkgPM(d, self.image_rootfs, self.opkg_conf, self.pkg_archs)
> + self.inc_opkg_image_gen = self.d.getVar('INC_IPK_IMAGE_GEN', True) or ""
> + if self.inc_opkg_image_gen != '1':
> + bb.utils.remove(self.image_rootfs, True)
> + self.pm = OpkgPM(d,
> + self.image_rootfs,
> + self.opkg_conf,
> + self.pkg_archs)
> + else:
> + self.pm = OpkgPM(d,
> + self.image_rootfs,
> + self.opkg_conf,
> + self.pkg_archs)
> + self.pm.recover_packaging_data()
> +
> + bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True)
>
> """
> This function was reused from the old implementation.
> @@ -508,6 +520,34 @@ class OpkgRootfs(Rootfs):
>
> self._multilib_sanity_test(dirs)
>
> + '''
> + While ipk incremental image generation is enabled, it will remove the
> + unneeded pkgs by comparing the old full manifest in previous existing
> + image and the new full manifest in the current image.
> + '''
> + def _create_incremental(self, pkgs_initial_install):
Can we name this function _remove_extra_packages()? Because it doesn't
actually install anything... it just removes packages, correct?
> + if self.inc_opkg_image_gen == "1":
> + # Parse full manifest in previous existing image creation session
> + old_full_manifest = self.manifest.parse_full_manifest()
> +
> + # Create full manifest for the current image session, the old one
> + # will be replaced by the new one.
> + self.manifest.create_full()
> +
> + # Parse full manifest in current image creation session
> + new_full_manifest = self.manifest.parse_full_manifest()
> +
> + pkg_to_remove = list()
> + for pkg in old_full_manifest:
> + if pkg not in new_full_manifest:
> + pkg_to_remove.append(pkg)
> +
> + self.pm.update()
this update could be removed, if we move the function call below
self.pm.handle_bad_recommendations(). See below.
> +
> + if pkg_to_remove != []:
> + bb.note('decremental removed: %s' % ' '.join(pkg_to_remove))
> + self.pm.remove(pkg_to_remove)
> +
> def _create(self):
> pkgs_to_install = self.manifest.parse_initial_manifest()
> opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS', True)
> @@ -518,6 +558,9 @@ class OpkgRootfs(Rootfs):
> if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1":
> self.pm.write_index()
>
> + if self.inc_opkg_image_gen == "1":
> + self._create_incremental(pkgs_to_install)
Why not move this piece lower, below
self.pm.handle_bad_recommendations()? Correct me if I'm wrong, but I
believe the rootfs pre process commands should be executed before we do
anything on the rootfs generation.
> + execute_pre_post_process(self.d, opkg_pre_process_cmds)
>
> self.pm.update()
> @@ -540,6 +583,9 @@ class OpkgRootfs(Rootfs):
> execute_pre_post_process(self.d, opkg_post_process_cmds)
> execute_pre_post_process(self.d, rootfs_post_install_cmds)
>
> + if self.inc_opkg_image_gen == "1":
> + self.pm.backup_packaging_data()
> +
> def _get_delayed_postinsts(self):
> pkg_list = []
> status_file = os.path.join(self.image_rootfs,
> --
> 1.8.1.2
>
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH 4/5] rootfs.py: tweak _multilib_sanity_test for ipk incremental image generation
2014-02-20 7:06 [PATCH V2 0/5] manifest.py/package_manager.py/rootfs.py: support ipk incremental image generation Hongxu Jia
` (2 preceding siblings ...)
2014-02-20 7:06 ` [PATCH 3/5] rootfs.py: " Hongxu Jia
@ 2014-02-20 7:06 ` Hongxu Jia
2014-02-20 7:06 ` [PATCH 5/5] rootfs.py: support BAD_RECOMMENDATIONS " Hongxu Jia
4 siblings, 0 replies; 10+ messages in thread
From: Hongxu Jia @ 2014-02-20 7:06 UTC (permalink / raw)
To: openembedded-core; +Cc: saul.wold
The _multilib_sanity_test installs multilib packages in a temporary
root fs, and compare with the current image to figure out duplicated
files that come from different packages.
While incremental image generation enabled and the previous image
was existed, there was an Multilib check error:
...
ERROR: Multilib check error: duplicate files tmp/work/qemux86_64-poky-
linux/core-image-minimal/1.0-r0/multilib/lib32/lib/libc.so.6 tmp/work/
qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs/lib/libc.so.6
is not the same
...
The reason is the file in the existing image has been prelinked by
previous image generation and the file in a temporary root fs is not
prelinked, even though both of them came from the same package, the
Multilib check failed.
[YOCTO #1894]
Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
---
meta/lib/oe/rootfs.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 49 insertions(+), 2 deletions(-)
diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
index 93106c2..6120bce 100644
--- a/meta/lib/oe/rootfs.py
+++ b/meta/lib/oe/rootfs.py
@@ -3,6 +3,8 @@ from oe.utils import execute_pre_post_process
from oe.utils import contains as base_contains
from oe.package_manager import *
from oe.manifest import *
+import oe.path
+import filecmp
import shutil
import os
import subprocess
@@ -458,13 +460,58 @@ class OpkgRootfs(Rootfs):
bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True)
+ def _prelink_file(self, root_dir, filename):
+ bb.note('prelink %s in %s' % (filename, root_dir))
+ prelink_cfg = oe.path.join(root_dir,
+ self.d.expand('${sysconfdir}/prelink.conf'))
+ if not os.path.exists(prelink_cfg):
+ shutil.copy(self.d.expand('${STAGING_DIR_NATIVE}${sysconfdir_native}/prelink.conf'),
+ prelink_cfg)
+
+ cmd_prelink = self.d.expand('${STAGING_DIR_NATIVE}${sbindir_native}/prelink')
+ self._exec_shell_cmd([cmd_prelink,
+ '--root',
+ root_dir,
+ '-amR',
+ '-N',
+ '-c',
+ self.d.expand('${sysconfdir}/prelink.conf')])
+
+ '''
+ Compare two files with the same key twice to see if they are equal.
+ If they are not equal, it means they are duplicated and come from
+ different packages.
+ 1st: Comapre them directly;
+ 2nd: While incremental image creation is enabled, one of the
+ files could be probaly prelinked in the previous image
+ creation and the file has been changed, so we need to
+ prelink the other one and compare them.
+ '''
+ def _file_equal(self, key, f1, f2):
+
+ # Both of them are not prelinked
+ if filecmp.cmp(f1, f2):
+ return True
+
+ if self.image_rootfs not in f1:
+ self._prelink_file(f1.replace(key, ''), f1)
+
+ if self.image_rootfs not in f2:
+ self._prelink_file(f2.replace(key, ''), f2)
+
+ # Both of them are prelinked
+ if filecmp.cmp(f1, f2):
+ return True
+
+ # Not equal
+ return False
+
"""
This function was reused from the old implementation.
See commit: "image.bbclass: Added variables for multilib support." by
Lianhao Lu.
"""
def _multilib_sanity_test(self, dirs):
- import filecmp
allow_replace = self.d.getVar("MULTILIBRE_ALLOW_REP", True)
if allow_replace is None:
@@ -488,7 +535,7 @@ class OpkgRootfs(Rootfs):
else:
if os.path.exists(files[key]) and \
os.path.exists(item) and \
- not filecmp.cmp(files[key], item):
+ not self._file_equal(key, files[key], item):
valid = False
bb.fatal("%s duplicate files %s %s is not the same\n" %
(error_prompt, item, files[key]))
--
1.8.1.2
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 5/5] rootfs.py: support BAD_RECOMMENDATIONS for ipk incremental image generation
2014-02-20 7:06 [PATCH V2 0/5] manifest.py/package_manager.py/rootfs.py: support ipk incremental image generation Hongxu Jia
` (3 preceding siblings ...)
2014-02-20 7:06 ` [PATCH 4/5] rootfs.py: tweak _multilib_sanity_test for " Hongxu Jia
@ 2014-02-20 7:06 ` Hongxu Jia
2014-02-20 15:22 ` Laurentiu Palcu
4 siblings, 1 reply; 10+ messages in thread
From: Hongxu Jia @ 2014-02-20 7:06 UTC (permalink / raw)
To: openembedded-core; +Cc: saul.wold
While incremental image generation enabled and the previous image is
existed, if BAD_RECOMMENDATIONS is changed, the operation on the
existing image is complicated, so remove the existing image in this
situation.
The same with PACKAGE_EXCLUDE and NO_RECOMMENDATIONS.
[YOCTO #1894]
Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
---
meta/lib/oe/rootfs.py | 29 ++++++++++++++++++++++++++++-
1 file changed, 28 insertions(+), 1 deletion(-)
diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
index 6120bce..80be490 100644
--- a/meta/lib/oe/rootfs.py
+++ b/meta/lib/oe/rootfs.py
@@ -445,7 +445,7 @@ class OpkgRootfs(Rootfs):
self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True)
self.inc_opkg_image_gen = self.d.getVar('INC_IPK_IMAGE_GEN', True) or ""
- if self.inc_opkg_image_gen != '1':
+ if self._remove_existing_image():
bb.utils.remove(self.image_rootfs, True)
self.pm = OpkgPM(d,
self.image_rootfs,
@@ -595,6 +595,33 @@ class OpkgRootfs(Rootfs):
bb.note('decremental removed: %s' % ' '.join(pkg_to_remove))
self.pm.remove(pkg_to_remove)
+ '''
+ Compare with previous existing image creation, if some conditions
+ triggered, the previous existing image should be removed.
+ The conditions include any of 'PACKAGE_EXCLUDE, NO_RECOMMENDATIONS
+ and BAD_RECOMMENDATIONS' has been changed.
+ '''
+ def _remove_existing_image(self):
+ if self.inc_opkg_image_gen != "1":
+ return True
+
+ vars_list_file = self.d.expand('${T}/vars_list')
+
+ old_vars_list = ""
+ if os.path.exists(vars_list_file):
+ old_vars_list = open(vars_list_file, 'r+').read()
+
+ new_vars_list = '%s:%s:%s\n' % \
+ ((self.d.getVar('BAD_RECOMMENDATIONS', True) or '').strip(),
+ (self.d.getVar('NO_RECOMMENDATIONS', True) or '').strip(),
+ (self.d.getVar('PACKAGE_EXCLUDE', True) or '').strip())
+ open(vars_list_file, 'w+').write(new_vars_list)
+
+ if old_vars_list != new_vars_list:
+ return True
+
+ return False
+
def _create(self):
pkgs_to_install = self.manifest.parse_initial_manifest()
opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS', True)
--
1.8.1.2
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [PATCH 5/5] rootfs.py: support BAD_RECOMMENDATIONS for ipk incremental image generation
2014-02-20 7:06 ` [PATCH 5/5] rootfs.py: support BAD_RECOMMENDATIONS " Hongxu Jia
@ 2014-02-20 15:22 ` Laurentiu Palcu
0 siblings, 0 replies; 10+ messages in thread
From: Laurentiu Palcu @ 2014-02-20 15:22 UTC (permalink / raw)
To: Hongxu Jia; +Cc: saul.wold, openembedded-core
On Thu, Feb 20, 2014 at 03:06:55PM +0800, Hongxu Jia wrote:
> While incremental image generation enabled and the previous image is
> existed, if BAD_RECOMMENDATIONS is changed, the operation on the
> existing image is complicated, so remove the existing image in this
> situation.
>
> The same with PACKAGE_EXCLUDE and NO_RECOMMENDATIONS.
>
> [YOCTO #1894]
>
> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
> ---
> meta/lib/oe/rootfs.py | 29 ++++++++++++++++++++++++++++-
> 1 file changed, 28 insertions(+), 1 deletion(-)
>
> diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
> index 6120bce..80be490 100644
> --- a/meta/lib/oe/rootfs.py
> +++ b/meta/lib/oe/rootfs.py
> @@ -445,7 +445,7 @@ class OpkgRootfs(Rootfs):
> self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True)
>
> self.inc_opkg_image_gen = self.d.getVar('INC_IPK_IMAGE_GEN', True) or ""
> - if self.inc_opkg_image_gen != '1':
> + if self._remove_existing_image():
> bb.utils.remove(self.image_rootfs, True)
> self.pm = OpkgPM(d,
> self.image_rootfs,
> @@ -595,6 +595,33 @@ class OpkgRootfs(Rootfs):
> bb.note('decremental removed: %s' % ' '.join(pkg_to_remove))
> self.pm.remove(pkg_to_remove)
>
> + '''
> + Compare with previous existing image creation, if some conditions
> + triggered, the previous existing image should be removed.
> + The conditions include any of 'PACKAGE_EXCLUDE, NO_RECOMMENDATIONS
> + and BAD_RECOMMENDATIONS' has been changed.
> + '''
> + def _remove_existing_image(self):
Since we're not removing the image, but the old rootfs, a better name
for this would be: _remove_old_rootfs().
> + if self.inc_opkg_image_gen != "1":
> + return True
> +
> + vars_list_file = self.d.expand('${T}/vars_list')
> +
> + old_vars_list = ""
> + if os.path.exists(vars_list_file):
> + old_vars_list = open(vars_list_file, 'r+').read()
> +
> + new_vars_list = '%s:%s:%s\n' % \
> + ((self.d.getVar('BAD_RECOMMENDATIONS', True) or '').strip(),
> + (self.d.getVar('NO_RECOMMENDATIONS', True) or '').strip(),
> + (self.d.getVar('PACKAGE_EXCLUDE', True) or '').strip())
> + open(vars_list_file, 'w+').write(new_vars_list)
> +
> + if old_vars_list != new_vars_list:
> + return True
> +
> + return False
> +
> def _create(self):
> pkgs_to_install = self.manifest.parse_initial_manifest()
> opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS', True)
> --
> 1.8.1.2
>
^ permalink raw reply [flat|nested] 10+ messages in thread