* [PATCHv2 0/3] Add extract() method to package manager
@ 2016-05-12 11:28 mariano.lopez
2016-05-12 11:28 ` [PATCHv2 1/3] package_manager.py: Move opkg_query() outside of Indexer class mariano.lopez
` (2 more replies)
0 siblings, 3 replies; 4+ messages in thread
From: mariano.lopez @ 2016-05-12 11:28 UTC (permalink / raw)
To: openembedded-core
From: Mariano Lopez <mariano.lopez@linux.intel.com>
Sometimes it is needed to have the content of a package outside of an image,
these series add this capability using a temp directory.
Changes in v2:
- Add default values when parsing package information. This way it won't
throw an exception when trying to save an undefined variable
The following changes since commit 28433319ad8299aa23b1fcfdddbe100b29e86517:
bitbake: toaster: tests browser Add test for creating a project (2016-05-11 11:32:58 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib mariano/bug9569v2
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=mariano/bug9569v2
Mariano Lopez (3):
package_manager.py: Move opkg_query() outside of Indexer class
package_manager.py: Add extract() method for opkg and dpkg
package_manager.py: Add extract() method for RPM package manager
meta/lib/oe/package_manager.py | 321 ++++++++++++++++++++++++++++++++++-------
1 file changed, 266 insertions(+), 55 deletions(-)
--
2.6.6
^ permalink raw reply [flat|nested] 4+ messages in thread
* [PATCHv2 1/3] package_manager.py: Move opkg_query() outside of Indexer class
2016-05-12 11:28 [PATCHv2 0/3] Add extract() method to package manager mariano.lopez
@ 2016-05-12 11:28 ` mariano.lopez
2016-05-12 11:28 ` [PATCHv2 2/3] package_manager.py: Add extract() method for opkg and dpkg mariano.lopez
2016-05-12 11:28 ` [PATCHv2 3/3] package_manager.py: Add extract() method for RPM package manager mariano.lopez
2 siblings, 0 replies; 4+ messages in thread
From: mariano.lopez @ 2016-05-12 11:28 UTC (permalink / raw)
To: openembedded-core
From: Mariano Lopez <mariano.lopez@linux.intel.com>
When using the opkg and apt-get package managers the function
opkg_query() can be useful when query for package information.
This change moves the function outside the Indexer class so
the Indexer, OpkgPM, DpkgPM can benefit from it.
[YOCTO #9569]
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
meta/lib/oe/package_manager.py | 104 ++++++++++++++++++++---------------------
1 file changed, 51 insertions(+), 53 deletions(-)
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index b4b359a..427518d 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -27,6 +27,55 @@ def create_index(arg):
return None
+"""
+This method parse the output from the package managerand return
+a dictionary with the information of the packages. This is used
+when the packages are in deb or ipk format.
+"""
+def opkg_query(cmd_output):
+ verregex = re.compile(' \([=<>]* [^ )]*\)')
+ output = dict()
+ filename = ""
+ dep = []
+ pkg = ""
+ for line in cmd_output.splitlines():
+ line = line.rstrip()
+ if ':' in line:
+ if line.startswith("Package: "):
+ pkg = line.split(": ")[1]
+ elif line.startswith("Architecture: "):
+ arch = line.split(": ")[1]
+ elif line.startswith("Version: "):
+ ver = line.split(": ")[1]
+ elif line.startswith("File: "):
+ filename = line.split(": ")[1]
+ elif line.startswith("Depends: "):
+ depends = verregex.sub('', line.split(": ")[1])
+ for depend in depends.split(", "):
+ dep.append(depend)
+ elif line.startswith("Recommends: "):
+ recommends = verregex.sub('', line.split(": ")[1])
+ for recommend in recommends.split(", "):
+ dep.append("%s [REC]" % recommend)
+ else:
+ # IPK doesn't include the filename
+ if not filename:
+ filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
+ if pkg:
+ output[pkg] = {"arch":arch, "ver":ver,
+ "filename":filename, "deps": dep }
+ pkg = ""
+ filename = ""
+ dep = []
+
+ if pkg:
+ if not filename:
+ filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
+ output[pkg] = {"arch":arch, "ver":ver,
+ "filename":filename, "deps": dep }
+
+ return output
+
class Indexer(object):
__metaclass__ = ABCMeta
@@ -293,57 +342,6 @@ class PkgsList(object):
pass
- """
- This method parse the output from the package manager
- and return a dictionary with the information of the
- installed packages. This is used whne the packages are
- in deb or ipk format
- """
- def opkg_query(self, cmd_output):
- verregex = re.compile(' \([=<>]* [^ )]*\)')
- output = dict()
- filename = ""
- dep = []
- pkg = ""
- for line in cmd_output.splitlines():
- line = line.rstrip()
- if ':' in line:
- if line.startswith("Package: "):
- pkg = line.split(": ")[1]
- elif line.startswith("Architecture: "):
- arch = line.split(": ")[1]
- elif line.startswith("Version: "):
- ver = line.split(": ")[1]
- elif line.startswith("File: "):
- filename = line.split(": ")[1]
- elif line.startswith("Depends: "):
- depends = verregex.sub('', line.split(": ")[1])
- for depend in depends.split(", "):
- dep.append(depend)
- elif line.startswith("Recommends: "):
- recommends = verregex.sub('', line.split(": ")[1])
- for recommend in recommends.split(", "):
- dep.append("%s [REC]" % recommend)
- else:
- # IPK doesn't include the filename
- if not filename:
- filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
- if pkg:
- output[pkg] = {"arch":arch, "ver":ver,
- "filename":filename, "deps": dep }
- pkg = ""
- filename = ""
- dep = []
-
- if pkg:
- if not filename:
- filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
- output[pkg] = {"arch":arch, "ver":ver,
- "filename":filename, "deps": dep }
-
- return output
-
-
class RpmPkgsList(PkgsList):
def __init__(self, d, rootfs_dir, arch_var=None, os_var=None):
super(RpmPkgsList, self).__init__(d, rootfs_dir)
@@ -479,7 +477,7 @@ class OpkgPkgsList(PkgsList):
bb.fatal("Cannot get the installed packages list. Command '%s' "
"returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr))
- return self.opkg_query(cmd_output)
+ return opkg_query(cmd_output)
class DpkgPkgsList(PkgsList):
@@ -497,7 +495,7 @@ class DpkgPkgsList(PkgsList):
bb.fatal("Cannot get the installed packages list. Command '%s' "
"returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output))
- return self.opkg_query(cmd_output)
+ return opkg_query(cmd_output)
class PackageManager(object):
--
2.6.6
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [PATCHv2 2/3] package_manager.py: Add extract() method for opkg and dpkg
2016-05-12 11:28 [PATCHv2 0/3] Add extract() method to package manager mariano.lopez
2016-05-12 11:28 ` [PATCHv2 1/3] package_manager.py: Move opkg_query() outside of Indexer class mariano.lopez
@ 2016-05-12 11:28 ` mariano.lopez
2016-05-12 11:28 ` [PATCHv2 3/3] package_manager.py: Add extract() method for RPM package manager mariano.lopez
2 siblings, 0 replies; 4+ messages in thread
From: mariano.lopez @ 2016-05-12 11:28 UTC (permalink / raw)
To: openembedded-core
From: Mariano Lopez <mariano.lopez@linux.intel.com>
Sometimes it is needed to have the content of a package outside
the recipe context. This new method extract the content of an
IPK/DEB file to a tmpdir, without actually installing the package.
A new OpkgDpkgPM class was added to share the code for opkg and dpkg.
There were need some changes to opkg_query() in order to use it
with apt-cache output. Also set default values to avoid UnboundLocalError
[YOCTO #9569]
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
meta/lib/oe/package_manager.py | 140 +++++++++++++++++++++++++++++++++++++++--
1 file changed, 134 insertions(+), 6 deletions(-)
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 427518d..f5517a4 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -35,9 +35,12 @@ when the packages are in deb or ipk format.
def opkg_query(cmd_output):
verregex = re.compile(' \([=<>]* [^ )]*\)')
output = dict()
+ pkg = ""
+ arch = ""
+ ver = ""
filename = ""
dep = []
- pkg = ""
+ pkgarch = ""
for line in cmd_output.splitlines():
line = line.rstrip()
if ':' in line:
@@ -47,8 +50,10 @@ def opkg_query(cmd_output):
arch = line.split(": ")[1]
elif line.startswith("Version: "):
ver = line.split(": ")[1]
- elif line.startswith("File: "):
+ elif line.startswith("File: ") or line.startswith("Filename:"):
filename = line.split(": ")[1]
+ if "/" in filename:
+ filename = os.path.basename(filename)
elif line.startswith("Depends: "):
depends = verregex.sub('', line.split(": ")[1])
for depend in depends.split(", "):
@@ -57,16 +62,23 @@ def opkg_query(cmd_output):
recommends = verregex.sub('', line.split(": ")[1])
for recommend in recommends.split(", "):
dep.append("%s [REC]" % recommend)
- else:
+ elif line.startswith("PackageArch: "):
+ pkgarch = line.split(": ")[1]
+
+ # When there is a blank line save the package information
+ elif not line:
# IPK doesn't include the filename
if not filename:
filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
if pkg:
output[pkg] = {"arch":arch, "ver":ver,
- "filename":filename, "deps": dep }
+ "filename":filename, "deps": dep, "pkgarch":pkgarch }
pkg = ""
+ arch = ""
+ ver = ""
filename = ""
dep = []
+ pkgarch = ""
if pkg:
if not filename:
@@ -1397,7 +1409,70 @@ class RpmPM(PackageManager):
bb.utils.remove(f, True)
-class OpkgPM(PackageManager):
+class OpkgDpkgPM(PackageManager):
+ """
+ This is an abstract class. Do not instantiate this directly.
+ """
+ def __init__(self, d):
+ super(OpkgDpkgPM, self).__init__(d)
+
+ """
+ Returns a dictionary with the package info.
+
+ This method extracts the common parts for Opkg and Dpkg
+ """
+ def package_info(self, pkg, cmd):
+
+ try:
+ output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+ except subprocess.CalledProcessError as e:
+ bb.fatal("Unable to list available packages. Command '%s' "
+ "returned %d:\n%s" % (cmd, e.returncode, e.output))
+ return opkg_query(output)
+
+ """
+ Returns the path to a tmpdir where resides the contents of a package.
+
+ Deleting the tmpdir is responsability of the caller.
+
+ This method extracts the common parts for Opkg and Dpkg
+ """
+ def extract(self, pkg, pkg_path):
+
+ ar_cmd = bb.utils.which(os.getenv("PATH"), "ar")
+ tar_cmd = bb.utils.which(os.getenv("PATH"), "tar")
+
+ if not os.path.isfile(pkg_path):
+ bb.fatal("Unable to extract package for '%s'."
+ "File %s doesn't exists" % (pkg, pkg_path))
+
+ tmp_dir = tempfile.mkdtemp()
+ current_dir = os.getcwd()
+ os.chdir(tmp_dir)
+
+ try:
+ cmd = "%s x %s" % (ar_cmd, pkg_path)
+ output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+ cmd = "%s xf data.tar.*" % tar_cmd
+ output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+ except subprocess.CalledProcessError as e:
+ bb.utils.remove(tmp_dir, recurse=True)
+ bb.fatal("Unable to extract %s package. Command '%s' "
+ "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output))
+ except OSError as e:
+ bb.utils.remove(tmp_dir, recurse=True)
+ bb.fatal("Unable to extract %s package. Command '%s' "
+ "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename))
+
+ bb.note("Extracted %s to %s" % (pkg_path, tmp_dir))
+ bb.utils.remove(os.path.join(tmp_dir, "debian-binary"))
+ bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz"))
+ os.chdir(current_dir)
+
+ return tmp_dir
+
+
+class OpkgPM(OpkgDpkgPM):
def __init__(self, d, target_rootfs, config_file, archs, task_name='target'):
super(OpkgPM, self).__init__(d)
@@ -1732,8 +1807,34 @@ class OpkgPM(PackageManager):
self.opkg_dir,
symlinks=True)
+ """
+ Returns a dictionary with the package info.
+ """
+ def package_info(self, pkg):
+ cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg)
+ return super(OpkgPM, self).package_info(pkg, cmd)
+
+ """
+ Returns the path to a tmpdir where resides the contents of a package.
+
+ Deleting the tmpdir is responsability of the caller.
+ """
+ def extract(self, pkg):
+ pkg_info = self.package_info(pkg)
+ if not pkg_info:
+ bb.fatal("Unable to get information for package '%s' while "
+ "trying to extract the package." % pkg)
+
+ pkg_arch = pkg_info[pkg]["arch"]
+ pkg_filename = pkg_info[pkg]["filename"]
+ pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
+
+ tmp_dir = super(OpkgPM, self).extract(pkg, pkg_path)
+ bb.utils.remove(os.path.join(tmp_dir, "data.tar.gz"))
+
+ return tmp_dir
-class DpkgPM(PackageManager):
+class DpkgPM(OpkgDpkgPM):
def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None):
super(DpkgPM, self).__init__(d)
self.target_rootfs = target_rootfs
@@ -1744,6 +1845,7 @@ class DpkgPM(PackageManager):
self.apt_conf_dir = apt_conf_dir
self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf")
self.apt_get_cmd = bb.utils.which(os.getenv('PATH'), "apt-get")
+ self.apt_cache_cmd = bb.utils.which(os.getenv('PATH'), "apt-cache")
self.apt_args = d.getVar("APT_ARGS", True)
@@ -2027,6 +2129,32 @@ class DpkgPM(PackageManager):
def list_installed(self):
return DpkgPkgsList(self.d, self.target_rootfs).list_pkgs()
+ """
+ Returns a dictionary with the package info.
+ """
+ def package_info(self, pkg):
+ cmd = "%s show %s" % (self.apt_cache_cmd, pkg)
+ return super(DpkgPM, self).package_info(pkg, cmd)
+
+ """
+ Returns the path to a tmpdir where resides the contents of a package.
+
+ Deleting the tmpdir is responsability of the caller.
+ """
+ def extract(self, pkg):
+ pkg_info = self.package_info(pkg)
+ if not pkg_info:
+ bb.fatal("Unable to get information for package '%s' while "
+ "trying to extract the package." % pkg)
+
+ pkg_arch = pkg_info[pkg]["pkgarch"]
+ pkg_filename = pkg_info[pkg]["filename"]
+ pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
+
+ tmp_dir = super(DpkgPM, self).extract(pkg, pkg_path)
+ bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz"))
+
+ return tmp_dir
def generate_index_files(d):
classes = d.getVar('PACKAGE_CLASSES', True).replace("package_", "").split()
--
2.6.6
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [PATCHv2 3/3] package_manager.py: Add extract() method for RPM package manager
2016-05-12 11:28 [PATCHv2 0/3] Add extract() method to package manager mariano.lopez
2016-05-12 11:28 ` [PATCHv2 1/3] package_manager.py: Move opkg_query() outside of Indexer class mariano.lopez
2016-05-12 11:28 ` [PATCHv2 2/3] package_manager.py: Add extract() method for opkg and dpkg mariano.lopez
@ 2016-05-12 11:28 ` mariano.lopez
2 siblings, 0 replies; 4+ messages in thread
From: mariano.lopez @ 2016-05-12 11:28 UTC (permalink / raw)
To: openembedded-core
From: Mariano Lopez <mariano.lopez@linux.intel.com>
This new method extract the content of RPM file to a tmpdir,
without actually installing the package.
[YOCTO #9569]
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
meta/lib/oe/package_manager.py | 85 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 85 insertions(+)
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index f5517a4..1eedeb8 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -1408,6 +1408,91 @@ class RpmPM(PackageManager):
for f in rpm_db_locks:
bb.utils.remove(f, True)
+ """
+ Returns a dictionary with the package info.
+ """
+ def package_info(self, pkg):
+ cmd = "%s %s info --urls %s" % (self.smart_cmd, self.smart_opt, pkg)
+ try:
+ output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+ except subprocess.CalledProcessError as e:
+ bb.fatal("Unable to list available packages. Command '%s' "
+ "returned %d:\n%s" % (cmd, e.returncode, e.output))
+
+ # Set default values to avoid UnboundLocalError
+ arch = ""
+ ver = ""
+ filename = ""
+
+ #Parse output
+ for line in output.splitlines():
+ line = line.rstrip()
+ if line.startswith("Name:"):
+ pkg = line.split(": ")[1]
+ elif line.startswith("Version:"):
+ tmp_str = line.split(": ")[1]
+ ver, arch = tmp_str.split("@")
+ break
+
+ # Get filename
+ index = re.search("^URLs", output, re.MULTILINE)
+ tmp_str = output[index.end():]
+ for line in tmp_str.splitlines():
+ if "/" in line:
+ line = line.lstrip()
+ filename = line.split(" ")[0]
+ break
+
+ # To have the same data type than other package_info methods
+ pkg_dict = {}
+ pkg_dict[pkg] = {"arch":arch, "ver":ver, "filename":filename}
+
+ return pkg_dict
+
+ """
+ Returns the path to a tmpdir where resides the contents of a package.
+
+ Deleting the tmpdir is responsability of the caller.
+
+ """
+ def extract(self, pkg):
+ pkg_info = self.package_info(pkg)
+ if not pkg_info:
+ bb.fatal("Unable to get information for package '%s' while "
+ "trying to extract the package." % pkg)
+
+ pkg_arch = pkg_info[pkg]["arch"]
+ pkg_filename = pkg_info[pkg]["filename"]
+ pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
+
+ cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio")
+ rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio")
+
+ if not os.path.isfile(pkg_path):
+ bb.fatal("Unable to extract package for '%s'."
+ "File %s doesn't exists" % (pkg, pkg_path))
+
+ tmp_dir = tempfile.mkdtemp()
+ current_dir = os.getcwd()
+ os.chdir(tmp_dir)
+
+ try:
+ cmd = "%s %s | %s -idmv" % (rpm2cpio_cmd, pkg_path, cpio_cmd)
+ output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+ except subprocess.CalledProcessError as e:
+ bb.utils.remove(tmp_dir, recurse=True)
+ bb.fatal("Unable to extract %s package. Command '%s' "
+ "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output))
+ except OSError as e:
+ bb.utils.remove(tmp_dir, recurse=True)
+ bb.fatal("Unable to extract %s package. Command '%s' "
+ "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename))
+
+ bb.note("Extracted %s to %s" % (pkg_path, tmp_dir))
+ os.chdir(current_dir)
+
+ return tmp_dir
+
class OpkgDpkgPM(PackageManager):
"""
--
2.6.6
^ permalink raw reply related [flat|nested] 4+ messages in thread
end of thread, other threads:[~2016-05-12 19:32 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2016-05-12 11:28 [PATCHv2 0/3] Add extract() method to package manager mariano.lopez
2016-05-12 11:28 ` [PATCHv2 1/3] package_manager.py: Move opkg_query() outside of Indexer class mariano.lopez
2016-05-12 11:28 ` [PATCHv2 2/3] package_manager.py: Add extract() method for opkg and dpkg mariano.lopez
2016-05-12 11:28 ` [PATCHv2 3/3] package_manager.py: Add extract() method for RPM package manager mariano.lopez
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.