[PATCH 0/3] Add extract() method to package manager

[PATCH 0/3] Add extract() method to package manager

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

Sometimes it is needed to have the content of a package outside of an image,
these series add this capability using a temp directory.

The following changes since commit 7ca60ec8bff7656b4e52f5a4d238913e851da089:

  test-empty-image: Fix LIC_FILES_CHKSUM typo (2016-05-06 10:48:06 +0100)

are available in the git repository at:

  git://git.yoctoproject.org/poky-contrib mariano/bug9569
  http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=mariano/bug9569

Mariano Lopez (3):
  package_manager.py: Move opkg_query() outside of Indexer class
  package_manager.py: Add extract() method for opkg and dpkg
  package_manager.py: Add extract() method for RPM package manager

 meta/lib/oe/package_manager.py | 312 +++++++++++++++++++++++++++++++++--------
 1 file changed, 257 insertions(+), 55 deletions(-)

-- 
2.6.6

[PATCH 1/3] package_manager.py: Move opkg_query() outside of Indexer class

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

When using the opkg and apt-get package managers the function
opkg_query() can be useful when query for package information.

This change moves the function outside the Indexer class so
the Indexer, OpkgPM, DpkgPM can benefit from it.

[YOCTO #9569]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/lib/oe/package_manager.py | 104 ++++++++++++++++++++---------------------
 1 file changed, 51 insertions(+), 53 deletions(-)

diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index b4b359a..427518d 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -27,6 +27,55 @@ def create_index(arg):
 
     return None
 
+"""
+This method parse the output from the package managerand return
+a dictionary with the information of the packages. This is used
+when the packages are in deb or ipk format.
+"""
+def opkg_query(cmd_output):
+    verregex = re.compile(' \([=<>]* [^ )]*\)')
+    output = dict()
+    filename = ""
+    dep = []
+    pkg = ""
+    for line in cmd_output.splitlines():
+        line = line.rstrip()
+        if ':' in line:
+            if line.startswith("Package: "):
+                pkg = line.split(": ")[1]
+            elif line.startswith("Architecture: "):
+                arch = line.split(": ")[1]
+            elif line.startswith("Version: "):
+                ver = line.split(": ")[1]
+            elif line.startswith("File: "):
+                filename = line.split(": ")[1]
+            elif line.startswith("Depends: "):
+                depends = verregex.sub('', line.split(": ")[1])
+                for depend in depends.split(", "):
+                    dep.append(depend)
+            elif line.startswith("Recommends: "):
+                recommends = verregex.sub('', line.split(": ")[1])
+                for recommend in recommends.split(", "):
+                    dep.append("%s [REC]" % recommend)
+        else:
+            # IPK doesn't include the filename
+            if not filename:
+                filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
+            if pkg:
+                output[pkg] = {"arch":arch, "ver":ver,
+                        "filename":filename, "deps": dep }
+            pkg = ""
+            filename = ""
+            dep = []
+
+    if pkg:
+        if not filename:
+            filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
+        output[pkg] = {"arch":arch, "ver":ver,
+                "filename":filename, "deps": dep }
+
+    return output
+
 
 class Indexer(object):
     __metaclass__ = ABCMeta
@@ -293,57 +342,6 @@ class PkgsList(object):
         pass
 
 
-    """
-    This method parse the output from the package manager
-    and return a dictionary with the information of the
-    installed packages. This is used whne the packages are
-    in deb or ipk format
-    """
-    def opkg_query(self, cmd_output):
-        verregex = re.compile(' \([=<>]* [^ )]*\)')
-        output = dict()
-        filename = ""
-        dep = []
-        pkg = ""
-        for line in cmd_output.splitlines():
-            line = line.rstrip()
-            if ':' in line:
-                if line.startswith("Package: "):
-                    pkg = line.split(": ")[1]
-                elif line.startswith("Architecture: "):
-                    arch = line.split(": ")[1]
-                elif line.startswith("Version: "):
-                    ver = line.split(": ")[1]
-                elif line.startswith("File: "):
-                    filename = line.split(": ")[1]
-                elif line.startswith("Depends: "):
-                    depends = verregex.sub('', line.split(": ")[1])
-                    for depend in depends.split(", "):
-                        dep.append(depend)
-                elif line.startswith("Recommends: "):
-                    recommends = verregex.sub('', line.split(": ")[1])
-                    for recommend in recommends.split(", "):
-                        dep.append("%s [REC]" % recommend)
-            else:
-                # IPK doesn't include the filename
-                if not filename:
-                    filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
-                if pkg:
-                    output[pkg] = {"arch":arch, "ver":ver,
-                            "filename":filename, "deps": dep }
-                pkg = ""
-                filename = ""
-                dep = []
-
-        if pkg:
-            if not filename:
-                filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
-            output[pkg] = {"arch":arch, "ver":ver,
-                    "filename":filename, "deps": dep }
-
-        return output
-
-
 class RpmPkgsList(PkgsList):
     def __init__(self, d, rootfs_dir, arch_var=None, os_var=None):
         super(RpmPkgsList, self).__init__(d, rootfs_dir)
@@ -479,7 +477,7 @@ class OpkgPkgsList(PkgsList):
             bb.fatal("Cannot get the installed packages list. Command '%s' "
                      "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr))
 
-        return self.opkg_query(cmd_output)
+        return opkg_query(cmd_output)
 
 
 class DpkgPkgsList(PkgsList):
@@ -497,7 +495,7 @@ class DpkgPkgsList(PkgsList):
             bb.fatal("Cannot get the installed packages list. Command '%s' "
                      "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output))
 
-        return self.opkg_query(cmd_output)
+        return opkg_query(cmd_output)
 
 
 class PackageManager(object):
-- 
2.6.6

Re: [PATCH 1/3] package_manager.py: Move opkg_query() outside of Indexer class

[Joshua G Lock]

On Wed, 2016-05-11 at 12:31 +0000, mariano.lopez@linux.intel.com wrote:
> From: Mariano Lopez <mariano.lopez@linux.intel.com>
> 
> When using the opkg and apt-get package managers the function
> opkg_query() can be useful when query for package information.
> 
> This change moves the function outside the Indexer class so
> the Indexer, OpkgPM, DpkgPM can benefit from it.
> 
> [YOCTO #9569]
> 
> Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
> ---
>  meta/lib/oe/package_manager.py | 104 ++++++++++++++++++++-----------
> ----------
>  1 file changed, 51 insertions(+), 53 deletions(-)
> 
> diff --git a/meta/lib/oe/package_manager.py
> b/meta/lib/oe/package_manager.py
> index b4b359a..427518d 100644
> --- a/meta/lib/oe/package_manager.py
> +++ b/meta/lib/oe/package_manager.py
> @@ -27,6 +27,55 @@ def create_index(arg):
>  
>      return None
>  
> +"""
> +This method parse the output from the package managerand return
> +a dictionary with the information of the packages. This is used
> +when the packages are in deb or ipk format.
> +"""
> +def opkg_query(cmd_output):
> +    verregex = re.compile(' \([=<>]* [^ )]*\)')
> +    output = dict()
> +    filename = ""
> +    dep = []
> +    pkg = ""

Here we assign a default value for dep, pkg and filename but not arch
and ver. 

I feel it's safest we assign defaults for all of the variables that are
otherwise only created if a matching line is found, otherwise we might
run into unexpected UnboundLocalError's which cause nasty crashes.

> +    for line in cmd_output.splitlines():
> +        line = line.rstrip()
> +        if ':' in line:
> +            if line.startswith("Package: "):
> +                pkg = line.split(": ")[1]
> +            elif line.startswith("Architecture: "):
> +                arch = line.split(": ")[1]
> +            elif line.startswith("Version: "):
> +                ver = line.split(": ")[1]
> +            elif line.startswith("File: "):
> +                filename = line.split(": ")[1]
> +            elif line.startswith("Depends: "):
> +                depends = verregex.sub('', line.split(": ")[1])
> +                for depend in depends.split(", "):
> +                    dep.append(depend)
> +            elif line.startswith("Recommends: "):
> +                recommends = verregex.sub('', line.split(": ")[1])
> +                for recommend in recommends.split(", "):
> +                    dep.append("%s [REC]" % recommend)
> +        else:
> +            # IPK doesn't include the filename
> +            if not filename:
> +                filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
> +            if pkg:
> +                output[pkg] = {"arch":arch, "ver":ver,
> +                        "filename":filename, "deps": dep }
> +            pkg = ""
> +            filename = ""
> +            dep = []
> +
> +    if pkg:
> +        if not filename:
> +            filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
> +        output[pkg] = {"arch":arch, "ver":ver,
> +                "filename":filename, "deps": dep }
> +
> +    return output
> +
>  
>  class Indexer(object):
>      __metaclass__ = ABCMeta
> @@ -293,57 +342,6 @@ class PkgsList(object):
>          pass
>  
>  
> -    """
> -    This method parse the output from the package manager
> -    and return a dictionary with the information of the
> -    installed packages. This is used whne the packages are
> -    in deb or ipk format
> -    """
> -    def opkg_query(self, cmd_output):
> -        verregex = re.compile(' \([=<>]* [^ )]*\)')
> -        output = dict()
> -        filename = ""
> -        dep = []
> -        pkg = ""
> -        for line in cmd_output.splitlines():
> -            line = line.rstrip()
> -            if ':' in line:
> -                if line.startswith("Package: "):
> -                    pkg = line.split(": ")[1]
> -                elif line.startswith("Architecture: "):
> -                    arch = line.split(": ")[1]
> -                elif line.startswith("Version: "):
> -                    ver = line.split(": ")[1]
> -                elif line.startswith("File: "):
> -                    filename = line.split(": ")[1]
> -                elif line.startswith("Depends: "):
> -                    depends = verregex.sub('', line.split(": ")[1])
> -                    for depend in depends.split(", "):
> -                        dep.append(depend)
> -                elif line.startswith("Recommends: "):
> -                    recommends = verregex.sub('', line.split(":
> ")[1])
> -                    for recommend in recommends.split(", "):
> -                        dep.append("%s [REC]" % recommend)
> -            else:
> -                # IPK doesn't include the filename
> -                if not filename:
> -                    filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
> -                if pkg:
> -                    output[pkg] = {"arch":arch, "ver":ver,
> -                            "filename":filename, "deps": dep }
> -                pkg = ""
> -                filename = ""
> -                dep = []
> -
> -        if pkg:
> -            if not filename:
> -                filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
> -            output[pkg] = {"arch":arch, "ver":ver,
> -                    "filename":filename, "deps": dep }
> -
> -        return output
> -
> -
>  class RpmPkgsList(PkgsList):
>      def __init__(self, d, rootfs_dir, arch_var=None, os_var=None):
>          super(RpmPkgsList, self).__init__(d, rootfs_dir)
> @@ -479,7 +477,7 @@ class OpkgPkgsList(PkgsList):
>              bb.fatal("Cannot get the installed packages list.
> Command '%s' "
>                       "returned %d and stderr:\n%s" % (cmd,
> p.returncode, cmd_stderr))
>  
> -        return self.opkg_query(cmd_output)
> +        return opkg_query(cmd_output)
>  
>  
>  class DpkgPkgsList(PkgsList):
> @@ -497,7 +495,7 @@ class DpkgPkgsList(PkgsList):
>              bb.fatal("Cannot get the installed packages list.
> Command '%s' "
>                       "returned %d:\n%s" % (' '.join(cmd),
> e.returncode, e.output))
>  
> -        return self.opkg_query(cmd_output)
> +        return opkg_query(cmd_output)
>  
>  
>  class PackageManager(object):
> -- 
> 2.6.6
> 

[PATCH 2/3] package_manager.py: Add extract() method for opkg and dpkg

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

Sometimes it is needed to have the content of a package outside
the recipe context.  This new method extract the content of an
IPK/DEB file to a tmpdir, without actually installing the package.

A new OpkgDpkgPM class was added to share the code for opkg and dpkg.

There were need some changes to opkg_query() in order to use it
with apt-cache output.

[YOCTO #9569]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/lib/oe/package_manager.py | 134 +++++++++++++++++++++++++++++++++++++++--
 1 file changed, 129 insertions(+), 5 deletions(-)

diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 427518d..0830da9 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -38,6 +38,7 @@ def opkg_query(cmd_output):
     filename = ""
     dep = []
     pkg = ""
+    pkgarch = ""
     for line in cmd_output.splitlines():
         line = line.rstrip()
         if ':' in line:
@@ -47,8 +48,10 @@ def opkg_query(cmd_output):
                 arch = line.split(": ")[1]
             elif line.startswith("Version: "):
                 ver = line.split(": ")[1]
-            elif line.startswith("File: "):
+            elif line.startswith("File: ") or line.startswith("Filename:"):
                 filename = line.split(": ")[1]
+                if "/" in filename:
+                    filename = os.path.basename(filename)
             elif line.startswith("Depends: "):
                 depends = verregex.sub('', line.split(": ")[1])
                 for depend in depends.split(", "):
@@ -57,15 +60,20 @@ def opkg_query(cmd_output):
                 recommends = verregex.sub('', line.split(": ")[1])
                 for recommend in recommends.split(", "):
                     dep.append("%s [REC]" % recommend)
-        else:
+            elif line.startswith("PackageArch: "):
+                pkgarch = line.split(": ")[1]
+
+        # When there is a blank line save the package information
+        elif not line:
             # IPK doesn't include the filename
             if not filename:
                 filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
             if pkg:
                 output[pkg] = {"arch":arch, "ver":ver,
-                        "filename":filename, "deps": dep }
+                        "filename":filename, "deps": dep, "pkgarch":pkgarch }
             pkg = ""
             filename = ""
+            pkgarch = ""
             dep = []
 
     if pkg:
@@ -1397,7 +1405,70 @@ class RpmPM(PackageManager):
             bb.utils.remove(f, True)
 
 
-class OpkgPM(PackageManager):
+class OpkgDpkgPM(PackageManager):
+    """
+    This is an abstract class. Do not instantiate this directly.
+    """
+    def __init__(self, d):
+        super(OpkgDpkgPM, self).__init__(d)
+
+    """
+    Returns a dictionary with the package info.
+
+    This method extracts the common parts for Opkg and Dpkg
+    """
+    def package_info(self, pkg, cmd):
+
+        try:
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+        except subprocess.CalledProcessError as e:
+            bb.fatal("Unable to list available packages. Command '%s' "
+                     "returned %d:\n%s" % (cmd, e.returncode, e.output))
+        return opkg_query(output)
+
+    """
+    Returns the path to a tmpdir where resides the contents of a package.
+
+    Deleting the tmpdir is responsability of the caller.
+
+    This method extracts the common parts for Opkg and Dpkg
+    """
+    def extract(self, pkg, pkg_path):
+
+        ar_cmd = bb.utils.which(os.getenv("PATH"), "ar")
+        tar_cmd = bb.utils.which(os.getenv("PATH"), "tar")
+
+        if not os.path.isfile(pkg_path):
+            bb.fatal("Unable to extract package for '%s'."
+                     "File %s doesn't exists" % (pkg, pkg_path))
+
+        tmp_dir = tempfile.mkdtemp()
+        current_dir = os.getcwd()
+        os.chdir(tmp_dir)
+
+        try:
+            cmd = "%s x %s" % (ar_cmd, pkg_path)
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+            cmd = "%s xf data.tar.*" % tar_cmd
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+        except subprocess.CalledProcessError as e:
+            bb.utils.remove(tmp_dir, recurse=True)
+            bb.fatal("Unable to extract %s package. Command '%s' "
+                     "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output))
+        except OSError as e:
+            bb.utils.remove(tmp_dir, recurse=True)
+            bb.fatal("Unable to extract %s package. Command '%s' "
+                     "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename))
+
+        bb.note("Extracted %s to %s" % (pkg_path, tmp_dir))
+        bb.utils.remove(os.path.join(tmp_dir, "debian-binary"))
+        bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz"))
+        os.chdir(current_dir)
+
+        return tmp_dir
+
+
+class OpkgPM(OpkgDpkgPM):
     def __init__(self, d, target_rootfs, config_file, archs, task_name='target'):
         super(OpkgPM, self).__init__(d)
 
@@ -1732,8 +1803,34 @@ class OpkgPM(PackageManager):
                             self.opkg_dir,
                             symlinks=True)
 
+    """
+    Returns a dictionary with the package info.
+    """
+    def package_info(self, pkg):
+        cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg)
+        return super(OpkgPM, self).package_info(pkg, cmd)
+
+    """
+    Returns the path to a tmpdir where resides the contents of a package.
+
+    Deleting the tmpdir is responsability of the caller.
+    """
+    def extract(self, pkg):
+        pkg_info = self.package_info(pkg)
+        if not pkg_info:
+            bb.fatal("Unable to get information for package '%s' while "
+                     "trying to extract the package."  % pkg)
 
-class DpkgPM(PackageManager):
+        pkg_arch = pkg_info[pkg]["arch"]
+        pkg_filename = pkg_info[pkg]["filename"]
+        pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
+
+        tmp_dir = super(OpkgPM, self).extract(pkg, pkg_path)
+        bb.utils.remove(os.path.join(tmp_dir, "data.tar.gz"))
+
+        return tmp_dir
+
+class DpkgPM(OpkgDpkgPM):
     def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None):
         super(DpkgPM, self).__init__(d)
         self.target_rootfs = target_rootfs
@@ -1744,6 +1841,7 @@ class DpkgPM(PackageManager):
             self.apt_conf_dir = apt_conf_dir
         self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf")
         self.apt_get_cmd = bb.utils.which(os.getenv('PATH'), "apt-get")
+        self.apt_cache_cmd = bb.utils.which(os.getenv('PATH'), "apt-cache")
 
         self.apt_args = d.getVar("APT_ARGS", True)
 
@@ -2027,6 +2125,32 @@ class DpkgPM(PackageManager):
     def list_installed(self):
         return DpkgPkgsList(self.d, self.target_rootfs).list_pkgs()
 
+    """
+    Returns a dictionary with the package info.
+    """
+    def package_info(self, pkg):
+        cmd = "%s show %s" % (self.apt_cache_cmd, pkg)
+        return super(DpkgPM, self).package_info(pkg, cmd)
+
+    """
+    Returns the path to a tmpdir where resides the contents of a package.
+
+    Deleting the tmpdir is responsability of the caller.
+    """
+    def extract(self, pkg):
+        pkg_info = self.package_info(pkg)
+        if not pkg_info:
+            bb.fatal("Unable to get information for package '%s' while "
+                     "trying to extract the package."  % pkg)
+
+        pkg_arch = pkg_info[pkg]["pkgarch"]
+        pkg_filename = pkg_info[pkg]["filename"]
+        pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
+
+        tmp_dir = super(DpkgPM, self).extract(pkg, pkg_path)
+        bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz"))
+
+        return tmp_dir
 
 def generate_index_files(d):
     classes = d.getVar('PACKAGE_CLASSES', True).replace("package_", "").split()
-- 
2.6.6

Re: [PATCH 2/3] package_manager.py: Add extract() method for opkg and dpkg

[Joshua G Lock]

On Wed, 2016-05-11 at 12:31 +0000, mariano.lopez@linux.intel.com wrote:
> From: Mariano Lopez <mariano.lopez@linux.intel.com>
> 
> Sometimes it is needed to have the content of a package outside
> the recipe context.  This new method extract the content of an
> IPK/DEB file to a tmpdir, without actually installing the package.
> 
> A new OpkgDpkgPM class was added to share the code for opkg and dpkg.
> 
> There were need some changes to opkg_query() in order to use it
> with apt-cache output.
> 
> [YOCTO #9569]
> 
> Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
> ---
>  meta/lib/oe/package_manager.py | 134
> +++++++++++++++++++++++++++++++++++++++--
>  1 file changed, 129 insertions(+), 5 deletions(-)
> 
> diff --git a/meta/lib/oe/package_manager.py
> b/meta/lib/oe/package_manager.py
> index 427518d..0830da9 100644
> --- a/meta/lib/oe/package_manager.py
> +++ b/meta/lib/oe/package_manager.py
> @@ -38,6 +38,7 @@ def opkg_query(cmd_output):
>      filename = ""
>      dep = []
>      pkg = ""
> +    pkgarch = ""

Hard to tell from the context here but it looks like arch and ver don't
have default values? Potential UnboundLocalError?

>      for line in cmd_output.splitlines():
>          line = line.rstrip()
>          if ':' in line:
> @@ -47,8 +48,10 @@ def opkg_query(cmd_output):
>                  arch = line.split(": ")[1]
>              elif line.startswith("Version: "):
>                  ver = line.split(": ")[1]
> -            elif line.startswith("File: "):
> +            elif line.startswith("File: ") or
> line.startswith("Filename:"):
>                  filename = line.split(": ")[1]
> +                if "/" in filename:
> +                    filename = os.path.basename(filename)
>              elif line.startswith("Depends: "):
>                  depends = verregex.sub('', line.split(": ")[1])
>                  for depend in depends.split(", "):
> @@ -57,15 +60,20 @@ def opkg_query(cmd_output):
>                  recommends = verregex.sub('', line.split(": ")[1])
>                  for recommend in recommends.split(", "):
>                      dep.append("%s [REC]" % recommend)
> -        else:
> +            elif line.startswith("PackageArch: "):
> +                pkgarch = line.split(": ")[1]
> +
> +        # When there is a blank line save the package information
> +        elif not line:
>              # IPK doesn't include the filename
>              if not filename:
>                  filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
>              if pkg:
>                  output[pkg] = {"arch":arch, "ver":ver,
> -                        "filename":filename, "deps": dep }
> +                        "filename":filename, "deps": dep,
> "pkgarch":pkgarch }
>              pkg = ""
>              filename = ""
> +            pkgarch = ""
>              dep = []
>  
>      if pkg:
> @@ -1397,7 +1405,70 @@ class RpmPM(PackageManager):
>              bb.utils.remove(f, True)
>  
>  
> -class OpkgPM(PackageManager):
> +class OpkgDpkgPM(PackageManager):
> +    """
> +    This is an abstract class. Do not instantiate this directly.
> +    """
> +    def __init__(self, d):
> +        super(OpkgDpkgPM, self).__init__(d)
> +
> +    """
> +    Returns a dictionary with the package info.
> +
> +    This method extracts the common parts for Opkg and Dpkg
> +    """
> +    def package_info(self, pkg, cmd):
> +
> +        try:
> +            output = subprocess.check_output(cmd,
> stderr=subprocess.STDOUT, shell=True)
> +        except subprocess.CalledProcessError as e:
> +            bb.fatal("Unable to list available packages. Command
> '%s' "
> +                     "returned %d:\n%s" % (cmd, e.returncode,
> e.output))
> +        return opkg_query(output)
> +
> +    """
> +    Returns the path to a tmpdir where resides the contents of a
> package.
> +
> +    Deleting the tmpdir is responsability of the caller.
> +
> +    This method extracts the common parts for Opkg and Dpkg
> +    """
> +    def extract(self, pkg, pkg_path):
> +
> +        ar_cmd = bb.utils.which(os.getenv("PATH"), "ar")
> +        tar_cmd = bb.utils.which(os.getenv("PATH"), "tar")
> +
> +        if not os.path.isfile(pkg_path):
> +            bb.fatal("Unable to extract package for '%s'."
> +                     "File %s doesn't exists" % (pkg, pkg_path))
> +
> +        tmp_dir = tempfile.mkdtemp()
> +        current_dir = os.getcwd()
> +        os.chdir(tmp_dir)
> +
> +        try:
> +            cmd = "%s x %s" % (ar_cmd, pkg_path)
> +            output = subprocess.check_output(cmd,
> stderr=subprocess.STDOUT, shell=True)
> +            cmd = "%s xf data.tar.*" % tar_cmd
> +            output = subprocess.check_output(cmd,
> stderr=subprocess.STDOUT, shell=True)
> +        except subprocess.CalledProcessError as e:
> +            bb.utils.remove(tmp_dir, recurse=True)
> +            bb.fatal("Unable to extract %s package. Command '%s' "
> +                     "returned %d:\n%s" % (pkg_path, cmd,
> e.returncode, e.output))
> +        except OSError as e:
> +            bb.utils.remove(tmp_dir, recurse=True)
> +            bb.fatal("Unable to extract %s package. Command '%s' "
> +                     "returned %d:\n%s at %s" % (pkg_path, cmd,
> e.errno, e.strerror, e.filename))
> +
> +        bb.note("Extracted %s to %s" % (pkg_path, tmp_dir))
> +        bb.utils.remove(os.path.join(tmp_dir, "debian-binary"))
> +        bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz"))
> +        os.chdir(current_dir)
> +
> +        return tmp_dir
> +
> +
> +class OpkgPM(OpkgDpkgPM):
>      def __init__(self, d, target_rootfs, config_file, archs,
> task_name='target'):
>          super(OpkgPM, self).__init__(d)
>  
> @@ -1732,8 +1803,34 @@ class OpkgPM(PackageManager):
>                              self.opkg_dir,
>                              symlinks=True)
>  
> +    """
> +    Returns a dictionary with the package info.
> +    """
> +    def package_info(self, pkg):
> +        cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg)
> +        return super(OpkgPM, self).package_info(pkg, cmd)
> +
> +    """
> +    Returns the path to a tmpdir where resides the contents of a
> package.
> +
> +    Deleting the tmpdir is responsability of the caller.
> +    """
> +    def extract(self, pkg):
> +        pkg_info = self.package_info(pkg)
> +        if not pkg_info:
> +            bb.fatal("Unable to get information for package '%s'
> while "
> +                     "trying to extract the package."  % pkg)
>  
> -class DpkgPM(PackageManager):
> +        pkg_arch = pkg_info[pkg]["arch"]
> +        pkg_filename = pkg_info[pkg]["filename"]
> +        pkg_path = os.path.join(self.deploy_dir, pkg_arch,
> pkg_filename)
> +
> +        tmp_dir = super(OpkgPM, self).extract(pkg, pkg_path)
> +        bb.utils.remove(os.path.join(tmp_dir, "data.tar.gz"))
> +
> +        return tmp_dir
> +
> +class DpkgPM(OpkgDpkgPM):
>      def __init__(self, d, target_rootfs, archs, base_archs,
> apt_conf_dir=None):
>          super(DpkgPM, self).__init__(d)
>          self.target_rootfs = target_rootfs
> @@ -1744,6 +1841,7 @@ class DpkgPM(PackageManager):
>              self.apt_conf_dir = apt_conf_dir
>          self.apt_conf_file = os.path.join(self.apt_conf_dir,
> "apt.conf")
>          self.apt_get_cmd = bb.utils.which(os.getenv('PATH'), "apt-
> get")
> +        self.apt_cache_cmd = bb.utils.which(os.getenv('PATH'), "apt-
> cache")
>  
>          self.apt_args = d.getVar("APT_ARGS", True)
>  
> @@ -2027,6 +2125,32 @@ class DpkgPM(PackageManager):
>      def list_installed(self):
>          return DpkgPkgsList(self.d, self.target_rootfs).list_pkgs()
>  
> +    """
> +    Returns a dictionary with the package info.
> +    """
> +    def package_info(self, pkg):
> +        cmd = "%s show %s" % (self.apt_cache_cmd, pkg)
> +        return super(DpkgPM, self).package_info(pkg, cmd)
> +
> +    """
> +    Returns the path to a tmpdir where resides the contents of a
> package.
> +
> +    Deleting the tmpdir is responsability of the caller.
> +    """
> +    def extract(self, pkg):
> +        pkg_info = self.package_info(pkg)
> +        if not pkg_info:
> +            bb.fatal("Unable to get information for package '%s'
> while "
> +                     "trying to extract the package."  % pkg)
> +
> +        pkg_arch = pkg_info[pkg]["pkgarch"]
> +        pkg_filename = pkg_info[pkg]["filename"]
> +        pkg_path = os.path.join(self.deploy_dir, pkg_arch,
> pkg_filename)
> +
> +        tmp_dir = super(DpkgPM, self).extract(pkg, pkg_path)
> +        bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz"))
> +
> +        return tmp_dir
>  
>  def generate_index_files(d):
>      classes = d.getVar('PACKAGE_CLASSES', True).replace("package_",
> "").split()
> -- 
> 2.6.6
> 

[PATCH 3/3] package_manager.py: Add extract() method for RPM package manager

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

This new method extract the content of RPM file to a tmpdir,
without actually installing the package.

[YOCTO #9569]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/lib/oe/package_manager.py | 80 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)

diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 0830da9..d95564a 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -1404,6 +1404,86 @@ class RpmPM(PackageManager):
         for f in rpm_db_locks:
             bb.utils.remove(f, True)
 
+    """
+    Returns a dictionary with the package info.
+    """
+    def package_info(self, pkg):
+        cmd = "%s %s info --urls %s" % (self.smart_cmd, self.smart_opt, pkg)
+        try:
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+        except subprocess.CalledProcessError as e:
+            bb.fatal("Unable to list available packages. Command '%s' "
+                     "returned %d:\n%s" % (cmd, e.returncode, e.output))
+
+        #Parse output
+        for line in output.splitlines():
+            line = line.rstrip()
+            if line.startswith("Name:"):
+                pkg = line.split(": ")[1]
+            elif line.startswith("Version:"):
+                tmp_str = line.split(": ")[1]
+                ver, arch = tmp_str.split("@")
+                break
+
+        # Get filename
+        index = re.search("^URLs", output, re.MULTILINE)
+        tmp_str = output[index.end():]
+        for line in tmp_str.splitlines():
+            if "/" in line:
+                line = line.lstrip()
+                filename = line.split(" ")[0]
+                break
+
+        # To have the same data type than other package_info methods
+        pkg_dict = {}
+        pkg_dict[pkg] = {"arch":arch, "ver":ver, "filename":filename}
+
+        return pkg_dict
+
+    """
+    Returns the path to a tmpdir where resides the contents of a package.
+
+    Deleting the tmpdir is responsability of the caller.
+
+    """
+    def extract(self, pkg):
+        pkg_info = self.package_info(pkg)
+        if not pkg_info:
+            bb.fatal("Unable to get information for package '%s' while "
+                     "trying to extract the package."  % pkg)
+
+        pkg_arch = pkg_info[pkg]["arch"]
+        pkg_filename = pkg_info[pkg]["filename"]
+        pkg_path = os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
+
+        cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio")
+        rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio")
+
+        if not os.path.isfile(pkg_path):
+            bb.fatal("Unable to extract package for '%s'."
+                     "File %s doesn't exists" % (pkg, pkg_path))
+
+        tmp_dir = tempfile.mkdtemp()
+        current_dir = os.getcwd()
+        os.chdir(tmp_dir)
+
+        try:
+            cmd = "%s %s | %s -idmv" % (rpm2cpio_cmd, pkg_path, cpio_cmd)
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+        except subprocess.CalledProcessError as e:
+            bb.utils.remove(tmp_dir, recurse=True)
+            bb.fatal("Unable to extract %s package. Command '%s' "
+                     "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output))
+        except OSError as e:
+            bb.utils.remove(tmp_dir, recurse=True)
+            bb.fatal("Unable to extract %s package. Command '%s' "
+                     "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename))
+
+        bb.note("Extracted %s to %s" % (pkg_path, tmp_dir))
+        os.chdir(current_dir)
+
+        return tmp_dir
+
 
 class OpkgDpkgPM(PackageManager):
     """
-- 
2.6.6

Re: [PATCH 3/3] package_manager.py: Add extract() method for RPM package manager

[Joshua G Lock]

On Wed, 2016-05-11 at 12:31 +0000, mariano.lopez@linux.intel.com wrote:
> From: Mariano Lopez <mariano.lopez@linux.intel.com>
> 
> This new method extract the content of RPM file to a tmpdir,
> without actually installing the package.
> 
> [YOCTO #9569]
> 
> Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
> ---
>  meta/lib/oe/package_manager.py | 80
> ++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 80 insertions(+)
> 
> diff --git a/meta/lib/oe/package_manager.py
> b/meta/lib/oe/package_manager.py
> index 0830da9..d95564a 100644
> --- a/meta/lib/oe/package_manager.py
> +++ b/meta/lib/oe/package_manager.py
> @@ -1404,6 +1404,86 @@ class RpmPM(PackageManager):
>          for f in rpm_db_locks:
>              bb.utils.remove(f, True)
>  
> +    """
> +    Returns a dictionary with the package info.
> +    """
> +    def package_info(self, pkg):
> +        cmd = "%s %s info --urls %s" % (self.smart_cmd,
> self.smart_opt, pkg)
> +        try:
> +            output = subprocess.check_output(cmd,
> stderr=subprocess.STDOUT, shell=True)
> +        except subprocess.CalledProcessError as e:
> +            bb.fatal("Unable to list available packages. Command
> '%s' "
> +                     "returned %d:\n%s" % (cmd, e.returncode,
> e.output))
> +
> +        #Parse output
> +        for line in output.splitlines():
> +            line = line.rstrip()
> +            if line.startswith("Name:"):
> +                pkg = line.split(": ")[1]
> +            elif line.startswith("Version:"):
> +                tmp_str = line.split(": ")[1]
> +                ver, arch = tmp_str.split("@")
> +                break

Could pkg, ver and arch be potentially undefined here?

> +
> +        # Get filename
> +        index = re.search("^URLs", output, re.MULTILINE)
> +        tmp_str = output[index.end():]
> +        for line in tmp_str.splitlines():
> +            if "/" in line:
> +                line = line.lstrip()
> +                filename = line.split(" ")[0]
> +                break
> +
> +        # To have the same data type than other package_info methods
> +        pkg_dict = {}
> +        pkg_dict[pkg] = {"arch":arch, "ver":ver,
> "filename":filename}
> +
> +        return pkg_dict
> +
> +    """
> +    Returns the path to a tmpdir where resides the contents of a
> package.
> +
> +    Deleting the tmpdir is responsability of the caller.
> +
> +    """
> +    def extract(self, pkg):
> +        pkg_info = self.package_info(pkg)
> +        if not pkg_info:
> +            bb.fatal("Unable to get information for package '%s'
> while "
> +                     "trying to extract the package."  % pkg)
> +
> +        pkg_arch = pkg_info[pkg]["arch"]
> +        pkg_filename = pkg_info[pkg]["filename"]
> +        pkg_path = os.path.join(self.deploy_dir, pkg_arch,
> pkg_filename)
> +
> +        cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio")
> +        rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio")
> +
> +        if not os.path.isfile(pkg_path):
> +            bb.fatal("Unable to extract package for '%s'."
> +                     "File %s doesn't exists" % (pkg, pkg_path))
> +
> +        tmp_dir = tempfile.mkdtemp()
> +        current_dir = os.getcwd()
> +        os.chdir(tmp_dir)
> +
> +        try:
> +            cmd = "%s %s | %s -idmv" % (rpm2cpio_cmd, pkg_path,
> cpio_cmd)
> +            output = subprocess.check_output(cmd,
> stderr=subprocess.STDOUT, shell=True)
> +        except subprocess.CalledProcessError as e:
> +            bb.utils.remove(tmp_dir, recurse=True)
> +            bb.fatal("Unable to extract %s package. Command '%s' "
> +                     "returned %d:\n%s" % (pkg_path, cmd,
> e.returncode, e.output))
> +        except OSError as e:
> +            bb.utils.remove(tmp_dir, recurse=True)
> +            bb.fatal("Unable to extract %s package. Command '%s' "
> +                     "returned %d:\n%s at %s" % (pkg_path, cmd,
> e.errno, e.strerror, e.filename))
> +
> +        bb.note("Extracted %s to %s" % (pkg_path, tmp_dir))
> +        os.chdir(current_dir)
> +
> +        return tmp_dir
> +
>  
>  class OpkgDpkgPM(PackageManager):
>      """
> -- 
> 2.6.6
> 

Re: [PATCH 3/3] package_manager.py: Add extract() method for RPM package manager

[Mariano Lopez]

On 05/12/2016 04:31 AM, Joshua G Lock wrote:
> On Wed, 2016-05-11 at 12:31 +0000, mariano.lopez@linux.intel.com wrote:
>> From: Mariano Lopez <mariano.lopez@linux.intel.com>
>>
>> +        #Parse output
>> +        for line in output.splitlines():
>> +            line = line.rstrip()
>> +            if line.startswith("Name:"):
>> +                pkg = line.split(": ")[1]
>> +            elif line.startswith("Version:"):
>> +                tmp_str = line.split(": ")[1]
>> +                ver, arch = tmp_str.split("@")
>> +                break
> Could pkg, ver and arch be potentially undefined here?
>
>

I'll send a v2 with default values

Mariano Lopez