Re: [PATCH] lib/oe/package_manager: Introducing PACKAGE_FEED_BASE_PATHS/PACKAGE_FEED_ARCHS

[Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>]

Documentation related to this feature is already in the manual but 
feature is still not present in the code, so requesting it for merge..


On 10/05/2015 08:12 AM, leonardo.sandoval.gonzalez@linux.intel.com wrote:
> From: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
>
> The recently introduced PACKAGE_FEED_PREFIX is not flexible enough for
> constructing URIs, because the same PREFIX is used for all PACKAGE_FEED_URIS.
> Also, the string 'PREFIX' is confusing because it is not at the beginning of
> the URI. The variable PACKAGE_FEED_BASE_PATHS replaces PACKAGE_FEED_PREFIX,
> allowing multiple base paths to be appended on each PACKAGE_FEED_URIS. In the
> other hand, a new variable called PACKAGE_FEED_ARCHS, similar in concept to
> PACKAGE_BASE_PATHS, defines package architectures defined by the user.
>
> To demonstrate the usage of the PACKAGE_FEED_URIS, PACKAGE_FEED_BASE_PATHS and
> PACKAGE_FEED_ARCHS, let's assume these variables are set on local.conf
>
>      PACKAGE_FEED_URIS = "https://example.com/packagerepos/release \
>                           https://example.com/packagerepos/updates"
>      PACKAGE_FEED_BASE_PATHS = "rpm rpm-dev"
>      PACKAGE_FEED_ARCHS = "all core2-64"
>
> the resulting feeds would be
>
>      https://example.com/packagerepos/release/rpm/all
>      https://example.com/packagerepos/release/rpm/core2-64
>      https://example.com/packagerepos/release/rpm-dev/all
>      https://example.com/packagerepos/release/rpm-dev/core2-64
>      https://example.com/packagerepos/updates/rpm/all
>      https://example.com/packagerepos/updates/rpm/core2-64
>      https://example.com/packagerepos/updates/rpm-dev/all
>      https://example.com/packagerepos/updates/rpm-dev/core2-64
>
> Signed-off-by: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
> ---
>   meta/lib/oe/package_manager.py | 154 ++++++++++++++++++++++++++---------------
>   1 file changed, 99 insertions(+), 55 deletions(-)
>
> diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
> index 630b957..e182027 100644
> --- a/meta/lib/oe/package_manager.py
> +++ b/meta/lib/oe/package_manager.py
> @@ -8,7 +8,7 @@ import re
>   import bb
>   import tempfile
>   import oe.utils
> -
> +import string
>   
>   # this can be used by all PM backends to create the index files in parallel
>   def create_index(arg):
> @@ -522,7 +522,8 @@ class PackageManager(object):
>           self.deploy_dir = None
>           self.deploy_lock = None
>           self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True) or ""
> -        self.feed_prefix = self.d.getVar('PACKAGE_FEED_PREFIX', True) or ""
> +        self.feed_base_paths = self.d.getVar('PACKAGE_FEED_BASE_PATHS', True) or ""
> +        self.feed_archs = self.d.getVar('PACKAGE_FEED_ARCHS', True)
>   
>       """
>       Update the package manager package database.
> @@ -629,6 +630,25 @@ class PackageManager(object):
>   
>           self.deploy_lock = None
>   
> +    """
> +    Construct URIs based on the following pattern: uri/base_path where 'uri'
> +    and 'base_path' correspond to each element of the corresponding array
> +    argument leading to len(uris) x len(base_paths) elements on the returned
> +    array
> +    """
> +    def construct_uris(self, uris, base_paths):
> +        def _append(arr1, arr2, sep='/'):
> +            res = []
> +            narr1 = map(lambda a: string.rstrip(a, sep), arr1)
> +            narr2 = map(lambda a: string.lstrip(string.rstrip(a, sep), sep), arr2)
> +            for a1 in narr1:
> +                if arr2:
> +                    for a2 in narr2:
> +                        res.append("%s%s%s" % (a1, sep, a2))
> +                else:
> +                    res.append(a1)
> +            return res
> +        return _append(uris, base_paths)
>   
>   class RpmPM(PackageManager):
>       def __init__(self,
> @@ -672,42 +692,55 @@ class RpmPM(PackageManager):
>           if self.feed_uris == "":
>               return
>   
> -        # List must be prefered to least preferred order
> -        default_platform_extra = set()
> -        platform_extra = set()
> -        bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or ""
> -        for mlib in self.ml_os_list:
> -            for arch in self.ml_prefix_list[mlib]:
> -                plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib]
> -                if mlib == bbextendvariant:
> -                        default_platform_extra.add(plt)
> -                else:
> -                        platform_extra.add(plt)
> +        arch_list = []
> +        if self.feed_archs is not None:
> +            # User define feed architectures
> +            arch_list = self.feed_archs.split()
> +        else:
> +            # List must be prefered to least preferred order
> +            default_platform_extra = set()
> +            platform_extra = set()
> +            bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or ""
> +            for mlib in self.ml_os_list:
> +                for arch in self.ml_prefix_list[mlib]:
> +                    plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib]
> +                    if mlib == bbextendvariant:
> +                            default_platform_extra.add(plt)
> +                    else:
> +                            platform_extra.add(plt)
>   
> -        platform_extra = platform_extra.union(default_platform_extra)
> +            platform_extra = platform_extra.union(default_platform_extra)
>   
> -        arch_list = []
> -        for canonical_arch in platform_extra:
> -            arch = canonical_arch.split('-')[0]
> -            if not os.path.exists(os.path.join(self.deploy_dir, arch)):
> -                continue
> -            arch_list.append(arch)
> +            for canonical_arch in platform_extra:
> +                arch = canonical_arch.split('-')[0]
> +                if not os.path.exists(os.path.join(self.deploy_dir, arch)):
> +                    continue
> +                arch_list.append(arch)
> +
> +        feed_uris = self.construct_uris(self.feed_uris.split(), self.feed_base_paths.split())
>   
>           uri_iterator = 0
> -        channel_priority = 10 + 5 * len(self.feed_uris.split()) * len(arch_list)
> +        channel_priority = 10 + 5 * len(feed_uris) * (len(arch_list) if arch_list else 1)
>   
> -        for uri in self.feed_uris.split():
> -            full_uri = uri
> -            if self.feed_prefix:
> -                full_uri = os.path.join(uri, self.feed_prefix)
> -            for arch in arch_list:
> -                bb.note('Note: adding Smart channel url%d%s (%s)' %
> -                        (uri_iterator, arch, channel_priority))
> -                self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/%s -y'
> -                                   % (uri_iterator, arch, full_uri, arch))
> -                self._invoke_smart('channel --set url%d-%s priority=%d' %
> -                                   (uri_iterator, arch, channel_priority))
> +        for uri in feed_uris:
> +            if arch_list:
> +                for arch in arch_list:
> +                    bb.note('Note: adding Smart channel url%d%s (%s)' %
> +                            (uri_iterator, arch, channel_priority))
> +                    self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/%s -y'
> +                                       % (uri_iterator, arch, uri, arch))
> +                    self._invoke_smart('channel --set url%d-%s priority=%d' %
> +                                       (uri_iterator, arch, channel_priority))
> +                    channel_priority -= 5
> +            else:
> +                bb.note('Note: adding Smart channel url%d (%s)' %
> +                        (uri_iterator, channel_priority))
> +                self._invoke_smart('channel --add url%d type=rpm-md baseurl=%s -y'
> +                                   % (uri_iterator, uri))
> +                self._invoke_smart('channel --set url%d priority=%d' %
> +                                   (uri_iterator, channel_priority))
>                   channel_priority -= 5
> +
>               uri_iterator += 1
>   
>       '''
> @@ -1480,21 +1513,26 @@ class OpkgPM(PackageManager):
>           rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf'
>                                     % self.target_rootfs)
>   
> +        feed_uris = self.construct_uris(self.feed_uris.split(), self.feed_base_paths.split())
> +        archs = self.pkg_archs.split() if self.feed_archs is None else self.feed_archs.split()
> +
>           with open(rootfs_config, "w+") as config_file:
>               uri_iterator = 0
> -            for uri in self.feed_uris.split():
> -                full_uri = uri
> -                if self.feed_prefix:
> -                    full_uri = os.path.join(uri, self.feed_prefix)
> -
> -                for arch in self.pkg_archs.split():
> -                    if not os.path.exists(os.path.join(self.deploy_dir, arch)):
> -                        continue
> -                    bb.note('Note: adding opkg feed url-%s-%d (%s)' %
> -                        (arch, uri_iterator, full_uri))
> +            for uri in feed_uris:
> +                if archs:
> +                    for arch in archs:
> +                        if (self.feed_archs is None) and (not os.path.exists(os.path.join(self.deploy_dir, arch))):
> +                            continue
> +                        bb.note('Note: adding opkg feed url-%s-%d (%s)' %
> +                            (arch, uri_iterator, uri))
> +                        config_file.write("src/gz uri-%s-%d %s/%s\n" %
> +                                          (arch, uri_iterator, uri, arch))
> +                else:
> +                    bb.note('Note: adding opkg feed url-%d (%s)' %
> +                        (uri_iterator, uri))
> +                    config_file.write("src/gz uri-%d %s\n" %
> +                                      (uri_iterator, uri))
>   
> -                    config_file.write("src/gz uri-%s-%d %s/%s\n" %
> -                                      (arch, uri_iterator, full_uri, arch))
>                   uri_iterator += 1
>   
>       def update(self):
> @@ -1840,20 +1878,26 @@ class DpkgPM(PackageManager):
>                                       % self.target_rootfs)
>           arch_list = []
>   
> -        for arch in self.all_arch_list:
> -            if not os.path.exists(os.path.join(self.deploy_dir, arch)):
> -                continue
> -            arch_list.append(arch)
> +        if self.feed_archs is None:
> +            for arch in self.all_arch_list:
> +                if not os.path.exists(os.path.join(self.deploy_dir, arch)):
> +                    continue
> +                arch_list.append(arch)
> +        else:
> +            arch_list = self.feed_archs.split()
> +
> +        feed_uris = self.construct_uris(self.feed_uris.split(), self.feed_base_paths.split())
>   
>           with open(sources_conf, "w+") as sources_file:
> -            for uri in self.feed_uris.split():
> -                full_uri = uri
> -                if self.feed_prefix:
> -                    full_uri = os.path.join(uri, self.feed_prefix)
> -                for arch in arch_list:
> +            for uri in feed_uris:
> +                if arch_list:
> +                    for arch in arch_list:
> +                        bb.note('Note: adding dpkg channel at (%s)' % uri)
> +                        sources_file.write("deb %s/%s ./\n" %
> +                                           (uri, arch))
> +                else:
>                       bb.note('Note: adding dpkg channel at (%s)' % uri)
> -                    sources_file.write("deb %s/%s ./\n" %
> -                                       (full_uri, arch))
> +                    sources_file.write("deb %s ./\n" % uri)
>   
>       def _create_configs(self, archs, base_archs):
>           base_archs = re.sub("_", "-", base_archs)