Re: [PATCH 1/1] license.bbclass: Add support for deployed packages not in rootfs

["Aníbal Limón" <anibal.limon@linux.intel.com>]

On 04/08/15 09:04, Mariano Lopez wrote:
>
>
> On 08/03/2015 03:37 PM, Mariano Lopez wrote:
>>
>> On 08/03/2015 03:20 PM, Aníbal Limón wrote:
>>> Hi,
>>>
>>> Comments below,
>>>
>>> On 03/08/15 06:59, mariano.lopez@linux.intel.com wrote:
>>>> From: Mariano Lopez <mariano.lopez@linux.intel.com>
>>>>
>>>> This adds a new manifest file for the packages that were
>>>> deployed but not included into rootfs. The new manifest file
>>>> is in the same directory as the rootfs manifest but the name is
>>>> image_license.manifest.
>>>>
>>>> It also creates the directory structure for the packages and
>>>> add the license file in such directories.
>>>>
>>>> The bootloader is an example of such packages.
>>>>
>>>> [YOCTO #6772]
>>>>
>>>> Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
>>>> ---
>>>>   meta/classes/license.bbclass | 107 
>>>> +++++++++++++++++++++++++++++++++++++------
>>>>   1 file changed, 94 insertions(+), 13 deletions(-)
>>>>
>>>> diff --git a/meta/classes/license.bbclass 
>>>> b/meta/classes/license.bbclass
>>>> index 224d541..42d2748 100644
>>>> --- a/meta/classes/license.bbclass
>>>> +++ b/meta/classes/license.bbclass
>>>> @@ -26,18 +26,14 @@ python write_package_manifest() {
>>>>   }
>>>>     python license_create_manifest() {
>>>> -    import re
>>>>       import oe.packagedata
>>>>       from oe.rootfs import image_list_installed_packages
>>>>   -    bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE", True) or 
>>>> "").split()
>>>> -    bad_licenses = map(lambda l: canonical_license(d, l), 
>>>> bad_licenses)
>>>> -    bad_licenses = expand_wildcard_licenses(d, bad_licenses)
>>>> -
>>>>       build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', 
>>>> True)
>>>>       if build_images_from_feeds == "1":
>>>>           return 0
>>>>   +    # Files that are installed in the rootfs
>>>>       pkg_dic = {}
>>>>       for pkg in image_list_installed_packages(d).split("\n"):
>>>>           pkg_info = os.path.join(d.getVar('PKGDATA_DIR', True),
>>>> @@ -49,8 +45,90 @@ python license_create_manifest() {
>>>>               pkg_lic_name = "LICENSE_" + pkg_name
>>>>               pkg_dic[pkg_name]["LICENSE"] = 
>>>> pkg_dic[pkg_name][pkg_lic_name]
>>>>   -    license_manifest = 
>>>> os.path.join(d.getVar('LICENSE_DIRECTORY', True),
>>> Will be better to move this code into another function like 
>>> image_list_installed_package to
>>> re-utilize if needed in other parts of the code base.
>>>
>> It's a good idea, let me work in that and I will send another patch 
>> implementing this.
>>
>
> On a second thought, it won't be a reliable because it needs the 
> BB_TASKDEPDATA for do_rootfs.
>
> This is needed because there are some virtual dependency in the image 
> creation, such as virtual/kernel,
> virtual/bootloader and the only way to get what package provides such 
> dependency is using BB_TASKDEPDATA.
> If the function would be called from the do_rootfs task, it would 
> work, otherwise it would fail. So I think
> it's better to keep it this way.

It's that the case, can you define this function outside 
license_create_manifest but in the same file for split the functionality?.

     alimon
>
>>> --------------------------------------------------------------------------------------------------------- 
>>>
>>>> +    # Files that are part of the image but no installed in rootfs
>>>> +    imgpkg_dic = {}
>>>> +    taskdepdata = d.getVar("BB_TASKDEPDATA", True)
>>>> +    all_depends = imagetypes_getdepends(d)
>>>> +    all_depends += " %s" % d.getVar("EXTRA_IMAGEDEPENDS", True)
>>>> +    for depend in all_depends.split():
>>>> +        # Get package name without task
>>>> +        depend = depend.split(":")[0]
>>>> +        if not depend.endswith("native"):
>>>> +            pkgs_file = os.path.join(
>>>> +                    d.getVar('PKGDATA_DIR', True), depend)
>>>> +            # Search for the file on the installed packages with the
>>>> +            # same name as depend if not found fallback to search
>>>> +            # the provider for that depend.
>>>> +            if not os.path.isfile(pkgs_file):
>>>> +                pkgs_file = ""
>>>> +                for taskdep in taskdepdata.itervalues():
>>>> +                    # The fifth field of BB_TASKDEPDATA is PROVIDES
>>>> +                    if depend in taskdep[4]:
>>>> +                        pkgs_file = os.path.join(
>>>> +                                d.getVar('PKGDATA_DIR', True),
>>>> +                                taskdep[0])
>>>> +                        if os.path.isfile(pkgs_file):
>>>> +                            break
>>>> +                        else:
>>>> +                            pkgs_file = ""
>>>> +            if pkgs_file:
>>>> +                pkgs_dep = oe.packagedata.read_pkgdatafile(pkgs_file)
>>>> +                # There is no need to duplicate license info for
>>>> +                # derivated packages or packages in the other
>>>> +                # license manifest
>>>> +                for pkg in pkgs_dep['PACKAGES'].split():
>>>> +                    if (pkg.endswith("-dbg") or
>>>> +                            pkg.endswith("-dev") or
>>>> +                            pkg.endswith("-doc") or
>>>> +                            pkg.endswith("-locale") or
>>>> +                            pkg.endswith("-localedata") or
>>>> +                            pkg.endswith("-staticdev") or
>>>> +                            pkg in pkg_dic.keys()):
>>>> +                        continue
>>>> +                    pkg_data_file = os.path.join(
>>>> +                            d.getVar('PKGDATA_DIR', True),
>>>> +                            "runtime", pkg)
>>>> +                    pkg_data = oe.packagedata.read_pkgdatafile(
>>>> +                            pkg_data_file)
>>>> +                    pkg_pe = pkg_data.get("PE","0")
>>>> +                    if pkg_pe is "0":
>>>> +                        pkg_pf = "%s-%s" % (pkg_data["PV"], 
>>>> pkg_data["PR"])
>>>> +                    else:
>>>> +                        pkg_pf = "%s_%s-%s" % (pkg_pe,
>>>> +                                pkg_data["PV"], pkg_data["PR"])
>>>> +                    # There is no need to add the license for
>>>> +                    # packages that were not deployed
>>>> +                    pkg_deploy = os.path.join(
>>>> +                            d.getVar("TMPDIR", True), "work",
>>>> + d.getVar("MULTIMACH_TARGET_SYS", True),
>>>> +                            pkg_data["PN"], pkg_pf, "temp", 
>>>> "run.do_deploy")
>>>> +                    if os.path.isfile(pkg_deploy):
>>>> +                        imgpkg_dic[pkg] = pkg_data
>>>> +                        if not "LICENSE" in imgpkg_dic[pkg].keys():
>>>> +                            pkg_lic_name = "LICENSE_%s" % pkg
>>>> +                            imgpkg_dic[pkg]["LICENSE"] = \
>>>> + imgpkg_dic[pkg][pkg_lic_name]
>>>> +            # If for some reason, couldn't find the provider
>>>> +            # print a warning to add the license manually
>>>> +            else:
>>>> +                bb.warn("Couldn't find packages that provides "
>>>> +                        "%s, please add licenses manually" % depend)
>>> --------------------------------------------------------------------------------------------------------- 
>>>
>>>> +
>>>> +    rootfs_license_manifest = 
>>>> os.path.join(d.getVar('LICENSE_DIRECTORY', True),
>>>>                           d.getVar('IMAGE_NAME', True), 
>>>> 'license.manifest')
>>>> +    image_license_manifest = 
>>>> os.path.join(d.getVar('LICENSE_DIRECTORY', True),
>>>> +                        d.getVar('IMAGE_NAME', True), 
>>>> 'image_license.manifest')
>>>> +    write_license_files(d, rootfs_license_manifest, pkg_dic)
>>>> +    write_license_files(d, image_license_manifest, imgpkg_dic)
>>>> +}
>>>> +
>>>> +def write_license_files(d, license_manifest, pkg_dic):
>>>> +    import re
>>>> +
>>>> +    bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE", True) or 
>>>> "").split()
>>>> +    bad_licenses = map(lambda l: canonical_license(d, l), 
>>>> bad_licenses)
>>>> +    bad_licenses = expand_wildcard_licenses(d, bad_licenses)
>>>> +
>>>>       with open(license_manifest, "w") as license_file:
>>>>           for pkg in sorted(pkg_dic):
>>>>               if bad_licenses:
>>>> @@ -98,15 +176,16 @@ python license_create_manifest() {
>>>>       if copy_lic_manifest == "1":
>>>>           rootfs_license_dir = 
>>>> os.path.join(d.getVar('IMAGE_ROOTFS', 'True'),
>>>>                                   'usr', 'share', 'common-licenses')
>>>> -        os.makedirs(rootfs_license_dir)
>>>> +        bb.utils.mkdirhier(rootfs_license_dir)
>>>>           rootfs_license_manifest = os.path.join(rootfs_license_dir,
>>>> - 'license.manifest')
>>>> +                os.path.split(license_manifest)[1])
>>> Why split this manifest path?
>>>> +
>>>>           os.link(license_manifest, rootfs_license_manifest)
>>>>             if copy_lic_dirs == "1":
>>>>               for pkg in sorted(pkg_dic):
>>>>                   pkg_rootfs_license_dir = 
>>>> os.path.join(rootfs_license_dir, pkg)
>>>> -                os.makedirs(pkg_rootfs_license_dir)
>>>> +                bb.utils.mkdirhier(pkg_rootfs_license_dir)
>>>>                   pkg_license_dir = 
>>>> os.path.join(d.getVar('LICENSE_DIRECTORY', True),
>>>> pkg_dic[pkg]["PN"])
>>>>                   licenses = os.listdir(pkg_license_dir)
>>>> @@ -124,14 +203,16 @@ python license_create_manifest() {
>>>>                           if not os.path.exists(rootfs_license):
>>>>                               os.link(pkg_license, rootfs_license)
>>>>   -                        os.symlink(os.path.join('..', lic), 
>>>> pkg_rootfs_license)
>>>> +                        if not os.path.exists(pkg_rootfs_license):
>>> Why test if exist?, this code creates symlink into recipe license.
>>>> + os.symlink(os.path.join('..', lic), pkg_rootfs_license)
>>>>                       else:
>>>> -                        if oe.license.license_ok(canonical_license(d,
>>>> -                            lic), bad_licenses) == False:
>>>> +                        if 
>>>> ((oe.license.license_ok(canonical_license(d,
>>>> +                            lic), bad_licenses) == False) or
>>>> + os.path.exists(pkg_rootfs_license)):
>>> Why test if exist?.
>>>>                               continue
>>>>                             os.link(pkg_license, pkg_rootfs_license)
>>>> -}
>>>> +
>>>>     python do_populate_lic() {
>>>>       """
>>>
>>> Cheers,
>>>     alimon
>>>
>> Regards,
>>
>> Mariano Lopez
> Regards,
>
> Mariano Lopez