Re: [PATCH 1/4] license_class: Reimplemented manifest creation in python

[Richard Purdie <richard.purdie@linuxfoundation.org>]

On Wed, 2014-10-29 at 12:34 -0600, Aníbal Limón wrote:
> Reimplemented license_manifest_create from shell to python, in
> order to use oe.license module for handle INCOMPATIBLE_LICENSE
> and add License prorities into OR's evaluation.
> 
> Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
> ---
>  meta/classes/license.bbclass | 161 ++++++++++++++++++++++++-------------------
>  1 file changed, 90 insertions(+), 71 deletions(-)
> 
> diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
> index a34ea39..c8a86ce 100644
> --- a/meta/classes/license.bbclass
> +++ b/meta/classes/license.bbclass
> @@ -25,78 +25,97 @@ python write_package_manifest() {
>          'w+').write(image_list_installed_packages(d))
>  }
>  
> -license_create_manifest() {
> -        # Test if BUILD_IMAGES_FROM_FEEDS is defined in env
> -        if [ -n "${BUILD_IMAGES_FROM_FEEDS}" ]; then
> -          exit 0
> -        fi
> -
> -	INSTALLED_PKGS=`cat ${LICENSE_DIRECTORY}/${IMAGE_NAME}/package.manifest`
> -	LICENSE_MANIFEST="${LICENSE_DIRECTORY}/${IMAGE_NAME}/license.manifest"
> -	# remove existing license.manifest file
> -	if [ -f ${LICENSE_MANIFEST} ]; then
> -		rm ${LICENSE_MANIFEST}
> -	fi
> -	touch ${LICENSE_MANIFEST}
> -	for pkg in ${INSTALLED_PKGS}; do
> -		filename=`ls ${PKGDATA_DIR}/runtime-reverse/${pkg}| head -1`
> -		pkged_pn="$(sed -n 's/^PN: //p' ${filename})"
> -
> -		# check to see if the package name exists in the manifest. if so, bail.
> -		if grep -q "^PACKAGE NAME: ${pkg}" ${LICENSE_MANIFEST}; then
> -			continue
> -		fi
> -
> -		pkged_pv="$(sed -n 's/^PV: //p' ${filename})"
> -		pkged_name="$(basename $(readlink ${filename}))"
> -		pkged_lic="$(sed -n "/^LICENSE_${pkged_name}: /{ s/^LICENSE_${pkged_name}: //; s/[|&()*]/ /g; s/  */ /g; p }" ${filename})"
> -		if [ -z ${pkged_lic} ]; then
> -			# fallback checking value of LICENSE
> -			pkged_lic="$(sed -n "/^LICENSE: /{ s/^LICENSE: //; s/[|&()*]/ /g; s/  */ /g; p }" ${filename})"
> -		fi
> -
> -		echo "PACKAGE NAME:" ${pkg} >> ${LICENSE_MANIFEST}
> -		echo "PACKAGE VERSION:" ${pkged_pv} >> ${LICENSE_MANIFEST}
> -		echo "RECIPE NAME:" ${pkged_pn} >> ${LICENSE_MANIFEST}
> -		printf "LICENSE:" >> ${LICENSE_MANIFEST}
> -		for lic in ${pkged_lic}; do
> -			# to reference a license file trim trailing + symbol
> -			if ! [ -e "${LICENSE_DIRECTORY}/${pkged_pn}/generic_${lic%+}" ]; then
> -				bbwarn "The license listed ${lic} was not in the licenses collected for ${pkged_pn}"
> -			fi
> -                        printf " ${lic}" >> ${LICENSE_MANIFEST}
> -		done
> -		printf "\n\n" >> ${LICENSE_MANIFEST}
> -	done
> -
> -	# Two options here:
> -	# - Just copy the manifest
> -	# - Copy the manifest and the license directories
> -	# With both options set we see a .5 M increase in core-image-minimal
> -	if [ "${COPY_LIC_MANIFEST}" = "1" ]; then
> -		mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/
> -		cp ${LICENSE_MANIFEST} ${IMAGE_ROOTFS}/usr/share/common-licenses/license.manifest
> -		if [ "${COPY_LIC_DIRS}" = "1" ]; then
> -			for pkg in ${INSTALLED_PKGS}; do
> -				mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}
> -				pkged_pn="$(oe-pkgdata-util lookup-recipe ${PKGDATA_DIR} ${pkg})"
> -				for lic in `ls ${LICENSE_DIRECTORY}/${pkged_pn}`; do
> -					# Really don't need to copy the generics as they're 
> -					# represented in the manifest and in the actual pkg licenses
> -					# Doing so would make your image quite a bit larger
> -					if [ "${lic#generic_}" = "${lic}" ]; then
> -						cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
> -					else
> -						if [ ! -f ${IMAGE_ROOTFS}/usr/share/common-licenses/${lic} ]; then
> -							cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/
> -						fi
> -						ln -sf ../${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
> -					fi
> -				done
> -			done
> -		fi
> -	fi
> +python license_create_manifest() {
> +    import shutil
> +    import re
>  
> +    build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', True)
> +    if build_images_from_feeds == "1":
> +        return 0
> +
> +    pkg_dic = {}
> +    package_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
> +                        d.getVar('IMAGE_NAME', True), 'package.manifest')
> +    with open(package_manifest, "r") as package_file:
> +        pkg_list = package_file.read().split()
> +        for pkg in pkg_list:
> +            pkg_info = os.path.join(d.getVar('PKGDATA_DIR', True),
> +                                    'runtime-reverse', pkg)
> +            with open(pkg_info, "r") as pkg_info_file:
> +                pkg_dic[pkg] = {}
> +
> +                pkg_lic_name = os.path.basename(os.readlink(pkg_info))
> +
> +                for line in pkg_info_file.read().split("\n"):
> +                    if re.match("^PN: .*$", line):
> +                        pkg_dic[pkg]["PN"] = re.search("PN: (.*)$", line).group(1)
> +
> +                    if re.match("^PV: .*$", line):
> +                        pkg_dic[pkg]["PV"] = re.search("PV: (.*)$", line).group(1)
> +
> +                    if re.match("^LICENSE_%s: (.*)$" % re.escape(pkg_lic_name)
> +                                , line):
> +                        pkg_dic[pkg]["LICENSE"] = re.search("^LICENSE_%s: (.*)$"
> +                                    % pkg_lic_name, line).group(1)
> +                    elif re.match("^LICENSE: (.*)$", line):
> +                        pkg_dic[pkg]["LICENSE"] = re.search("^LICENSE: (.*)$",
> +                                    line).group(1)

Could you have a look at oe.packagedata.read_pkgdatafile(x) and see if
the above could use that instead please?

> +    license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
> +                        d.getVar('IMAGE_NAME', True), 'license.manifest')
> +    with open(license_manifest, "w") as license_file:
> +        for pkg in sorted(pkg_dic):
> +            license_file.write("PACKAGE NAME: %s\n" % pkg)
> +            license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
> +            license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
> +            license_file.write("LICENSE:")
> +
> +            licenses = re.sub('[|&()*]', '', pkg_dic[pkg]["LICENSE"])
> +            licenses = re.sub('  *', ' ', licenses)
> +            for lic in licenses.split():
> +                lic = re.sub('\+', '', lic)
> +                lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
> +                                        pkg_dic[pkg]["PN"], "generic_%s" % lic)
> +                if not os.path.exists(lic_file):
> +                   bb.warn("The license listed %s was not in the "\ 
> +                            "licenses collected for recipe %s" 
> +                            % (lic, pkg_dic[pkg]["PN"]))
> +                license_file.write(" %s" % lic)
> +            license_file.write("\n\n")
> +
> +    # Two options here:
> +    # - Just copy the manifest
> +    # - Copy the manifest and the license directories
> +    # With both options set we see a .5 M increase in core-image-minimal
> +    copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST', True)
> +    copy_lic_dirs = d.getVar('COPY_LIC_DIRS', True)
> +    if copy_lic_manifest == "1":
> +        rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS', 'True'), 
> +                                'usr', 'share', 'common-licenses')
> +        os.makedirs(rootfs_license_dir)
> +        rootfs_license_manifest = os.path.join(rootfs_license_dir,
> +                                                'license.manifest')
> +        shutil.copyfile(license_manifest, rootfs_license_manifest)
> +
> +        if copy_lic_dirs == "1":
> +            for pkg in sorted(pkg_dic):
> +                pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
> +                os.makedirs(pkg_rootfs_license_dir)
> +                pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
> +                                            pkg_dic[pkg]["PN"]) 
> +                licenses = os.listdir(pkg_license_dir)
> +                for lic in licenses:
> +                    rootfs_license = os.path.join(rootfs_license_dir, lic)
> +                    pkg_license = os.path.join(pkg_license_dir, lic)
> +                    pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
> +
> +                    if re.match("^generic_.*$", lic):
> +                        if not os.path.exists(pkg_rootfs_license):
> +                            shutil.copyfile(pkg_license, rootfs_license)
> +
> +                        os.symlink(os.path.join('..', lic), pkg_rootfs_license)
> +                    else:
> +                        shutil.copyfile(pkg_license, pkg_rootfs_license)

One small tweak we could make here is to hardlink these files. There are
several copies of many of them and it would reduce image size a bit. I
appreciate the original doesn't but I've been doing this in other places
and this is probably another we should put on the list...

Otherwise looks good, thanks.

Richard