* [PATCH 1/3] license_class: Reimplemented manifest creation in python
2015-05-06 14:52 [PATCH 0/3] Add support to handle INCOMPATIBLE_LICENSE Aníbal Limón
@ 2015-05-06 14:52 ` Aníbal Limón
2015-05-08 17:30 ` Flanagan, Elizabeth
2015-05-06 14:52 ` [PATCH 2/3] license_class: Generalize license_ok function Aníbal Limón
2015-05-06 14:52 ` [PATCH 3/3] license: Add support for handle INCOMPATIBLE_LICENSE in manifest creation Aníbal Limón
2 siblings, 1 reply; 6+ messages in thread
From: Aníbal Limón @ 2015-05-06 14:52 UTC (permalink / raw)
To: openembedded-core
Reimplemented license_manifest_create from shell to python for
INCOMPATIBLE_LICENSE handle using oe.license module.
Optimizations are made to avoid license copy now uses a hardlink
and symbolic link this helps to save space during build.
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
meta/classes/license.bbclass | 163 ++++++++++++++++++++++---------------------
1 file changed, 82 insertions(+), 81 deletions(-)
diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index d9409a9..975867d 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -25,87 +25,88 @@ python write_package_manifest() {
'w+').write(image_list_installed_packages(d))
}
-license_create_manifest() {
- # Test if BUILD_IMAGES_FROM_FEEDS is defined in env
- if [ -n "${BUILD_IMAGES_FROM_FEEDS}" ]; then
- exit 0
- fi
-
- INSTALLED_PKGS=`cat ${LICENSE_DIRECTORY}/${IMAGE_NAME}/package.manifest`
- LICENSE_MANIFEST="${LICENSE_DIRECTORY}/${IMAGE_NAME}/license.manifest"
- # remove existing license.manifest file
- if [ -f ${LICENSE_MANIFEST} ]; then
- rm ${LICENSE_MANIFEST}
- fi
- touch ${LICENSE_MANIFEST}
- for pkg in ${INSTALLED_PKGS}; do
- filename=`ls ${PKGDATA_DIR}/runtime-reverse/${pkg}| head -1`
- pkged_pn="$(sed -n 's/^PN: //p' ${filename})"
-
- # check to see if the package name exists in the manifest. if so, bail.
- if grep -q "^PACKAGE NAME: ${pkg}" ${LICENSE_MANIFEST}; then
- continue
- fi
-
- pkged_pv="$(sed -n 's/^PV: //p' ${filename})"
- pkged_name="$(basename $(readlink ${filename}))"
- pkged_lic="$(sed -n "/^LICENSE_${pkged_name}: /{ s/^LICENSE_${pkged_name}: //; p }" ${filename})"
- pkged_size="$(sed -n "/^PKGSIZE_${pkged_name}: /{ s/^PKGSIZE_${pkged_name}: //; p }" ${filename})"
- if [ -z "${pkged_lic}" ]; then
- # fallback checking value of LICENSE
- pkged_lic="$(sed -n "/^LICENSE: /{ s/^LICENSE: //; p }" ${filename})"
- fi
-
- echo "PACKAGE NAME:" ${pkg} >> ${LICENSE_MANIFEST}
- echo "PACKAGE VERSION:" ${pkged_pv} >> ${LICENSE_MANIFEST}
- echo "RECIPE NAME:" ${pkged_pn} >> ${LICENSE_MANIFEST}
- echo "LICENSE:" ${pkged_lic} >> ${LICENSE_MANIFEST}
- echo "" >> ${LICENSE_MANIFEST}
-
- # If the package doesn't contain any file, that is, its size is 0, the license
- # isn't relevant as far as the final image is concerned. So doing license check
- # doesn't make much sense, skip it.
- if [ "$pkged_size" = "0" ]; then
- continue
- fi
-
- lics="$(echo ${pkged_lic} | sed "s/[|&()*]/ /g" | sed "s/ */ /g" )"
- for lic in ${lics}; do
- # to reference a license file trim trailing + symbol
- if ! [ -e "${LICENSE_DIRECTORY}/${pkged_pn}/generic_${lic%+}" ]; then
- bbwarn "The license listed ${lic} was not in the licenses collected for ${pkged_pn}"
- fi
- done
- done
-
- # Two options here:
- # - Just copy the manifest
- # - Copy the manifest and the license directories
- # With both options set we see a .5 M increase in core-image-minimal
- if [ "${COPY_LIC_MANIFEST}" = "1" ]; then
- mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/
- cp ${LICENSE_MANIFEST} ${IMAGE_ROOTFS}/usr/share/common-licenses/license.manifest
- if [ "${COPY_LIC_DIRS}" = "1" ]; then
- for pkg in ${INSTALLED_PKGS}; do
- mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}
- pkged_pn="$(oe-pkgdata-util -p ${PKGDATA_DIR} lookup-recipe ${pkg})"
- for lic in `ls ${LICENSE_DIRECTORY}/${pkged_pn}`; do
- # Really don't need to copy the generics as they're
- # represented in the manifest and in the actual pkg licenses
- # Doing so would make your image quite a bit larger
- if [ "${lic#generic_}" = "${lic}" ]; then
- cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
- else
- if [ ! -f ${IMAGE_ROOTFS}/usr/share/common-licenses/${lic} ]; then
- cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/
- fi
- ln -sf ../${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
- fi
- done
- done
- fi
- fi
-
+python license_create_manifest() {
+ import re
+ import oe.packagedata
+
+ build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', True)
+ if build_images_from_feeds == "1":
+ return 0
+
+ pkg_dic = {}
+ package_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+ d.getVar('IMAGE_NAME', True), 'package.manifest')
+ with open(package_manifest, "r") as package_file:
+ pkg_list = package_file.read().split()
+ for pkg in pkg_list:
+ pkg_info = os.path.join(d.getVar('PKGDATA_DIR', True),
+ 'runtime-reverse', pkg)
+ pkg_name = os.path.basename(os.readlink(pkg_info))
+
+ pkg_dic[pkg_name] = oe.packagedata.read_pkgdatafile(pkg_info)
+ if not "LICENSE" in pkg_dic[pkg_name].keys():
+ pkg_lic_name = "LICENSE_" + pkg_name
+ pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name]
+
+ license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+ d.getVar('IMAGE_NAME', True), 'license.manifest')
+ with open(license_manifest, "w") as license_file:
+ for pkg in sorted(pkg_dic):
+ license_file.write("PACKAGE NAME: %s\n" % pkg)
+ license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
+ license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
+ license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"])
+
+ # If the package doesn't contain any file, that is, its size is 0, the license
+ # isn't relevant as far as the final image is concerned. So doing license check
+ # doesn't make much sense, skip it.
+ if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
+ continue
+
+ licenses = re.sub('[|&()*]', '', pkg_dic[pkg]["LICENSE"])
+ licenses = re.sub(' *', ' ', licenses)
+ for lic in licenses.split():
+ lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+ pkg_dic[pkg]["PN"], "generic_%s" %
+ re.sub('\+', '', lic))
+ if not os.path.exists(lic_file):
+ bb.warn("The license listed %s was not in the "\
+ "licenses collected for recipe %s"
+ % (lic, pkg_dic[pkg]["PN"]))
+
+ # Two options here:
+ # - Just copy the manifest
+ # - Copy the manifest and the license directories
+ # With both options set we see a .5 M increase in core-image-minimal
+ copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST', True)
+ copy_lic_dirs = d.getVar('COPY_LIC_DIRS', True)
+ if copy_lic_manifest == "1":
+ rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS', 'True'),
+ 'usr', 'share', 'common-licenses')
+ os.makedirs(rootfs_license_dir)
+ rootfs_license_manifest = os.path.join(rootfs_license_dir,
+ 'license.manifest')
+ os.link(license_manifest, rootfs_license_manifest)
+
+ if copy_lic_dirs == "1":
+ for pkg in sorted(pkg_dic):
+ pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
+ os.makedirs(pkg_rootfs_license_dir)
+ pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+ pkg_dic[pkg]["PN"])
+ licenses = os.listdir(pkg_license_dir)
+ for lic in licenses:
+ rootfs_license = os.path.join(rootfs_license_dir, lic)
+ pkg_license = os.path.join(pkg_license_dir, lic)
+ pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
+
+ if re.match("^generic_.*$", lic):
+ if not os.path.exists(rootfs_license):
+ os.link(pkg_license, rootfs_license)
+
+ os.symlink(os.path.join('..', lic), pkg_rootfs_license)
+ else:
+ os.link(pkg_license, pkg_rootfs_license)
}
python do_populate_lic() {
--
1.8.4.5
^ permalink raw reply related [flat|nested] 6+ messages in thread* Re: [PATCH 1/3] license_class: Reimplemented manifest creation in python
2015-05-06 14:52 ` [PATCH 1/3] license_class: Reimplemented manifest creation in python Aníbal Limón
@ 2015-05-08 17:30 ` Flanagan, Elizabeth
2015-05-08 20:38 ` Aníbal Limón
0 siblings, 1 reply; 6+ messages in thread
From: Flanagan, Elizabeth @ 2015-05-08 17:30 UTC (permalink / raw)
To: Aníbal Limón; +Cc: Patches and discussions about the oe-core layer
On 6 May 2015 at 15:52, Aníbal Limón <anibal.limon@linux.intel.com> wrote:
> Reimplemented license_manifest_create from shell to python for
> INCOMPATIBLE_LICENSE handle using oe.license module.
>
> Optimizations are made to avoid license copy now uses a hardlink
> and symbolic link this helps to save space during build.
>
> Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
> ---
> meta/classes/license.bbclass | 163 ++++++++++++++++++++++---------------------
> 1 file changed, 82 insertions(+), 81 deletions(-)
>
> diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
> index d9409a9..975867d 100644
> --- a/meta/classes/license.bbclass
> +++ b/meta/classes/license.bbclass
> @@ -25,87 +25,88 @@ python write_package_manifest() {
> 'w+').write(image_list_installed_packages(d))
> }
>
> -license_create_manifest() {
> - # Test if BUILD_IMAGES_FROM_FEEDS is defined in env
> - if [ -n "${BUILD_IMAGES_FROM_FEEDS}" ]; then
> - exit 0
> - fi
> -
> - INSTALLED_PKGS=`cat ${LICENSE_DIRECTORY}/${IMAGE_NAME}/package.manifest`
> - LICENSE_MANIFEST="${LICENSE_DIRECTORY}/${IMAGE_NAME}/license.manifest"
> - # remove existing license.manifest file
> - if [ -f ${LICENSE_MANIFEST} ]; then
> - rm ${LICENSE_MANIFEST}
> - fi
> - touch ${LICENSE_MANIFEST}
> - for pkg in ${INSTALLED_PKGS}; do
> - filename=`ls ${PKGDATA_DIR}/runtime-reverse/${pkg}| head -1`
> - pkged_pn="$(sed -n 's/^PN: //p' ${filename})"
> -
> - # check to see if the package name exists in the manifest. if so, bail.
> - if grep -q "^PACKAGE NAME: ${pkg}" ${LICENSE_MANIFEST}; then
> - continue
> - fi
> -
> - pkged_pv="$(sed -n 's/^PV: //p' ${filename})"
> - pkged_name="$(basename $(readlink ${filename}))"
> - pkged_lic="$(sed -n "/^LICENSE_${pkged_name}: /{ s/^LICENSE_${pkged_name}: //; p }" ${filename})"
> - pkged_size="$(sed -n "/^PKGSIZE_${pkged_name}: /{ s/^PKGSIZE_${pkged_name}: //; p }" ${filename})"
> - if [ -z "${pkged_lic}" ]; then
> - # fallback checking value of LICENSE
> - pkged_lic="$(sed -n "/^LICENSE: /{ s/^LICENSE: //; p }" ${filename})"
> - fi
> -
> - echo "PACKAGE NAME:" ${pkg} >> ${LICENSE_MANIFEST}
> - echo "PACKAGE VERSION:" ${pkged_pv} >> ${LICENSE_MANIFEST}
> - echo "RECIPE NAME:" ${pkged_pn} >> ${LICENSE_MANIFEST}
> - echo "LICENSE:" ${pkged_lic} >> ${LICENSE_MANIFEST}
> - echo "" >> ${LICENSE_MANIFEST}
> -
> - # If the package doesn't contain any file, that is, its size is 0, the license
> - # isn't relevant as far as the final image is concerned. So doing license check
> - # doesn't make much sense, skip it.
> - if [ "$pkged_size" = "0" ]; then
> - continue
> - fi
> -
> - lics="$(echo ${pkged_lic} | sed "s/[|&()*]/ /g" | sed "s/ */ /g" )"
> - for lic in ${lics}; do
> - # to reference a license file trim trailing + symbol
> - if ! [ -e "${LICENSE_DIRECTORY}/${pkged_pn}/generic_${lic%+}" ]; then
> - bbwarn "The license listed ${lic} was not in the licenses collected for ${pkged_pn}"
> - fi
> - done
> - done
> -
> - # Two options here:
> - # - Just copy the manifest
> - # - Copy the manifest and the license directories
> - # With both options set we see a .5 M increase in core-image-minimal
> - if [ "${COPY_LIC_MANIFEST}" = "1" ]; then
> - mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/
> - cp ${LICENSE_MANIFEST} ${IMAGE_ROOTFS}/usr/share/common-licenses/license.manifest
> - if [ "${COPY_LIC_DIRS}" = "1" ]; then
> - for pkg in ${INSTALLED_PKGS}; do
> - mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}
> - pkged_pn="$(oe-pkgdata-util -p ${PKGDATA_DIR} lookup-recipe ${pkg})"
> - for lic in `ls ${LICENSE_DIRECTORY}/${pkged_pn}`; do
> - # Really don't need to copy the generics as they're
> - # represented in the manifest and in the actual pkg licenses
> - # Doing so would make your image quite a bit larger
> - if [ "${lic#generic_}" = "${lic}" ]; then
> - cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
> - else
> - if [ ! -f ${IMAGE_ROOTFS}/usr/share/common-licenses/${lic} ]; then
> - cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/
> - fi
> - ln -sf ../${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
> - fi
> - done
> - done
> - fi
> - fi
> -
> +python license_create_manifest() {
> + import re
> + import oe.packagedata
> +
> + build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', True)
> + if build_images_from_feeds == "1":
> + return 0
> +
> + pkg_dic = {}
> + package_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
> + d.getVar('IMAGE_NAME', True), 'package.manifest')
Why aren't you using [image|sdk]_list_installed_packages to get the
package list here?
> + with open(package_manifest, "r") as package_file:
> + pkg_list = package_file.read().split()
> + for pkg in pkg_list:
> + pkg_info = os.path.join(d.getVar('PKGDATA_DIR', True),
> + 'runtime-reverse', pkg)
> + pkg_name = os.path.basename(os.readlink(pkg_info))
> +
> + pkg_dic[pkg_name] = oe.packagedata.read_pkgdatafile(pkg_info)
> + if not "LICENSE" in pkg_dic[pkg_name].keys():
> + pkg_lic_name = "LICENSE_" + pkg_name
> + pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name]
> +
> + license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
> + d.getVar('IMAGE_NAME', True), 'license.manifest')
> + with open(license_manifest, "w") as license_file:
> + for pkg in sorted(pkg_dic):
> + license_file.write("PACKAGE NAME: %s\n" % pkg)
> + license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
> + license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
> + license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"])
> +
> + # If the package doesn't contain any file, that is, its size is 0, the license
> + # isn't relevant as far as the final image is concerned. So doing license check
> + # doesn't make much sense, skip it.
> + if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
> + continue
> +
> + licenses = re.sub('[|&()*]', '', pkg_dic[pkg]["LICENSE"])
> + licenses = re.sub(' *', ' ', licenses)
> + for lic in licenses.split():
> + lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
> + pkg_dic[pkg]["PN"], "generic_%s" %
> + re.sub('\+', '', lic))
> + if not os.path.exists(lic_file):
> + bb.warn("The license listed %s was not in the "\
> + "licenses collected for recipe %s"
> + % (lic, pkg_dic[pkg]["PN"]))
> +
> + # Two options here:
> + # - Just copy the manifest
> + # - Copy the manifest and the license directories
> + # With both options set we see a .5 M increase in core-image-minimal
> + copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST', True)
> + copy_lic_dirs = d.getVar('COPY_LIC_DIRS', True)
> + if copy_lic_manifest == "1":
> + rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS', 'True'),
> + 'usr', 'share', 'common-licenses')
> + os.makedirs(rootfs_license_dir)
> + rootfs_license_manifest = os.path.join(rootfs_license_dir,
> + 'license.manifest')
> + os.link(license_manifest, rootfs_license_manifest)
> +
> + if copy_lic_dirs == "1":
> + for pkg in sorted(pkg_dic):
> + pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
> + os.makedirs(pkg_rootfs_license_dir)
> + pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
> + pkg_dic[pkg]["PN"])
> + licenses = os.listdir(pkg_license_dir)
> + for lic in licenses:
> + rootfs_license = os.path.join(rootfs_license_dir, lic)
> + pkg_license = os.path.join(pkg_license_dir, lic)
> + pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
> +
> + if re.match("^generic_.*$", lic):
> + if not os.path.exists(rootfs_license):
> + os.link(pkg_license, rootfs_license)
> +
> + os.symlink(os.path.join('..', lic), pkg_rootfs_license)
> + else:
> + os.link(pkg_license, pkg_rootfs_license)
> }
>
> python do_populate_lic() {
> --
> 1.8.4.5
>
--
Elizabeth Flanagan
Yocto Project
Build and Release
^ permalink raw reply [flat|nested] 6+ messages in thread* Re: [PATCH 1/3] license_class: Reimplemented manifest creation in python
2015-05-08 17:30 ` Flanagan, Elizabeth
@ 2015-05-08 20:38 ` Aníbal Limón
0 siblings, 0 replies; 6+ messages in thread
From: Aníbal Limón @ 2015-05-08 20:38 UTC (permalink / raw)
To: Flanagan, Elizabeth; +Cc: Patches and discussions about the oe-core layer
Hi,
I tested also using SSTATE and works good, see my comments below.
On 08/05/15 12:30, Flanagan, Elizabeth wrote:
> On 6 May 2015 at 15:52, Aníbal Limón <anibal.limon@linux.intel.com> wrote:
>> Reimplemented license_manifest_create from shell to python for
>> INCOMPATIBLE_LICENSE handle using oe.license module.
>>
>> Optimizations are made to avoid license copy now uses a hardlink
>> and symbolic link this helps to save space during build.
>>
>> Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
>> ---
>> meta/classes/license.bbclass | 163 ++++++++++++++++++++++---------------------
>> 1 file changed, 82 insertions(+), 81 deletions(-)
>>
>> diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
>> index d9409a9..975867d 100644
>> --- a/meta/classes/license.bbclass
>> +++ b/meta/classes/license.bbclass
>> @@ -25,87 +25,88 @@ python write_package_manifest() {
>> 'w+').write(image_list_installed_packages(d))
>> }
>>
>> -license_create_manifest() {
>> - # Test if BUILD_IMAGES_FROM_FEEDS is defined in env
>> - if [ -n "${BUILD_IMAGES_FROM_FEEDS}" ]; then
>> - exit 0
>> - fi
>> -
>> - INSTALLED_PKGS=`cat ${LICENSE_DIRECTORY}/${IMAGE_NAME}/package.manifest`
>> - LICENSE_MANIFEST="${LICENSE_DIRECTORY}/${IMAGE_NAME}/license.manifest"
>> - # remove existing license.manifest file
>> - if [ -f ${LICENSE_MANIFEST} ]; then
>> - rm ${LICENSE_MANIFEST}
>> - fi
>> - touch ${LICENSE_MANIFEST}
>> - for pkg in ${INSTALLED_PKGS}; do
>> - filename=`ls ${PKGDATA_DIR}/runtime-reverse/${pkg}| head -1`
>> - pkged_pn="$(sed -n 's/^PN: //p' ${filename})"
>> -
>> - # check to see if the package name exists in the manifest. if so, bail.
>> - if grep -q "^PACKAGE NAME: ${pkg}" ${LICENSE_MANIFEST}; then
>> - continue
>> - fi
>> -
>> - pkged_pv="$(sed -n 's/^PV: //p' ${filename})"
>> - pkged_name="$(basename $(readlink ${filename}))"
>> - pkged_lic="$(sed -n "/^LICENSE_${pkged_name}: /{ s/^LICENSE_${pkged_name}: //; p }" ${filename})"
>> - pkged_size="$(sed -n "/^PKGSIZE_${pkged_name}: /{ s/^PKGSIZE_${pkged_name}: //; p }" ${filename})"
>> - if [ -z "${pkged_lic}" ]; then
>> - # fallback checking value of LICENSE
>> - pkged_lic="$(sed -n "/^LICENSE: /{ s/^LICENSE: //; p }" ${filename})"
>> - fi
>> -
>> - echo "PACKAGE NAME:" ${pkg} >> ${LICENSE_MANIFEST}
>> - echo "PACKAGE VERSION:" ${pkged_pv} >> ${LICENSE_MANIFEST}
>> - echo "RECIPE NAME:" ${pkged_pn} >> ${LICENSE_MANIFEST}
>> - echo "LICENSE:" ${pkged_lic} >> ${LICENSE_MANIFEST}
>> - echo "" >> ${LICENSE_MANIFEST}
>> -
>> - # If the package doesn't contain any file, that is, its size is 0, the license
>> - # isn't relevant as far as the final image is concerned. So doing license check
>> - # doesn't make much sense, skip it.
>> - if [ "$pkged_size" = "0" ]; then
>> - continue
>> - fi
>> -
>> - lics="$(echo ${pkged_lic} | sed "s/[|&()*]/ /g" | sed "s/ */ /g" )"
>> - for lic in ${lics}; do
>> - # to reference a license file trim trailing + symbol
>> - if ! [ -e "${LICENSE_DIRECTORY}/${pkged_pn}/generic_${lic%+}" ]; then
>> - bbwarn "The license listed ${lic} was not in the licenses collected for ${pkged_pn}"
>> - fi
>> - done
>> - done
>> -
>> - # Two options here:
>> - # - Just copy the manifest
>> - # - Copy the manifest and the license directories
>> - # With both options set we see a .5 M increase in core-image-minimal
>> - if [ "${COPY_LIC_MANIFEST}" = "1" ]; then
>> - mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/
>> - cp ${LICENSE_MANIFEST} ${IMAGE_ROOTFS}/usr/share/common-licenses/license.manifest
>> - if [ "${COPY_LIC_DIRS}" = "1" ]; then
>> - for pkg in ${INSTALLED_PKGS}; do
>> - mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}
>> - pkged_pn="$(oe-pkgdata-util -p ${PKGDATA_DIR} lookup-recipe ${pkg})"
>> - for lic in `ls ${LICENSE_DIRECTORY}/${pkged_pn}`; do
>> - # Really don't need to copy the generics as they're
>> - # represented in the manifest and in the actual pkg licenses
>> - # Doing so would make your image quite a bit larger
>> - if [ "${lic#generic_}" = "${lic}" ]; then
>> - cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
>> - else
>> - if [ ! -f ${IMAGE_ROOTFS}/usr/share/common-licenses/${lic} ]; then
>> - cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/
>> - fi
>> - ln -sf ../${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
>> - fi
>> - done
>> - done
>> - fi
>> - fi
>> -
>> +python license_create_manifest() {
>> + import re
>> + import oe.packagedata
>> +
>> + build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', True)
>> + if build_images_from_feeds == "1":
>> + return 0
>> +
>> + pkg_dic = {}
>> + package_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
>> + d.getVar('IMAGE_NAME', True), 'package.manifest')
> Why aren't you using [image|sdk]_list_installed_packages to get the
> package list here?
I made this change sending v3.
Cheers,
alimon
>> + with open(package_manifest, "r") as package_file:
>> + pkg_list = package_file.read().split()
>> + for pkg in pkg_list:
>> + pkg_info = os.path.join(d.getVar('PKGDATA_DIR', True),
>> + 'runtime-reverse', pkg)
>> + pkg_name = os.path.basename(os.readlink(pkg_info))
>> +
>> + pkg_dic[pkg_name] = oe.packagedata.read_pkgdatafile(pkg_info)
>> + if not "LICENSE" in pkg_dic[pkg_name].keys():
>> + pkg_lic_name = "LICENSE_" + pkg_name
>> + pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name]
>> +
>> + license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
>> + d.getVar('IMAGE_NAME', True), 'license.manifest')
>> + with open(license_manifest, "w") as license_file:
>> + for pkg in sorted(pkg_dic):
>> + license_file.write("PACKAGE NAME: %s\n" % pkg)
>> + license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
>> + license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
>> + license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"])
>> +
>> + # If the package doesn't contain any file, that is, its size is 0, the license
>> + # isn't relevant as far as the final image is concerned. So doing license check
>> + # doesn't make much sense, skip it.
>> + if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
>> + continue
>> +
>> + licenses = re.sub('[|&()*]', '', pkg_dic[pkg]["LICENSE"])
>> + licenses = re.sub(' *', ' ', licenses)
>> + for lic in licenses.split():
>> + lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
>> + pkg_dic[pkg]["PN"], "generic_%s" %
>> + re.sub('\+', '', lic))
>> + if not os.path.exists(lic_file):
>> + bb.warn("The license listed %s was not in the "\
>> + "licenses collected for recipe %s"
>> + % (lic, pkg_dic[pkg]["PN"]))
>> +
>> + # Two options here:
>> + # - Just copy the manifest
>> + # - Copy the manifest and the license directories
>> + # With both options set we see a .5 M increase in core-image-minimal
>> + copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST', True)
>> + copy_lic_dirs = d.getVar('COPY_LIC_DIRS', True)
>> + if copy_lic_manifest == "1":
>> + rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS', 'True'),
>> + 'usr', 'share', 'common-licenses')
>> + os.makedirs(rootfs_license_dir)
>> + rootfs_license_manifest = os.path.join(rootfs_license_dir,
>> + 'license.manifest')
>> + os.link(license_manifest, rootfs_license_manifest)
>> +
>> + if copy_lic_dirs == "1":
>> + for pkg in sorted(pkg_dic):
>> + pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
>> + os.makedirs(pkg_rootfs_license_dir)
>> + pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
>> + pkg_dic[pkg]["PN"])
>> + licenses = os.listdir(pkg_license_dir)
>> + for lic in licenses:
>> + rootfs_license = os.path.join(rootfs_license_dir, lic)
>> + pkg_license = os.path.join(pkg_license_dir, lic)
>> + pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
>> +
>> + if re.match("^generic_.*$", lic):
>> + if not os.path.exists(rootfs_license):
>> + os.link(pkg_license, rootfs_license)
>> +
>> + os.symlink(os.path.join('..', lic), pkg_rootfs_license)
>> + else:
>> + os.link(pkg_license, pkg_rootfs_license)
>> }
>>
>> python do_populate_lic() {
>> --
>> 1.8.4.5
>>
>
>
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH 2/3] license_class: Generalize license_ok function
2015-05-06 14:52 [PATCH 0/3] Add support to handle INCOMPATIBLE_LICENSE Aníbal Limón
2015-05-06 14:52 ` [PATCH 1/3] license_class: Reimplemented manifest creation in python Aníbal Limón
@ 2015-05-06 14:52 ` Aníbal Limón
2015-05-06 14:52 ` [PATCH 3/3] license: Add support for handle INCOMPATIBLE_LICENSE in manifest creation Aníbal Limón
2 siblings, 0 replies; 6+ messages in thread
From: Aníbal Limón @ 2015-05-06 14:52 UTC (permalink / raw)
To: openembedded-core
Add dont_want_licenses as parameter to license_ok function and move it
to oe.license module in order to use in other modules.
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
meta/classes/license.bbclass | 21 ++++-----------------
meta/lib/oe/license.py | 14 ++++++++++++++
2 files changed, 18 insertions(+), 17 deletions(-)
diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index 975867d..780b9d5 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -341,36 +341,23 @@ def incompatible_license(d, dont_want_licenses, package=None):
take into consideration 'or' operand. dont_want_licenses should be passed
as canonical (SPDX) names.
"""
- import re
import oe.license
- from fnmatch import fnmatchcase as fnmatch
license = d.getVar("LICENSE_%s" % package, True) if package else None
if not license:
license = d.getVar('LICENSE', True)
- def license_ok(license):
- for dwl in dont_want_licenses:
- # If you want to exclude license named generically 'X', we
- # surely want to exclude 'X+' as well. In consequence, we
- # will exclude a trailing '+' character from LICENSE in
- # case INCOMPATIBLE_LICENSE is not a 'X+' license.
- lic = license
- if not re.search('\+$', dwl):
- lic = re.sub('\+', '', license)
- if fnmatch(lic, dwl):
- return False
- return True
-
# Handles an "or" or two license sets provided by
# flattened_licenses(), pick one that works if possible.
def choose_lic_set(a, b):
- return a if all(license_ok(lic) for lic in a) else b
+ return a if all(oe.license.license_ok(lic, dont_want_licenses) \
+ for lic in a) else b
try:
licenses = oe.license.flattened_licenses(license, choose_lic_set)
except oe.license.LicenseError as exc:
bb.fatal('%s: %s' % (d.getVar('P', True), exc))
- return any(not license_ok(canonical_license(d, l)) for l in licenses)
+ return any(not oe.license.license_ok(canonical_license(d, l), \
+ dont_want_licenses) for l in licenses)
def check_license_flags(d):
"""
diff --git a/meta/lib/oe/license.py b/meta/lib/oe/license.py
index 31ca15b..bc146a28 100644
--- a/meta/lib/oe/license.py
+++ b/meta/lib/oe/license.py
@@ -5,6 +5,20 @@ import ast
import re
from fnmatch import fnmatchcase as fnmatch
+def license_ok(license, dont_want_licenses):
+ """ Return False if License exist in dont_want_licenses else True """
+ for dwl in dont_want_licenses:
+ # If you want to exclude license named generically 'X', we
+ # surely want to exclude 'X+' as well. In consequence, we
+ # will exclude a trailing '+' character from LICENSE in
+ # case INCOMPATIBLE_LICENSE is not a 'X+' license.
+ lic = license
+ if not re.search('\+$', dwl):
+ lic = re.sub('\+', '', license)
+ if fnmatch(lic, dwl):
+ return False
+ return True
+
class LicenseError(Exception):
pass
--
1.8.4.5
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 3/3] license: Add support for handle INCOMPATIBLE_LICENSE in manifest creation
2015-05-06 14:52 [PATCH 0/3] Add support to handle INCOMPATIBLE_LICENSE Aníbal Limón
2015-05-06 14:52 ` [PATCH 1/3] license_class: Reimplemented manifest creation in python Aníbal Limón
2015-05-06 14:52 ` [PATCH 2/3] license_class: Generalize license_ok function Aníbal Limón
@ 2015-05-06 14:52 ` Aníbal Limón
2 siblings, 0 replies; 6+ messages in thread
From: Aníbal Limón @ 2015-05-06 14:52 UTC (permalink / raw)
To: openembedded-core
When INCOMPATIBLE_LICENSE's is specified it need to be removed from
license.manifest and also avoid copy to target image.
Add ManifestVisitor that walk the license string searching for
INCOMPATIBLE_LICENSE's if found remove it.
[YOCTO #6765]
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
meta/classes/license.bbclass | 29 ++++++++++++++++--
meta/lib/oe/license.py | 70 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 96 insertions(+), 3 deletions(-)
diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index 780b9d5..54ab123 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -29,6 +29,10 @@ python license_create_manifest() {
import re
import oe.packagedata
+ bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE", True) or "").split()
+ bad_licenses = map(lambda l: canonical_license(d, l), bad_licenses)
+ bad_licenses = expand_wildcard_licenses(d, bad_licenses)
+
build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', True)
if build_images_from_feeds == "1":
return 0
@@ -52,6 +56,18 @@ python license_create_manifest() {
d.getVar('IMAGE_NAME', True), 'license.manifest')
with open(license_manifest, "w") as license_file:
for pkg in sorted(pkg_dic):
+ if bad_licenses:
+ try:
+ (pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \
+ oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"],
+ bad_licenses, canonical_license, d)
+ except oe.license.LicenseError as exc:
+ bb.fatal('%s: %s' % (d.getVar('P', True), exc))
+ else:
+ pkg_dic[pkg]["LICENSES"] = re.sub('[|&()*]', '', pkg_dic[pkg]["LICENSE"])
+ pkg_dic[pkg]["LICENSES"] = re.sub(' *', ' ', pkg_dic[pkg]["LICENSES"])
+ pkg_dic[pkg]["LICENSES"] = pkg_dic[pkg]["LICENSES"].split()
+
license_file.write("PACKAGE NAME: %s\n" % pkg)
license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
@@ -63,9 +79,7 @@ python license_create_manifest() {
if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
continue
- licenses = re.sub('[|&()*]', '', pkg_dic[pkg]["LICENSE"])
- licenses = re.sub(' *', ' ', licenses)
- for lic in licenses.split():
+ for lic in pkg_dic[pkg]["LICENSES"]:
lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
pkg_dic[pkg]["PN"], "generic_%s" %
re.sub('\+', '', lic))
@@ -101,11 +115,20 @@ python license_create_manifest() {
pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
if re.match("^generic_.*$", lic):
+ generic_lic = re.search("^generic_(.*)$", lic).group(1)
+ if oe.license.license_ok(canonical_license(d,
+ generic_lic), bad_licenses) == False:
+ continue
+
if not os.path.exists(rootfs_license):
os.link(pkg_license, rootfs_license)
os.symlink(os.path.join('..', lic), pkg_rootfs_license)
else:
+ if oe.license.license_ok(canonical_license(d,
+ lic), bad_licenses) == False:
+ continue
+
os.link(pkg_license, pkg_rootfs_license)
}
diff --git a/meta/lib/oe/license.py b/meta/lib/oe/license.py
index bc146a28..c2e523c 100644
--- a/meta/lib/oe/license.py
+++ b/meta/lib/oe/license.py
@@ -129,3 +129,73 @@ def is_included(licensestr, whitelist=None, blacklist=None):
return False, excluded
else:
return True, included
+
+class ManifestVisitor(LicenseVisitor):
+ """Walk license tree (parsed from a string) removing the incompatible
+ licenses specified"""
+ def __init__(self, dont_want_licenses, canonical_license, d):
+ self._dont_want_licenses = dont_want_licenses
+ self._canonical_license = canonical_license
+ self._d = d
+ self._operators = []
+
+ self.licenses = []
+ self.licensestr = ''
+
+ LicenseVisitor.__init__(self)
+
+ def visit(self, node):
+ if isinstance(node, ast.Str):
+ lic = node.s
+
+ if license_ok(self._canonical_license(self._d, lic),
+ self._dont_want_licenses) == True:
+ if self._operators:
+ ops = []
+ for op in self._operators:
+ if op == '[':
+ ops.append(op)
+ elif op == ']':
+ ops.append(op)
+ else:
+ if not ops:
+ ops.append(op)
+ elif ops[-1] in ['[', ']']:
+ ops.append(op)
+ else:
+ ops[-1] = op
+
+ if ops[0] != ']':
+ self.licensestr += ' '
+ self.licensestr += ' '.join(ops)
+ if len(ops) == 1:
+ self.licensestr += ' '
+
+ self.licensestr += lic
+ self.licenses.append(lic)
+ self._operators = []
+ elif isinstance(node, ast.BitAnd):
+ self._operators.append("&")
+ elif isinstance(node, ast.BitOr):
+ self._operators.append("|")
+ elif isinstance(node, ast.List):
+ self._operators.append("[")
+ elif isinstance(node, ast.Load):
+ self.licensestr += ']'
+
+ self.generic_visit(node)
+
+def manifest_licenses(licensestr, dont_want_licenses, canonical_license, d):
+ """Given a license string and dont_want_licenses list,
+ return license string filtered and a list of licenses"""
+ manifest = ManifestVisitor(dont_want_licenses, canonical_license, d)
+
+ # Replace '()' to '[]' for handle in ast as List and Load types.
+ licensestr = licensestr.replace('(', '[').replace(')', ']')
+ try:
+ manifest.visit_string(licensestr)
+ except SyntaxError as exc:
+ raise LicenseSyntaxError(licensestr, exc)
+ manifest.licensestr = manifest.licensestr.replace('[', '(').replace(']', ')')
+
+ return (manifest.licensestr, manifest.licenses)
--
1.8.4.5
^ permalink raw reply related [flat|nested] 6+ messages in thread