Re: [OE-core][PATCH] classes/package: Add extended packaged data

["Alexandre Belloni" <alexandre.belloni@bootlin.com>]

Hello Josh,

On 13/07/2021 11:08:46-0500, Joshua Watt wrote:
> Adds extended package data which is encoded as JSON which allows it to
> encode more structure than the "flat" package data files. The extended
> data might be much larger than the standard package data, so it is not
> read by default and instead requires
> oe.packagedata.read_subpkgdata_extended() to be called
> 
> Currently, the file sizes and ELF debug sources are saved off into the
> extended package data
> 

This break non-gpl3 builds:

https://autobuilder.yoctoproject.org/typhoon/#/builders/75/builds/3685

ERROR: libgcrypt-1.9.3-r0 do_package: One or more file debug sources were not consumed by a package: /usr/bin/dumpsexp
ERROR: libidn2-2.3.1-r0 do_package: One or more file debug sources were not consumed by a package: /usr/bin/idn2


> Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
> ---
>  meta/classes/package.bbclass | 38 ++++++++++++++++++++++++++++++++++--
>  meta/lib/oe/packagedata.py   |  9 +++++++++
>  2 files changed, 45 insertions(+), 2 deletions(-)
> 
> diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
> index cf30f33f3d..a0e7f464a9 100644
> --- a/meta/classes/package.bbclass
> +++ b/meta/classes/package.bbclass
> @@ -1225,6 +1225,14 @@ python split_and_strip_files () {
>                  # Modified the file so clear the cache
>                  cpath.updatecache(file)
>  
> +    def strip_pkgd_prefix(f):
> +        nonlocal dvar
> +
> +        if f.startswith(dvar):
> +            return f[len(dvar):]
> +
> +        return f
> +
>      #
>      # First lets process debug splitting
>      #
> @@ -1238,6 +1246,8 @@ python split_and_strip_files () {
>                  for file in staticlibs:
>                      results.append( (file,source_info(file, d)) )
>  
> +        d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, s in results})
> +
>          sources = set()
>          for r in results:
>              sources.update(r[1])
> @@ -1622,6 +1632,8 @@ fi
>      with open(data_file, 'w') as fd:
>          fd.write("PACKAGES: %s\n" % packages)
>  
> +    pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or []
> +
>      pn = d.getVar('PN')
>      global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split()
>      variants = (d.getVar('MULTILIB_VARIANTS') or "").split()
> @@ -1641,17 +1653,32 @@ fi
>              pkgval = pkg
>              d.setVar('PKG_%s' % pkg, pkg)
>  
> +        extended_data = {
> +            "files_info": {}
> +        }
> +
>          pkgdestpkg = os.path.join(pkgdest, pkg)
>          files = {}
> +        files_extra = {}
>          total_size = 0
>          seen = set()
>          for f in pkgfiles[pkg]:
> -            relpth = os.path.relpath(f, pkgdestpkg)
> +            fpath = os.sep + os.path.relpath(f, pkgdestpkg)
> +
>              fstat = os.lstat(f)
> -            files[os.sep + relpth] = fstat.st_size
> +            files[fpath] = fstat.st_size
> +
> +            extended_data["files_info"].setdefault(fpath, {})
> +            extended_data["files_info"][fpath]['size'] = fstat.st_size
> +
>              if fstat.st_ino not in seen:
>                  seen.add(fstat.st_ino)
>                  total_size += fstat.st_size
> +
> +            if fpath in pkgdebugsource:
> +                extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath]
> +                del pkgdebugsource[fpath]
> +
>          d.setVar('FILES_INFO', json.dumps(files, sort_keys=True))
>  
>          process_postinst_on_target(pkg, d.getVar("MLPREFIX"))
> @@ -1672,6 +1699,10 @@ fi
>  
>              sf.write('%s_%s: %d\n' % ('PKGSIZE', pkg, total_size))
>  
> +        subdata_extended_file = pkgdatadir + "/runtime/%s.json" % pkg
> +        with open(subdata_extended_file, "w") as f:
> +            json.dump(extended_data, f, sort_keys=True, separators=(",", ":"))
> +
>          # Symlinks needed for rprovides lookup
>          rprov = d.getVar('RPROVIDES_%s' % pkg) or d.getVar('RPROVIDES')
>          if rprov:
> @@ -1694,6 +1725,9 @@ fi
>              packagedfile = pkgdatadir + '/runtime/%s.packaged' % pkg
>              open(packagedfile, 'w').close()
>  
> +    if pkgdebugsource:
> +        bb.fatal("One or more file debug sources were not consumed by a package: %s" % " ".join(pkgdebugsource.keys()))
> +
>      if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d):
>          write_extra_runtime_pkgs(variants, packages, pkgdatadir)
>  
> diff --git a/meta/lib/oe/packagedata.py b/meta/lib/oe/packagedata.py
> index a82085a792..60485cc79d 100644
> --- a/meta/lib/oe/packagedata.py
> +++ b/meta/lib/oe/packagedata.py
> @@ -57,6 +57,15 @@ def read_subpkgdata_dict(pkg, d):
>          ret[newvar] = subd[var]
>      return ret
>  
> +def read_subpkgdata_extended(pkg, d):
> +    import json
> +    fn = d.expand("${PKGDATA_DIR}/runtime/%s.json" % pkg)
> +    try:
> +        with open(fn, "r") as f:
> +            return json.load(f)
> +    except FileNotFoundError:
> +        return None
> +
>  def _pkgmap(d):
>      """Return a dictionary mapping package to recipe name."""
>  
> -- 
> 2.32.0
> 

> 
> 
> 


-- 
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com