Re: [OE-core] [PATCH] buildhistory: Label packages providing per-file dependencies in depends.dot

["Richard Purdie" <richard.purdie@linuxfoundation.org>]

On Wed, 2021-08-18 at 17:17 +0000, Andres Beltran wrote:
> Currently, depends.dot includes per-file dependencies but not the packages
> providing those files. This makes it hard to obtain all package
> dependencies by just looking at depends.dot.
> 
> Parse the RPROVIDES and FILERPROVIDES fields from pkgdata to map each of
> their values to the package providing the component. Include runtime
> packages as dependencies in depends.dot, together with the component
> provided by the package as a label.
> 
> Signed-off-by: Andres Beltran <abeltran@linux.microsoft.com>
> ---
>  meta/classes/buildhistory.bbclass |  4 +++-
>  meta/lib/oe/utils.py              | 29 +++++++++++++++++++++++++++--
>  2 files changed, 30 insertions(+), 3 deletions(-)
> 
> diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass
> index 8a1359acbe..9ec48a879a 100644
> --- a/meta/classes/buildhistory.bbclass
> +++ b/meta/classes/buildhistory.bbclass
> @@ -438,7 +438,7 @@ def buildhistory_list_installed(d, rootfs_type="image"):
>          output_file_full = os.path.join(d.getVar('WORKDIR'), output_file)
>  
>          with open(output_file_full, 'w') as output:
> -            output.write(format_pkg_list(pkgs, output_type))
> +            output.write(format_pkg_list(pkgs, output_type, d.getVar('PKGDATA_DIR')))
>  
>  python buildhistory_list_installed_image() {
>      buildhistory_list_installed(d)
> @@ -479,6 +479,8 @@ buildhistory_get_installed() {
>  	       -e 's:|: -> :' \
>  	       -e 's:"\[REC\]":[style=dotted]:' \
>  	       -e 's:"\([<>=]\+\)" "\([^"]*\)":[label="\1 \2"]:' \
> +	       -e 's:"\([*]\+\)" "\([^"]*\)":[label="\2"]:' \
> +	       -e 's:"\[RPROVIDES\]":[style=dashed]:' \
>  		$1/depends.tmp
>  	# Add header, sorted and de-duped contents and footer and then delete the temp file
>  	printf "digraph depends {\n    node [shape=plaintext]\n" > $1/depends.dot
> diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
> index 83d298906b..0f3dfd644d 100644
> --- a/meta/lib/oe/utils.py
> +++ b/meta/lib/oe/utils.py
> @@ -345,7 +345,26 @@ def squashspaces(string):
>      import re
>      return re.sub(r"\s+", " ", string).strip()
>  
> -def format_pkg_list(pkg_dict, ret_format=None):
> +def rprovides_map(pkgdata_dir, pkg_dict):
> +    # Map file -> pkg provider
> +    rprov_map = {}
> +
> +    for pkg in pkg_dict:
> +        with open(os.path.join(pkgdata_dir, 'runtime-reverse', pkg)) as f:
> +            for line in f:
> +                if line.startswith('RPROVIDES') or line.startswith('FILERPROVIDES'):
> +                    # List all components provided by pkg.
> +                    # Exclude version strings, i.e. those starting with (
> +                    provides = [x for x in line.split()[1:] if not x.startswith('(')]
> +                    for prov in provides:
> +                        if prov in rprov_map:
> +                            rprov_map[prov].append(pkg)
> +                        else:
> +                            rprov_map[prov] = [pkg]
> +
> +    return rprov_map
> +
> +def format_pkg_list(pkg_dict, ret_format=None, pkgdata_dir=None):
>      output = []
>  
>      if ret_format == "arch":
> @@ -358,9 +377,15 @@ def format_pkg_list(pkg_dict, ret_format=None):
>          for pkg in sorted(pkg_dict):
>              output.append("%s %s %s" % (pkg, pkg_dict[pkg]["arch"], pkg_dict[pkg]["ver"]))
>      elif ret_format == "deps":
> +        rprov_map = rprovides_map(pkgdata_dir, pkg_dict)
>          for pkg in sorted(pkg_dict):
>              for dep in pkg_dict[pkg]["deps"]:
> -                output.append("%s|%s" % (pkg, dep))
> +                if dep in rprov_map:
> +                    # There could be multiple providers within the image
> +                    for pkg_provider in rprov_map[dep]:
> +                        output.append("%s|%s * %s [RPROVIDES]" % (pkg, pkg_provider, dep))
> +                else:
> +                    output.append("%s|%s" % (pkg, dep))
>      else:
>          for pkg in sorted(pkg_dict):
>              output.append(pkg)

This seemed to cause failures when generating SDKs:

https://autobuilder.yoctoproject.org/typhoon/#/builders/53/builds/3914
https://autobuilder.yoctoproject.org/typhoon/#/builders/42/builds/3895
https://autobuilder.yoctoproject.org/typhoon/#/builders/74/builds/3882
https://autobuilder.yoctoproject.org/typhoon/#/builders/63/builds/3875
https://autobuilder.yoctoproject.org/typhoon/#/builders/59/builds/3884
https://autobuilder.yoctoproject.org/typhoon/#/builders/73/builds/3880

Cheers,

Richard