Re: [PATCH] package_rpm.bbclass: Optimise per file dependency handling

[Mark Hatle <mark.hatle@windriver.com>]

On 3/2/13 4:39 PM, Richard Purdie wrote:
> Currently the process for injecting the per file rpm dependencies into
> rpmbuild is painfully slow. Its done through the repeated execution of
> a script which has to return the correct value in each case. This continual
> execution means the CPU usage of rpmbuild is low.
>
> This patch allows the option of collapsing the per file dependencies to
> a per package basis and injecting them through the .spec file. This removes
> the execution overhead and allows rpmbuild to run at 100% of cpu.
>
> Ultimately it would be nice to inject the per file dependencies through
> the .spec file however that is not currently possible.
>
> Since few people use the per file dependency information, this patch
> goes for the faster approach. It can be enabled if anyone needs it although
> I'd mention that its being used to us as this code may well go away in
> the future if nobody complains.
>
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
> diff --git a/meta/classes/package_rpm.bbclass b/meta/classes/package_rpm.bbclass
> index 697bb36..3ac379d 100644
> --- a/meta/classes/package_rpm.bbclass
> +++ b/meta/classes/package_rpm.bbclass
> @@ -8,6 +8,10 @@ RPMBUILD="rpmbuild"
>   PKGWRITEDIRRPM = "${WORKDIR}/deploy-rpms"
>   PKGWRITEDIRSRPM = "${DEPLOY_DIR}/sources/deploy-srpm"
>
> +# Maintaining the perfile dependencies has singificant overhead when writing the
> +# packages. When set, this value merges them for efficiency.
> +MERGEPERFILEDEPS = "1"

Can this be a "?=" so it can be easily overridden?

> +
>   #
>   # Update the packages indexes ${DEPLOY_DIR_RPM}
>   #
> @@ -460,6 +464,78 @@ EOF
>   	fi
>   }
>
> +# Construct per file dependencies file
> +def write_rpm_perfiledata(srcname, d):
> +    workdir = d.getVar('WORKDIR', True)
> +    packages = d.getVar('PACKAGES', True)
> +    pkgd = d.getVar('PKGD', True)
> +
> +    def dump_filerdeps(varname, outfile, d):
> +        outfile.write("#!/usr/bin/env python\n\n")
> +        outfile.write("# Dependency table\n")
> +        outfile.write('deps = {\n')
> +        for pkg in packages.split():
> +            dependsflist_key = 'FILE' + varname + 'FLIST' + "_" + pkg
> +            dependsflist = (d.getVar(dependsflist_key, True) or "")
> +            for dfile in dependsflist.split():
> +                key = "FILE" + varname + "_" + dfile + "_" + pkg
> +                depends_dict = bb.utils.explode_dep_versions(d.getVar(key, True) or "")
> +                file = dfile.replace("@underscore@", "_")
> +                file = file.replace("@closebrace@", "]")
> +                file = file.replace("@openbrace@", "[")
> +                file = file.replace("@tab@", "\t")
> +                file = file.replace("@space@", " ")
> +                file = file.replace("@at@", "@")
> +                outfile.write('"' + pkgd + file + '" : "')
> +                for dep in depends_dict:
> +                    ver = depends_dict[dep]
> +                    if dep and ver:
> +                        ver = ver.replace("(","")
> +                        ver = ver.replace(")","")
> +                        outfile.write(dep + " " + ver + " ")
> +                    else:
> +                        outfile.write(dep + " ")
> +                outfile.write('",\n')
> +        outfile.write('}\n\n')
> +        outfile.write("import sys\n")
> +        outfile.write("while 1:\n")
> +        outfile.write("\tline = sys.stdin.readline().strip()\n")
> +        outfile.write("\tif not line:\n")
> +        outfile.write("\t\tsys.exit(0)\n")
> +        outfile.write("\tif line in deps:\n")
> +        outfile.write("\t\tprint(deps[line] + '\\n')\n")
> +
> +    # OE-core dependencies a.k.a. RPM requires
> +    outdepends = workdir + "/" + srcname + ".requires"
> +
> +    try:
> +        from __builtin__ import file
> +        dependsfile = file(outdepends, 'w')
> +    except OSError:
> +        raise bb.build.FuncFailed("unable to open spec file for writing.")
> +
> +    dump_filerdeps('RDEPENDS', dependsfile, d)
> +
> +    dependsfile.close()
> +    os.chmod(outdepends, 0755)
> +
> +    # OE-core / RPM Provides
> +    outprovides = workdir + "/" + srcname + ".provides"
> +
> +    try:
> +        from __builtin__ import file
> +        providesfile = file(outprovides, 'w')
> +    except OSError:
> +        raise bb.build.FuncFailed("unable to open spec file for writing.")
> +
> +    dump_filerdeps('RPROVIDES', providesfile, d)
> +
> +    providesfile.close()
> +    os.chmod(outprovides, 0755)
> +
> +    return (outdepends, outprovides)
> +
> +
>   python write_specfile () {
>       import textwrap
>       import oe.packagedata
> @@ -576,6 +652,17 @@ python write_specfile () {
>           scr = scr[:pos] + 'if [ "$1" = "0" ] ; then\n' + scr[pos:] + '\nfi'
>           return scr
>
> +    def get_perfile(varname, pkg, d):
> +        deps = []
> +        dependsflist_key = 'FILE' + varname + 'FLIST' + "_" + pkg
> +        dependsflist = (d.getVar(dependsflist_key, True) or "")
> +        for dfile in dependsflist.split():
> +            key = "FILE" + varname + "_" + dfile + "_" + pkg
> +            depends = d.getVar(key, True)
> +            if depends:
> +                deps.append(depends)
> +        return " ".join(deps)
> +

The above doesn't avoid duplicate entries.  Is this a concern?  (RPM doesn't 
care, it'll do it itself... but it may be slower due to this.)

>       packages = d.getVar('PACKAGES', True)
>       if not packages or packages == '':
>           bb.debug(1, "No packages; nothing to do")
> @@ -626,6 +713,8 @@ python write_specfile () {
>       spec_files_top = []
>       spec_files_bottom = []
>
> +    perfiledeps = (d.getVar("MERGEPERFILEDEPS", True) or "0") == "0"
> +
>       for pkg in packages.split():
>           localdata = bb.data.createCopy(d)
>
> @@ -679,6 +768,12 @@ python write_specfile () {
>           splitrprerm    = localdata.getVar('pkg_prerm', True)
>           splitrpostrm   = localdata.getVar('pkg_postrm', True)
>
> +
> +        if not perfiledeps:
> +            # Add in summary of per file dependencies
> +            splitrdepends = splitrdepends + " " + get_perfile('RDEPENDS', pkg, d)
> +            splitrprovides = splitrprovides + " " + get_perfile('RPROVIDES', pkg, d)
> +
>           # Gather special src/first package data
>           if srcname == splitname:
>               srcrdepends    = splitrdepends
> @@ -977,69 +1072,9 @@ python do_package_rpm () {
>       d.setVar('OUTSPECFILE', outspecfile)
>       bb.build.exec_func('write_specfile', d)
>
> -    # Construct per file dependencies file
> -    def dump_filerdeps(varname, outfile, d):
> -        outfile.write("#!/usr/bin/env python\n\n")
> -        outfile.write("# Dependency table\n")
> -        outfile.write('deps = {\n')
> -        for pkg in packages.split():
> -            dependsflist_key = 'FILE' + varname + 'FLIST' + "_" + pkg
> -            dependsflist = (d.getVar(dependsflist_key, True) or "")
> -            for dfile in dependsflist.split():
> -                key = "FILE" + varname + "_" + dfile + "_" + pkg
> -                depends_dict = bb.utils.explode_dep_versions(d.getVar(key, True) or "")
> -                file = dfile.replace("@underscore@", "_")
> -                file = file.replace("@closebrace@", "]")
> -                file = file.replace("@openbrace@", "[")
> -                file = file.replace("@tab@", "\t")
> -                file = file.replace("@space@", " ")
> -                file = file.replace("@at@", "@")
> -                outfile.write('"' + pkgd + file + '" : "')
> -                for dep in depends_dict:
> -                    ver = depends_dict[dep]
> -                    if dep and ver:
> -                        ver = ver.replace("(","")
> -                        ver = ver.replace(")","")
> -                        outfile.write(dep + " " + ver + " ")
> -                    else:
> -                        outfile.write(dep + " ")
> -                outfile.write('",\n')
> -        outfile.write('}\n\n')
> -        outfile.write("import sys\n")
> -        outfile.write("while 1:\n")
> -        outfile.write("\tline = sys.stdin.readline().strip()\n")
> -        outfile.write("\tif not line:\n")
> -        outfile.write("\t\tsys.exit(0)\n")
> -        outfile.write("\tif line in deps:\n")
> -        outfile.write("\t\tprint(deps[line] + '\\n')\n")
> -
> -    # OE-core dependencies a.k.a. RPM requires
> -    outdepends = workdir + "/" + srcname + ".requires"
> -
> -    try:
> -        from __builtin__ import file
> -        dependsfile = file(outdepends, 'w')
> -    except OSError:
> -        raise bb.build.FuncFailed("unable to open spec file for writing.")
> -
> -    dump_filerdeps('RDEPENDS', dependsfile, d)
> -
> -    dependsfile.close()
> -    os.chmod(outdepends, 0755)
> -
> -    # OE-core / RPM Provides
> -    outprovides = workdir + "/" + srcname + ".provides"
> -
> -    try:
> -        from __builtin__ import file
> -        providesfile = file(outprovides, 'w')
> -    except OSError:
> -        raise bb.build.FuncFailed("unable to open spec file for writing.")
> -
> -    dump_filerdeps('RPROVIDES', providesfile, d)
> -
> -    providesfile.close()
> -    os.chmod(outprovides, 0755)
> +    perfiledeps = (d.getVar("MERGEPERFILEDEPS", True) or "0") == "0"
> +    if perfiledeps:
> +        outdepends, outprovides = write_rpm_perfiledata(srcname, d)
>
>       # Setup the rpmbuild arguments...
>       rpmbuild = d.getVar('RPMBUILD', True)
> @@ -1062,8 +1097,12 @@ python do_package_rpm () {
>       cmd = cmd + " --define '_topdir " + workdir + "' --define '_rpmdir " + pkgwritedir + "'"
>       cmd = cmd + " --define '_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm'"
>       cmd = cmd + " --define '_use_internal_dependency_generator 0'"
> -    cmd = cmd + " --define '__find_requires " + outdepends + "'"
> -    cmd = cmd + " --define '__find_provides " + outprovides + "'"
> +    if perfiledeps:
> +        cmd = cmd + " --define '__find_requires " + outdepends + "'"
> +        cmd = cmd + " --define '__find_provides " + outprovides + "'"
> +    else:
> +        cmd = cmd + " --define '__find_requires %{nil}'"
> +        cmd = cmd + " --define '__find_provides %{nil}'"
>       cmd = cmd + " --define '_unpackaged_files_terminate_build 0'"
>       cmd = cmd + " --define 'debug_package %{nil}'"
>       cmd = cmd + " --define '_rpmfc_magic_path " + magicfile + "'"
>
>