Re: [PATCH 3/3] package_ipk: Parallelise ipk creation

[Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>]

On Wed, 2017-06-14 at 14:42 +0100, Richard Purdie wrote:
> Allow the creation of ipks to happen in parallel, making best use of resources
> on multiprocessor systems.
> 
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>  meta/classes/package_ipk.bbclass | 19 +++++++++++++++++--
>  1 file changed, 17 insertions(+), 2 deletions(-)
> 
> diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass
> index d2ce3b3..282d212 100644
> --- a/meta/classes/package_ipk.bbclass
> +++ b/meta/classes/package_ipk.bbclass
> @@ -17,6 +17,8 @@ OPKG_ARGS += "${@['', '--add-exclude ' + ' --add-exclude '.join((d.getVar('PACKA
>  OPKGLIBDIR = "${localstatedir}/lib"
>  
>  python do_package_ipk () {
> +    from multiprocessing import Process
> +
>      oldcwd = os.getcwd()
>  
>      workdir = d.getVar('WORKDIR')
> @@ -37,11 +39,24 @@ python do_package_ipk () {
>      if os.access(os.path.join(tmpdir, "stamps", "IPK_PACKAGE_INDEX_CLEAN"), os.R_OK):
>          os.unlink(os.path.join(tmpdir, "stamps", "IPK_PACKAGE_INDEX_CLEAN"))
>  
> -    for pkg in packages.split():
> -        ipk_write_pkg(pkg, d)


this part looks quite similar as the 2/3, perhaps a single function
inside package.bbclass?

> +    max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
> +    launched = []
> +    pkgs = packages.split()
> +    while pkgs:
> +        if len(launched) < max_process:
> +            p = Process(target=ipk_write_pkg, args=(pkgs.pop(), d))
> +            p.start()
> +            launched.append(p)
> +        for q in launched:
> +            # The finished processes are joined when calling is_alive()
> +            if not q.is_alive():
> +                launched.remove(q)
> +    for p in launched:
> +        p.join()
>  
>      os.chdir(oldcwd)
>  }
> +do_package_ipk[vardepsexclude] = "BB_NUMBER_THREADS"
>  
>  def ipk_write_pkg(pkg, d):
>      import re, copy
> -- 
> 2.7.4
>