[PATCH 1/2] package/package_manager: multiprocess_exec -> multiprocess_launch

[PATCH 1/2] package/package_manager: multiprocess_exec -> multiprocess_launch

[Richard Purdie]

After this replacement, the parent exception handling works so we don't
need subprocess wrapping with bb.error in the underlying functions.

The underlying contexts also have better module handling so the imports
can be cleaned up.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/package.bbclass   |  4 ++--
 meta/lib/oe/package.py         | 13 +++----------
 meta/lib/oe/package_manager.py |  5 ++---
 3 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 86a6090f1b0..fca51906be4 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -1105,7 +1105,7 @@ python split_and_strip_files () {
         for f in kernmods:
             sfiles.append((f, 16, strip))
 
-        oe.utils.multiprocess_exec(sfiles, oe.package.runstrip)
+        oe.utils.multiprocess_launch(oe.package.runstrip, sfiles, d)
 
     #
     # End of strip
@@ -1541,7 +1541,7 @@ python package_do_filedeps() {
         for files in chunks(pkgfiles[pkg], 100):
             pkglist.append((pkg, files, rpmdeps, pkgdest))
 
-    processed = oe.utils.multiprocess_exec( pkglist, oe.package.filedeprunner)
+    processed = oe.utils.multiprocess_launch(oe.package.filedeprunner, pkglist, d)
 
     provides_files = {}
     requires_files = {}
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index 4255143371c..fa3428ad618 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -1,3 +1,4 @@
+import stat
 import mmap
 import subprocess
 
@@ -11,8 +12,6 @@ def runstrip(arg):
     # 8 - shared library
     # 16 - kernel module
 
-    import stat, subprocess
-
     (file, elftype, strip) = arg
 
     newmode = None
@@ -37,16 +36,11 @@ def runstrip(arg):
     stripcmd.append(file)
     bb.debug(1, "runstrip: %s" % stripcmd)
 
-    try:
-        output = subprocess.check_output(stripcmd, stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError as e:
-        bb.error("runstrip: '%s' strip command failed with %s (%s)" % (stripcmd, e.returncode, e.output))
+    output = subprocess.check_output(stripcmd, stderr=subprocess.STDOUT)
 
     if newmode:
         os.chmod(file, origmode)
 
-    return
-
 # Detect .ko module by searching for "vermagic=" string
 def is_kernel_module(path):
     with open(path) as f:
@@ -164,8 +158,7 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d, qa_already_stripp
         elf_file = int(elffiles[file])
         sfiles.append((file, elf_file, strip_cmd))
 
-    oe.utils.multiprocess_exec(sfiles, runstrip)
-
+    oe.utils.multiprocess_launch(runstrip, sfiles, d)
 
 
 def file_translate(file):
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 9aa5847c8a3..64c8a912160 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -3,7 +3,6 @@ import os
 import glob
 import subprocess
 import shutil
-import multiprocessing
 import re
 import collections
 import bb
@@ -177,7 +176,7 @@ class OpkgIndexer(Indexer):
             bb.note("There are no packages in %s!" % self.deploy_dir)
             return
 
-        oe.utils.multiprocess_exec(index_cmds, create_index)
+        oe.utils.multiprocess_launch(create_index, index_cmds, self.d)
 
         if signer:
             feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE')
@@ -258,7 +257,7 @@ class DpkgIndexer(Indexer):
             bb.note("There are no packages in %s" % self.deploy_dir)
             return
 
-        oe.utils.multiprocess_exec(index_cmds, create_index)
+        oe.utils.multiprocess_launch(create_index, index_cmds, self.d)
         if self.d.getVar('PACKAGE_FEED_SIGN') == '1':
             raise NotImplementedError('Package feed signing not implementd for dpkg')
 
-- 
2.17.1

[PATCH 2/2] lib/oe/utils: Drop now unused multiprocess_exec

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oe/utils.py | 34 ----------------------------------
 1 file changed, 34 deletions(-)

diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
index 753b5775550..814ac0fd51c 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -248,40 +248,6 @@ def execute_pre_post_process(d, cmds):
             bb.note("Executing %s ..." % cmd)
             bb.build.exec_func(cmd, d)
 
-def multiprocess_exec(commands, function):
-    import signal
-    import multiprocessing
-
-    if not commands:
-        return []
-
-    def init_worker():
-        signal.signal(signal.SIGINT, signal.SIG_IGN)
-
-    fails = []
-
-    def failures(res):
-        fails.append(res)
-
-    nproc = min(multiprocessing.cpu_count(), len(commands))
-    pool = bb.utils.multiprocessingpool(nproc, init_worker)
-
-    try:
-        mapresult = pool.map_async(function, commands, error_callback=failures)
-
-        pool.close()
-        pool.join()
-        results = mapresult.get()
-    except KeyboardInterrupt:
-        pool.terminate()
-        pool.join()
-        raise
-
-    if fails:
-        raise fails[0]
-
-    return results
-
 # For each item in items, call the function 'target' with item as the first 
 # argument, extraargs as the other arguments and handle any exceptions in the
 # parent thread
-- 
2.17.1

✗ patchtest: failure for "package/package_manager: multi..." and 1 more

[Patchwork]

== Series Details ==

Series: "package/package_manager: multi..." and 1 more
Revision: 1
URL   : https://patchwork.openembedded.org/series/13133/
State : failure

== Summary ==


Thank you for submitting this patch series to OpenEmbedded Core. This is
an automated response. Several tests have been executed on the proposed
series by patchtest resulting in the following failures:



* Issue             Series does not apply on top of target branch [test_series_merge_on_head] 
  Suggested fix    Rebase your series on top of targeted branch
  Targeted branch  master (currently at ff0b682b80)



If you believe any of these test results are incorrect, please reply to the
mailing list (openembedded-core@lists.openembedded.org) raising your concerns.
Otherwise we would appreciate you correcting the issues and submitting a new
version of the patchset if applicable. Please ensure you add/increment the
version number when sending the new version (i.e. [PATCH] -> [PATCH v2] ->
[PATCH v3] -> ...).

---
Guidelines:     https://www.openembedded.org/wiki/Commit_Patch_Message_Guidelines
Test framework: http://git.yoctoproject.org/cgit/cgit.cgi/patchtest
Test suite:     http://git.yoctoproject.org/cgit/cgit.cgi/patchtest-oe