* [bitbake][scarthgap][2.8][PATCH 1/3] Use a "fork" multiprocessing context
2025-09-10 16:56 [bitbake][scarthgap][2.8][PATCH 0/3 V2] Patch review Steve Sakoman
@ 2025-09-10 16:56 ` Steve Sakoman
2025-09-10 16:56 ` [bitbake][scarthgap][2.8][PATCH 2/3] bitbake: Bump version to 2.8.1 Steve Sakoman
2025-09-10 16:56 ` [bitbake][scarthgap][2.8][PATCH 3/3] bb/fetch2/__init__.py: remove a DeprecationWarning in uri_replace() Steve Sakoman
2 siblings, 0 replies; 5+ messages in thread
From: Steve Sakoman @ 2025-09-10 16:56 UTC (permalink / raw)
To: bitbake-devel
From: Joshua Watt <JPEWhacker@gmail.com>
Python 3.14 changes the default multiprocessing context from "fork" to
"forkserver"; however bitbake heavily relies on "fork" to efficiently
pass data to the child processes. As such, make "fork" context in the bb
namespace and use it in place of the normal multiprocessing module.
Note that multiprocessing contexts were added in Python 3.4, so this
should be safe to use even before Python 3.14
[YOCTO #15858]
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Martin Jansa <martin.jansa@gmail.com>
Signed-off-by: Steve Sakoman <steve@sakoman.com>
---
lib/bb/__init__.py | 28 ++++++++++++++++++++++++++++
lib/bb/asyncrpc/serv.py | 2 +-
lib/bb/cooker.py | 2 +-
lib/bb/server/process.py | 2 +-
lib/bb/tests/support/httpserver.py | 4 ++--
lib/bb/utils.py | 4 +---
lib/hashserv/tests.py | 2 +-
7 files changed, 35 insertions(+), 9 deletions(-)
diff --git a/lib/bb/__init__.py b/lib/bb/__init__.py
index cdec9e4d6..f59c2515b 100644
--- a/lib/bb/__init__.py
+++ b/lib/bb/__init__.py
@@ -37,6 +37,34 @@ class BBHandledException(Exception):
import os
import logging
from collections import namedtuple
+import multiprocessing as mp
+
+# Python 3.14 changes the default multiprocessing context from "fork" to
+# "forkserver". However, bitbake heavily relies on "fork" behavior to
+# efficiently pass data to the child processes. Places that need this should do:
+# from bb import multiprocessing
+# in place of
+# import multiprocessing
+
+class MultiprocessingContext(object):
+ """
+ Multiprocessing proxy object that uses the "fork" context for a property if
+ available, otherwise goes to the main multiprocessing module. This allows
+ it to be a drop-in replacement for the multiprocessing module, but use the
+ fork context
+ """
+ def __init__(self):
+ super().__setattr__("_ctx", mp.get_context("fork"))
+
+ def __getattr__(self, name):
+ if hasattr(self._ctx, name):
+ return getattr(self._ctx, name)
+ return getattr(mp, name)
+
+ def __setattr__(self, name, value):
+ raise AttributeError(f"Unable to set attribute {name}")
+
+multiprocessing = MultiprocessingContext()
class NullHandler(logging.Handler):
diff --git a/lib/bb/asyncrpc/serv.py b/lib/bb/asyncrpc/serv.py
index a66117aca..953c02ef3 100644
--- a/lib/bb/asyncrpc/serv.py
+++ b/lib/bb/asyncrpc/serv.py
@@ -11,7 +11,7 @@ import os
import signal
import socket
import sys
-import multiprocessing
+from bb import multiprocessing
import logging
from .connection import StreamConnection, WebsocketConnection
from .exceptions import ClientError, ServerError, ConnectionClosedError, InvokeError
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 6fce19b46..778cbb589 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -12,7 +12,7 @@
import sys, os, glob, os.path, re, time
import itertools
import logging
-import multiprocessing
+from bb import multiprocessing
import threading
from io import StringIO, UnsupportedOperation
from contextlib import closing
diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
index 76b189291..34b3a2ae9 100644
--- a/lib/bb/server/process.py
+++ b/lib/bb/server/process.py
@@ -13,7 +13,7 @@
import bb
import bb.event
import logging
-import multiprocessing
+from bb import multiprocessing
import threading
import array
import os
diff --git a/lib/bb/tests/support/httpserver.py b/lib/bb/tests/support/httpserver.py
index 78f766005..03327e923 100644
--- a/lib/bb/tests/support/httpserver.py
+++ b/lib/bb/tests/support/httpserver.py
@@ -3,7 +3,7 @@
#
import http.server
-import multiprocessing
+from bb import multiprocessing
import os
import traceback
import signal
@@ -43,7 +43,7 @@ class HTTPService(object):
self.process = multiprocessing.Process(target=self.server.server_start, args=[self.root_dir, self.logger])
# The signal handler from testimage.bbclass can cause deadlocks here
- # if the HTTPServer is terminated before it can restore the standard
+ # if the HTTPServer is terminated before it can restore the standard
#signal behaviour
orig = signal.getsignal(signal.SIGTERM)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
diff --git a/lib/bb/utils.py b/lib/bb/utils.py
index d2f11e437..1b4fb93a3 100644
--- a/lib/bb/utils.py
+++ b/lib/bb/utils.py
@@ -14,7 +14,7 @@ import logging
import bb
import bb.msg
import locale
-import multiprocessing
+from bb import multiprocessing
import fcntl
import importlib
import importlib.machinery
@@ -1174,8 +1174,6 @@ def process_profilelog(fn, pout = None):
#
def multiprocessingpool(*args, **kwargs):
- import multiprocessing.pool
- #import multiprocessing.util
#multiprocessing.util.log_to_stderr(10)
# Deal with a multiprocessing bug where signals to the processes would be delayed until the work
# completes. Putting in a timeout means the signals (like SIGINT/SIGTERM) get processed.
diff --git a/lib/hashserv/tests.py b/lib/hashserv/tests.py
index 5349cd586..ed1ade749 100644
--- a/lib/hashserv/tests.py
+++ b/lib/hashserv/tests.py
@@ -11,7 +11,7 @@ from bb.asyncrpc import InvokeError
from .client import ClientPool
import hashlib
import logging
-import multiprocessing
+from bb import multiprocessing
import os
import sys
import tempfile
--
2.43.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [bitbake][scarthgap][2.8][PATCH 3/3] bb/fetch2/__init__.py: remove a DeprecationWarning in uri_replace()
2025-09-10 16:56 [bitbake][scarthgap][2.8][PATCH 0/3 V2] Patch review Steve Sakoman
2025-09-10 16:56 ` [bitbake][scarthgap][2.8][PATCH 1/3] Use a "fork" multiprocessing context Steve Sakoman
2025-09-10 16:56 ` [bitbake][scarthgap][2.8][PATCH 2/3] bitbake: Bump version to 2.8.1 Steve Sakoman
@ 2025-09-10 16:56 ` Steve Sakoman
2025-09-23 17:22 ` [bitbake-devel] " chris.laplante
2 siblings, 1 reply; 5+ messages in thread
From: Steve Sakoman @ 2025-09-10 16:56 UTC (permalink / raw)
To: bitbake-devel
From: Bin Lan <bin.lan.cn@windriver.com>
There is the following warning when executing to bitbake linux-yocto:
bitbake/lib/bb/fetch2/__init__.py:464: DeprecationWarning: 'count' is passed as positional argument
This is because the 4th parameter of re.sub(pattern, repl, string, count=0, flags=0)
is a keyword parameter. We use keyword arguments for parameters that are not positional.
Signed-off-by: Bin Lan <bin.lan.cn@windriver.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Chris Laplante <chris.laplante@agilent.com>
Signed-off-by: Steve Sakoman <steve@sakoman.com>
---
lib/bb/fetch2/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index 1a6ff25d4..7b75d5d83 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -460,7 +460,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
for k in replacements:
uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k])
#bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc]))
- result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1)
+ result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], count=1)
if loc == 2:
# Handle path manipulations
basename = None
--
2.43.0
^ permalink raw reply related [flat|nested] 5+ messages in thread