* [1.52][PATCH 1/7] tests/fetch: Update github urls
2021-11-09 14:50 [1.52][PATCH 0/7] Review request Anuj Mittal
@ 2021-11-09 14:50 ` Anuj Mittal
2021-11-09 14:50 ` [1.52][PATCH 2/7] fetch: Handle mirror user/password replacements correctly Anuj Mittal
` (5 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Anuj Mittal @ 2021-11-09 14:50 UTC (permalink / raw)
To: bitbake-devel
From: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit 07fca7e3ab696ba985b3ef86ab9031d688bf2df2)
Signed-off-by: Anuj Mittal <anuj.mittal@intel.com>
---
lib/bb/tests/fetch.py | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index a008e7cc5..18f1c5e9e 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -491,7 +491,7 @@ class GitDownloadDirectoryNamingTest(FetcherTest):
super(GitDownloadDirectoryNamingTest, self).setUp()
self.recipe_url = "git://git.openembedded.org/bitbake"
self.recipe_dir = "git.openembedded.org.bitbake"
- self.mirror_url = "git://github.com/openembedded/bitbake.git"
+ self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https"
self.mirror_dir = "github.com.openembedded.bitbake.git"
self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
@@ -539,7 +539,7 @@ class TarballNamingTest(FetcherTest):
super(TarballNamingTest, self).setUp()
self.recipe_url = "git://git.openembedded.org/bitbake"
self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
- self.mirror_url = "git://github.com/openembedded/bitbake.git"
+ self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https"
self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz"
self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
@@ -573,7 +573,7 @@ class GitShallowTarballNamingTest(FetcherTest):
super(GitShallowTarballNamingTest, self).setUp()
self.recipe_url = "git://git.openembedded.org/bitbake"
self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz"
- self.mirror_url = "git://github.com/openembedded/bitbake.git"
+ self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https"
self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz"
self.d.setVar('BB_GIT_SHALLOW', '1')
@@ -1004,7 +1004,7 @@ class FetcherNetworkTest(FetcherTest):
def test_git_submodule_dbus_broker(self):
# The following external repositories have show failures in fetch and unpack operations
# We want to avoid regressions!
- url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
+ url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# Previous cwd has been deleted
@@ -1020,7 +1020,7 @@ class FetcherNetworkTest(FetcherTest):
@skipIfNoNetwork()
def test_git_submodule_CLI11(self):
- url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main"
+ url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main"
fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# Previous cwd has been deleted
@@ -1035,12 +1035,12 @@ class FetcherNetworkTest(FetcherTest):
@skipIfNoNetwork()
def test_git_submodule_update_CLI11(self):
""" Prevent regression on update detection not finding missing submodule, or modules without needed commits """
- url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main"
+ url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main"
fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# CLI11 that pulls in a newer nlohmann-json
- url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main"
+ url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main"
fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# Previous cwd has been deleted
@@ -1054,7 +1054,7 @@ class FetcherNetworkTest(FetcherTest):
@skipIfNoNetwork()
def test_git_submodule_aktualizr(self):
- url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
+ url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# Previous cwd has been deleted
@@ -1074,7 +1074,7 @@ class FetcherNetworkTest(FetcherTest):
""" Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
# This repository also has submodules where the module (name), path and url do not align
- url = "gitsm://github.com/azure/iotedge.git;protocol=git;rev=d76e0316c6f324345d77c48a83ce836d09392699"
+ url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699"
fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# Previous cwd has been deleted
@@ -1250,7 +1250,7 @@ class FetchLatestVersionTest(FetcherTest):
test_git_uris = {
# version pattern "X.Y.Z"
- ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
+ ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
: "1.99.4",
# version pattern "vX.Y"
# mirror of git.infradead.org since network issues interfered with testing
@@ -1277,9 +1277,9 @@ class FetchLatestVersionTest(FetcherTest):
: "0.4.3",
("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))")
: "11.0.0",
- ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
+ ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
: "1.3.59",
- ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
+ ("remake", "git://github.com/rocky/remake.git;protocol=https", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
: "3.82+dbg0.9",
}
@@ -2066,7 +2066,7 @@ class GitShallowTest(FetcherTest):
@skipIfNoNetwork()
def test_bitbake(self):
- self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
+ self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir)
self.git('config core.bare true', cwd=self.srcdir)
self.git('fetch', cwd=self.srcdir)
--
2.31.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [1.52][PATCH 2/7] fetch: Handle mirror user/password replacements correctly
2021-11-09 14:50 [1.52][PATCH 0/7] Review request Anuj Mittal
2021-11-09 14:50 ` [1.52][PATCH 1/7] tests/fetch: Update github urls Anuj Mittal
@ 2021-11-09 14:50 ` Anuj Mittal
2021-11-09 14:50 ` [1.52][PATCH 3/7] cooker: check if upstream hash equivalence server is available Anuj Mittal
` (4 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Anuj Mittal @ 2021-11-09 14:50 UTC (permalink / raw)
To: bitbake-devel
From: Richard Purdie <richard.purdie@linuxfoundation.org>
Username or password replacements in URIs were being appended rather than
replaced in mirror url remapping. Fix this and add a test case.
[YOCTO #13823]
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit 66ad58bb87e5158aced572be4f1d5726bc97fcce)
Signed-off-by: Anuj Mittal <anuj.mittal@intel.com>
---
lib/bb/fetch2/__init__.py | 4 ++++
lib/bb/tests/fetch.py | 4 ++++
2 files changed, 8 insertions(+)
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index 8078991d7..b73b36951 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -430,6 +430,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
uri_replace_decoded = list(decodeurl(uri_replace))
logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded))
result_decoded = ['', '', '', '', '', {}]
+ # 0 - type, 1 - host, 2 - path, 3 - user, 4- pswd, 5 - params
for loc, i in enumerate(uri_find_decoded):
result_decoded[loc] = uri_decoded[loc]
regexp = i
@@ -449,6 +450,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
for l in replacements:
uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l])
result_decoded[loc][k] = uri_replace_decoded[loc][k]
+ elif (loc == 3 or loc == 4) and uri_replace_decoded[loc]:
+ # User/password in the replacement is just a straight replacement
+ result_decoded[loc] = uri_replace_decoded[loc]
elif (re.match(regexp, uri_decoded[loc])):
if not uri_replace_decoded[loc]:
result_decoded[loc] = ""
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 18f1c5e9e..25abf67ac 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -431,6 +431,10 @@ class MirrorUriTest(FetcherTest):
("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
: "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
+ ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http")
+ : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
+
+
#Renaming files doesn't work
#("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
#("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
--
2.31.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [1.52][PATCH 3/7] cooker: check if upstream hash equivalence server is available
2021-11-09 14:50 [1.52][PATCH 0/7] Review request Anuj Mittal
2021-11-09 14:50 ` [1.52][PATCH 1/7] tests/fetch: Update github urls Anuj Mittal
2021-11-09 14:50 ` [1.52][PATCH 2/7] fetch: Handle mirror user/password replacements correctly Anuj Mittal
@ 2021-11-09 14:50 ` Anuj Mittal
2021-11-09 14:50 ` [1.52][PATCH 4/7] tests/fetch: Update pcre.org address after github changes Anuj Mittal
` (3 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Anuj Mittal @ 2021-11-09 14:50 UTC (permalink / raw)
To: bitbake-devel
From: Jose Quaresma <quaresma.jose@gmail.com>
When the user specify an invalid upstream hash equivalence server in
BB_HASHSERVE_UPSTREAM notify the user that we can't connect the server.
Signed-off-by: Jose Quaresma <quaresma.jose@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit be45aeb9a84f30c28711e87e2d2a4a86320a8d94)
Signed-off-by: Anuj Mittal <anuj.mittal@intel.com>
---
lib/bb/cooker.py | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index af794b4c4..06b40c138 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -388,12 +388,22 @@ class BBCooker:
# Create a new hash server bound to a unix domain socket
if not self.hashserv:
dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
+ upstream = self.data.getVar("BB_HASHSERVE_UPSTREAM") or None
+ if upstream:
+ import socket
+ try:
+ sock = socket.create_connection(upstream.split(":"), 5)
+ sock.close()
+ except socket.error as e:
+ bb.warn("BB_HASHSERVE_UPSTREAM is not valid, unable to connect hash equivalence server at '%s': %s"
+ % (upstream, repr(e)))
+
self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR")
self.hashserv = hashserv.create_server(
self.hashservaddr,
dbfile,
sync=False,
- upstream=self.data.getVar("BB_HASHSERVE_UPSTREAM") or None,
+ upstream=upstream,
)
self.hashserv.serve_as_process()
self.data.setVar("BB_HASHSERVE", self.hashservaddr)
--
2.31.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [1.52][PATCH 4/7] tests/fetch: Update pcre.org address after github changes
2021-11-09 14:50 [1.52][PATCH 0/7] Review request Anuj Mittal
` (2 preceding siblings ...)
2021-11-09 14:50 ` [1.52][PATCH 3/7] cooker: check if upstream hash equivalence server is available Anuj Mittal
@ 2021-11-09 14:50 ` Anuj Mittal
2021-11-09 14:50 ` [1.52][PATCH 5/7] cooker: Handle parse threads disappearing to avoid hangs Anuj Mittal
` (2 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Anuj Mittal @ 2021-11-09 14:50 UTC (permalink / raw)
To: bitbake-devel
From: Richard Purdie <richard.purdie@linuxfoundation.org>
vcs.pcre.org was a redirect to github which we use for subversion testing.
With the protocol changes at github and the removal of the redirect, use a
direct address for github.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit 6230ca71eb7eb2a6db162e28a01727d00af5299b)
Signed-off-by: Anuj Mittal <anuj.mittal@intel.com>
---
lib/bb/tests/fetch.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 25abf67ac..7edbf50b5 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -1136,7 +1136,7 @@ class SVNTest(FetcherTest):
bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
# Github will emulate SVN. Use this to check if we're downloding...
- bb.process.run("svn propset svn:externals 'bitbake svn://vcs.pcre.org/pcre2/code' .",
+ bb.process.run("svn propset svn:externals 'bitbake https://github.com/PhilipHazel/pcre2.git' .",
cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
bb.process.run("svn commit --non-interactive -m 'Add external'",
cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
--
2.31.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [1.52][PATCH 5/7] cooker: Handle parse threads disappearing to avoid hangs
2021-11-09 14:50 [1.52][PATCH 0/7] Review request Anuj Mittal
` (3 preceding siblings ...)
2021-11-09 14:50 ` [1.52][PATCH 4/7] tests/fetch: Update pcre.org address after github changes Anuj Mittal
@ 2021-11-09 14:50 ` Anuj Mittal
2021-11-11 11:16 ` [bitbake-devel] " Richard Purdie
2021-11-09 14:50 ` [1.52][PATCH 6/7] runqueue: Fix runall option task deletion ordering issue Anuj Mittal
2021-11-09 14:50 ` [1.52][PATCH 7/7] runqueue: Fix runall option handling Anuj Mittal
6 siblings, 1 reply; 10+ messages in thread
From: Anuj Mittal @ 2021-11-09 14:50 UTC (permalink / raw)
To: bitbake-devel
From: Richard Purdie <richard.purdie@linuxfoundation.org>
If one of the parse threads disappears during parsing for some reason, bitbake
currently hangs. Avoid this (and zombie threads hanging around) by joining()
threads which have exited.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit dc86a533d951d13643ce446533370da804782afc)
Signed-off-by: Anuj Mittal <anuj.mittal@intel.com>
---
lib/bb/cooker.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 06b40c138..8ae8e4ecc 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -2040,6 +2040,7 @@ class Parser(multiprocessing.Process):
result = pending.pop()
else:
try:
+ time.sleep(0.25)
job = self.jobs.pop()
except IndexError:
self.results.close()
@@ -2218,7 +2219,7 @@ class CookerParser(object):
yield not cached, mc, infos
def parse_generator(self):
- while True:
+ while self.processes:
if self.parsed >= self.toparse:
break
@@ -2232,6 +2233,14 @@ class CookerParser(object):
raise value
else:
yield result
+ for process in self.processes.copy():
+ if not process.is_alive():
+ process.join()
+ self.processes.remove(process)
+
+ if not (self.parsed >= self.toparse):
+ raise bb.parse.ParseError("Not all recipes parsed, parser thread killed/died? Exiting.", None)
+
def parse_next(self):
result = []
--
2.31.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [bitbake-devel] [1.52][PATCH 5/7] cooker: Handle parse threads disappearing to avoid hangs
2021-11-09 14:50 ` [1.52][PATCH 5/7] cooker: Handle parse threads disappearing to avoid hangs Anuj Mittal
@ 2021-11-11 11:16 ` Richard Purdie
2021-11-12 2:50 ` Mittal, Anuj
0 siblings, 1 reply; 10+ messages in thread
From: Richard Purdie @ 2021-11-11 11:16 UTC (permalink / raw)
To: Anuj Mittal, bitbake-devel
On Tue, 2021-11-09 at 22:50 +0800, Anuj Mittal wrote:
> From: Richard Purdie <richard.purdie@linuxfoundation.org>
>
> If one of the parse threads disappears during parsing for some reason, bitbake
> currently hangs. Avoid this (and zombie threads hanging around) by joining()
> threads which have exited.
>
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> (cherry picked from commit dc86a533d951d13643ce446533370da804782afc)
> Signed-off-by: Anuj Mittal <anuj.mittal@intel.com>
> ---
> lib/bb/cooker.py | 11 ++++++++++-
> 1 file changed, 10 insertions(+), 1 deletion(-)
>
> diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
> index 06b40c138..8ae8e4ecc 100644
> --- a/lib/bb/cooker.py
> +++ b/lib/bb/cooker.py
> @@ -2040,6 +2040,7 @@ class Parser(multiprocessing.Process):
> result = pending.pop()
> else:
> try:
> + time.sleep(0.25)
> job = self.jobs.pop()
> except IndexError:
> self.results.close()
> @@ -2218,7 +2219,7 @@ class CookerParser(object):
> yield not cached, mc, infos
>
> def parse_generator(self):
> - while True:
> + while self.processes:
> if self.parsed >= self.toparse:
> break
>
> @@ -2232,6 +2233,14 @@ class CookerParser(object):
> raise value
> else:
> yield result
> + for process in self.processes.copy():
> + if not process.is_alive():
> + process.join()
> + self.processes.remove(process)
> +
> + if not (self.parsed >= self.toparse):
> + raise bb.parse.ParseError("Not all recipes parsed, parser thread killed/died? Exiting.", None)
> +
>
> def parse_next(self):
> result = []
This patch changed a bit in master due to issues being found so we'll see to
include those changes in the backport.
Cheers,
Richard
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [bitbake-devel] [1.52][PATCH 5/7] cooker: Handle parse threads disappearing to avoid hangs
2021-11-11 11:16 ` [bitbake-devel] " Richard Purdie
@ 2021-11-12 2:50 ` Mittal, Anuj
0 siblings, 0 replies; 10+ messages in thread
From: Mittal, Anuj @ 2021-11-12 2:50 UTC (permalink / raw)
To: richard.purdie@linuxfoundation.org,
bitbake-devel@lists.openembedded.org
On Thu, 2021-11-11 at 11:16 +0000, Richard Purdie wrote:
> On Tue, 2021-11-09 at 22:50 +0800, Anuj Mittal wrote:
> > From: Richard Purdie <richard.purdie@linuxfoundation.org>
> >
> > If one of the parse threads disappears during parsing for some
> > reason, bitbake
> > currently hangs. Avoid this (and zombie threads hanging around) by
> > joining()
> > threads which have exited.
> >
> > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> > (cherry picked from commit
> > dc86a533d951d13643ce446533370da804782afc)
> > Signed-off-by: Anuj Mittal <anuj.mittal@intel.com>
> > ---
> > lib/bb/cooker.py | 11 ++++++++++-
> > 1 file changed, 10 insertions(+), 1 deletion(-)
> >
> > diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
> > index 06b40c138..8ae8e4ecc 100644
> > --- a/lib/bb/cooker.py
> > +++ b/lib/bb/cooker.py
> > @@ -2040,6 +2040,7 @@ class Parser(multiprocessing.Process):
> > result = pending.pop()
> > else:
> > try:
> > + time.sleep(0.25)
> > job = self.jobs.pop()
> > except IndexError:
> > self.results.close()
> > @@ -2218,7 +2219,7 @@ class CookerParser(object):
> > yield not cached, mc, infos
> >
> > def parse_generator(self):
> > - while True:
> > + while self.processes:
> > if self.parsed >= self.toparse:
> > break
> >
> > @@ -2232,6 +2233,14 @@ class CookerParser(object):
> > raise value
> > else:
> > yield result
> > + for process in self.processes.copy():
> > + if not process.is_alive():
> > + process.join()
> > + self.processes.remove(process)
> > +
> > + if not (self.parsed >= self.toparse):
> > + raise bb.parse.ParseError("Not all recipes parsed,
> > parser thread killed/died? Exiting.", None)
> > +
> >
> > def parse_next(self):
> > result = []
>
> This patch changed a bit in master due to issues being found so we'll
> see to
> include those changes in the backport.
>
I have dropped this patch from my branch for now and will include both
this and subsequent fix later.
Thanks,
Anuj
^ permalink raw reply [flat|nested] 10+ messages in thread
* [1.52][PATCH 6/7] runqueue: Fix runall option task deletion ordering issue
2021-11-09 14:50 [1.52][PATCH 0/7] Review request Anuj Mittal
` (4 preceding siblings ...)
2021-11-09 14:50 ` [1.52][PATCH 5/7] cooker: Handle parse threads disappearing to avoid hangs Anuj Mittal
@ 2021-11-09 14:50 ` Anuj Mittal
2021-11-09 14:50 ` [1.52][PATCH 7/7] runqueue: Fix runall option handling Anuj Mittal
6 siblings, 0 replies; 10+ messages in thread
From: Anuj Mittal @ 2021-11-09 14:50 UTC (permalink / raw)
To: bitbake-devel
From: Richard Purdie <richard.purdie@linuxfoundation.org>
The runbuild option handling in runqueue was flawed as items deleted from the
main task list may be dependencies and hence cause index errors.
Rather than modify runtaskentries straight away, compute a new shorted list
and use that as an input to the second phase. This avoids the need to add tasks
back to the list meaning delcount can be simplifed to a simple counter.
The second use case in runonly doen't re-add items so doesn't have this
issue.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit 3428e3c54eb5cc03ff96f9cee6dc839afee7a419)
Signed-off-by: Anuj Mittal <anuj.mittal@intel.com>
---
lib/bb/runqueue.py | 35 ++++++++++++++++-------------------
1 file changed, 16 insertions(+), 19 deletions(-)
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 10511a09d..3300bb4f9 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -926,38 +926,35 @@ class RunQueueData:
#
# Once all active tasks are marked, prune the ones we don't need.
- delcount = {}
- for tid in list(self.runtaskentries.keys()):
- if tid not in runq_build:
- delcount[tid] = self.runtaskentries[tid]
- del self.runtaskentries[tid]
-
# Handle --runall
if self.cooker.configuration.runall:
# re-run the mark_active and then drop unused tasks from new list
- runq_build = {}
+ reduced_tasklist = set(self.runtaskentries.keys())
+ for tid in list(self.runtaskentries.keys()):
+ if tid not in runq_build:
+ reduced_tasklist.remove(tid)
for task in self.cooker.configuration.runall:
if not task.startswith("do_"):
task = "do_{0}".format(task)
runall_tids = set()
- for tid in list(self.runtaskentries):
+ for tid in reduced_tasklist:
wanttid = "{0}:{1}".format(fn_from_tid(tid), task)
- if wanttid in delcount:
- self.runtaskentries[wanttid] = delcount[wanttid]
if wanttid in self.runtaskentries:
runall_tids.add(wanttid)
for tid in list(runall_tids):
- mark_active(tid,1)
+ mark_active(tid, 1)
if self.cooker.configuration.force:
invalidate_task(tid, False)
- for tid in list(self.runtaskentries.keys()):
- if tid not in runq_build:
- delcount[tid] = self.runtaskentries[tid]
- del self.runtaskentries[tid]
+ delcount = set()
+ for tid in list(self.runtaskentries.keys()):
+ if tid not in runq_build:
+ delcount.add(tid)
+ del self.runtaskentries[tid]
+ if self.cooker.configuration.runall:
if len(self.runtaskentries) == 0:
bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the recipes of the taskgraphs of the targets %s" % (str(self.cooker.configuration.runall), str(self.targets)))
@@ -971,16 +968,16 @@ class RunQueueData:
for task in self.cooker.configuration.runonly:
if not task.startswith("do_"):
task = "do_{0}".format(task)
- runonly_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == task }
+ runonly_tids = [k for k in self.runtaskentries.keys() if taskname_from_tid(k) == task]
- for tid in list(runonly_tids):
- mark_active(tid,1)
+ for tid in runonly_tids:
+ mark_active(tid, 1)
if self.cooker.configuration.force:
invalidate_task(tid, False)
for tid in list(self.runtaskentries.keys()):
if tid not in runq_build:
- delcount[tid] = self.runtaskentries[tid]
+ delcount.add(tid)
del self.runtaskentries[tid]
if len(self.runtaskentries) == 0:
--
2.31.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [1.52][PATCH 7/7] runqueue: Fix runall option handling
2021-11-09 14:50 [1.52][PATCH 0/7] Review request Anuj Mittal
` (5 preceding siblings ...)
2021-11-09 14:50 ` [1.52][PATCH 6/7] runqueue: Fix runall option task deletion ordering issue Anuj Mittal
@ 2021-11-09 14:50 ` Anuj Mittal
6 siblings, 0 replies; 10+ messages in thread
From: Anuj Mittal @ 2021-11-09 14:50 UTC (permalink / raw)
To: bitbake-devel
From: Richard Purdie <richard.purdie@linuxfoundation.org>
The previous fix for runall option handling had a small bug in it, it
didn't clear the originally processed task list which meant it was running
too many tasks. Fix this so the list is reset and rebuild correctly.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit 87c9e120897ed04dfc64d4752fc602f9bfcb8645)
Signed-off-by: Anuj Mittal <anuj.mittal@intel.com>
---
lib/bb/runqueue.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 3300bb4f9..cd10da8b3 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -933,6 +933,7 @@ class RunQueueData:
for tid in list(self.runtaskentries.keys()):
if tid not in runq_build:
reduced_tasklist.remove(tid)
+ runq_build = {}
for task in self.cooker.configuration.runall:
if not task.startswith("do_"):
--
2.31.1
^ permalink raw reply related [flat|nested] 10+ messages in thread