Openembedded Core Discussions
 help / color / mirror / Atom feed
From: Joshua Watt <jpewhacker@gmail.com>
To: bitbake-devel@lists.openembedded.org,
	openembedded-core@lists.openembedded.org
Subject: [RFC v2 12/16] bitbake: runqueue: Pass dependency ID to task
Date: Thu,  9 Aug 2018 17:08:36 -0500	[thread overview]
Message-ID: <20180809220840.26697-13-JPEWhacker@gmail.com> (raw)
In-Reply-To: <20180809220840.26697-1-JPEWhacker@gmail.com>

The dependency ID is now passed to the task in the BB_DEPID variable

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
 bitbake/bin/bitbake-worker |  7 ++++---
 bitbake/lib/bb/runqueue.py | 10 ++++++----
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker
index 41ef6d848ac..9650c954359 100755
--- a/bitbake/bin/bitbake-worker
+++ b/bitbake/bin/bitbake-worker
@@ -136,7 +136,7 @@ def sigterm_handler(signum, frame):
     os.killpg(0, signal.SIGTERM)
     sys.exit()
 
-def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskhash, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
+def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskhash, depid, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
     # We need to setup the environment BEFORE the fork, since
     # a fork() or exec*() activates PSEUDO...
 
@@ -235,6 +235,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
 
                 the_data = bb_cache.loadDataFull(fn, appends)
                 the_data.setVar('BB_TASKHASH', taskhash)
+                the_data.setVar('BB_DEPID', depid)
 
                 bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", "")))
 
@@ -425,10 +426,10 @@ class BitbakeWorker(object):
         sys.exit(0)
 
     def handle_runtask(self, data):
-        fn, task, taskname, taskhash, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
+        fn, task, taskname, taskhash, depid, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
         workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
 
-        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, taskhash, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
+        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, taskhash, depid, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
 
         self.build_pids[pid] = task
         self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 3ae12d8e69f..909fa3aec22 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -2028,6 +2028,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
 
             taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
             taskhash = self.rqdata.get_task_hash(task)
+            depid = self.rqdata.get_task_depid(task)
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce):
                 if not mc in self.rq.fakeworker:
                     try:
@@ -2037,10 +2038,10 @@ class RunQueueExecuteTasks(RunQueueExecute):
                         self.rq.state = runQueueFailed
                         self.stats.taskFailed()
                         return True
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
+                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, depid, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
+                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, depid, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
                 self.rq.worker[mc].process.stdin.flush()
 
             self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
@@ -2455,13 +2456,14 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
 
             taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
             taskhash = self.rqdata.get_task_hash(task)
+            depid = self.rqdata.get_task_depid(task)
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
                 if not mc in self.rq.fakeworker:
                     self.rq.start_fakeworker(self, mc)
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
+                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, depid, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
+                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, depid, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
                 self.rq.worker[mc].process.stdin.flush()
 
             self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
-- 
2.17.1



  parent reply	other threads:[~2018-08-09 22:10 UTC|newest]

Thread overview: 95+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2018-07-16 20:37 [RFC 0/9] Hash Equivalency Server Joshua Watt
2018-07-16 20:37 ` [RFC 1/9] bitbake-worker: Pass taskhash as runtask parameter Joshua Watt
2018-07-16 20:37 ` [RFC 2/9] siggen: Split out stampfile hash fetch Joshua Watt
2018-07-16 20:37 ` [RFC 3/9] siggen: Split out task depend ID Joshua Watt
2018-07-16 20:37 ` [RFC 4/9] runqueue: Track task dependency ID Joshua Watt
2018-07-16 20:37 ` [RFC 5/9] runqueue: Pass dependency ID to task Joshua Watt
2018-07-16 20:37 ` [RFC 6/9] runqueue: Pass dependency ID to hash validate Joshua Watt
2018-07-16 20:37 ` [RFC 7/9] classes/sstate: Handle depid in hash check Joshua Watt
2018-07-16 20:37 ` [RFC 8/9] hashserver: Add initial reference server Joshua Watt
2018-07-17 12:11   ` [bitbake-devel] " Richard Purdie
2018-07-17 13:44     ` Joshua Watt
2018-07-18 13:53     ` Joshua Watt
2018-07-16 20:37 ` [RFC 9/9] sstate: Implement hash equivalence sstate Joshua Watt
2018-08-09 22:08 ` [RFC v2 00/16] Hash Equivalency Server Joshua Watt
2018-08-09 22:08   ` [RFC v2 01/16] bitbake: fork: Add os.fork() wrappers Joshua Watt
2018-08-09 22:08   ` [RFC v2 02/16] bitbake: persist_data: Fix leaking cursors causing deadlock Joshua Watt
2018-08-09 22:08   ` [RFC v2 03/16] bitbake: persist_data: Add key constraints Joshua Watt
2018-08-09 22:08   ` [RFC v2 04/16] bitbake: persist_data: Enable Write Ahead Log Joshua Watt
2018-08-09 22:08   ` [RFC v2 05/16] bitbake: persist_data: Disable enable_shared_cache Joshua Watt
2018-08-09 22:08   ` [RFC v2 06/16] bitbake: persist_data: Close databases across fork Joshua Watt
2018-08-09 22:08   ` [RFC v2 07/16] bitbake: tests/persist_data: Add tests Joshua Watt
2018-08-09 22:08   ` [RFC v2 08/16] bitbake: bitbake-worker: Pass taskhash as runtask parameter Joshua Watt
2018-08-09 22:08   ` [RFC v2 09/16] bitbake: siggen: Split out stampfile hash fetch Joshua Watt
2018-08-09 22:08   ` [RFC v2 10/16] bitbake: siggen: Split out task depend ID Joshua Watt
2018-08-09 22:08   ` [RFC v2 11/16] bitbake: runqueue: Track task dependency ID Joshua Watt
2018-08-09 22:08   ` Joshua Watt [this message]
2018-08-09 22:08   ` [RFC v2 13/16] bitbake: runqueue: Pass dependency ID to hash validate Joshua Watt
2018-08-09 22:08   ` [RFC v2 14/16] classes/sstate: Handle depid in hash check Joshua Watt
2018-08-09 22:08   ` [RFC v2 15/16] bitbake: hashserv: Add hash equivalence reference server Joshua Watt
2018-08-09 22:08   ` [RFC v2 16/16] sstate: Implement hash equivalence sstate Joshua Watt
2018-12-04  3:42   ` [PATCH v3 00/17] Hash Equivalency Server Joshua Watt
2018-12-04  3:42     ` [PATCH v3 01/17] bitbake: fork: Add os.fork() wrappers Joshua Watt
2018-12-04  3:42     ` [PATCH v3 02/17] bitbake: persist_data: Fix leaking cursors causing deadlock Joshua Watt
2018-12-04  3:42     ` [PATCH v3 03/17] bitbake: persist_data: Add key constraints Joshua Watt
2018-12-04  3:42     ` [PATCH v3 04/17] bitbake: persist_data: Enable Write Ahead Log Joshua Watt
2018-12-04  3:42     ` [PATCH v3 05/17] bitbake: persist_data: Disable enable_shared_cache Joshua Watt
2018-12-04  3:42     ` [PATCH v3 06/17] bitbake: persist_data: Close databases across fork Joshua Watt
2018-12-04  3:42     ` [PATCH v3 07/17] bitbake: tests/persist_data: Add tests Joshua Watt
2018-12-04  3:42     ` [PATCH v3 08/17] bitbake: bitbake-worker: Pass taskhash as runtask parameter Joshua Watt
2018-12-04  3:42     ` [PATCH v3 09/17] bitbake: siggen: Split out stampfile hash fetch Joshua Watt
2018-12-04  3:42     ` [PATCH v3 10/17] bitbake: siggen: Split out task depend ID Joshua Watt
2018-12-05 22:50       ` [bitbake-devel] " Richard Purdie
2018-12-06 14:58         ` Joshua Watt
2018-12-04  3:42     ` [PATCH v3 11/17] bitbake: runqueue: Track task dependency ID Joshua Watt
2018-12-04  3:42     ` [PATCH v3 12/17] bitbake: runqueue: Pass dependency ID to task Joshua Watt
2018-12-04  3:42     ` [PATCH v3 13/17] bitbake: runqueue: Pass dependency ID to hash validate Joshua Watt
2018-12-05 22:52       ` [bitbake-devel] " Richard Purdie
2018-12-04  3:42     ` [PATCH v3 14/17] classes/sstate: Handle depid in hash check Joshua Watt
2018-12-04  3:42     ` [PATCH v3 15/17] bitbake: hashserv: Add hash equivalence reference server Joshua Watt
2018-12-04  3:42     ` [PATCH v3 16/17] sstate: Implement hash equivalence sstate Joshua Watt
2018-12-04  3:42     ` [PATCH v3 17/17] classes/image-buildinfo: Remove unused argument Joshua Watt
2018-12-18 15:30     ` [PATCH v4 00/10] Hash Equivalency Server Joshua Watt
2018-12-18 15:30       ` [PATCH v4 01/10] bitbake: fork: Add os.fork() wrappers Joshua Watt
2018-12-18 15:30       ` [PATCH v4 02/10] bitbake: persist_data: Close databases across fork Joshua Watt
2018-12-18 15:30       ` [PATCH v4 03/10] bitbake: tests/persist_data: Add tests Joshua Watt
2018-12-18 15:30       ` [PATCH v4 04/10] bitbake: siggen: Split out task unique hash Joshua Watt
2018-12-18 15:30       ` [PATCH v4 05/10] bitbake: runqueue: Track " Joshua Watt
2018-12-18 15:30       ` [PATCH v4 06/10] bitbake: runqueue: Pass unique hash to task Joshua Watt
2018-12-18 15:30       ` [PATCH v4 07/10] bitbake: runqueue: Pass unique hash to hash validate Joshua Watt
2018-12-18 16:24         ` Richard Purdie
2018-12-18 16:31           ` Joshua Watt
2018-12-18 15:30       ` [PATCH v4 08/10] classes/sstate: Handle unihash in hash check Joshua Watt
2018-12-18 15:31       ` [PATCH v4 09/10] bitbake: hashserv: Add hash equivalence reference server Joshua Watt
2018-12-18 15:31       ` [PATCH v4 10/10] sstate: Implement hash equivalence sstate Joshua Watt
2018-12-19  3:10       ` [PATCH v5 0/8] Hash Equivalency Server Joshua Watt
2018-12-19  3:10         ` [PATCH v5 1/8] bitbake: tests/persist_data: Add tests Joshua Watt
2018-12-19  3:10         ` [PATCH v5 2/8] bitbake: siggen: Split out task unique hash Joshua Watt
2018-12-19  3:10         ` [PATCH v5 3/8] bitbake: runqueue: Track " Joshua Watt
2019-01-05  7:49           ` Alejandro Hernandez
2019-01-06  3:09             ` Joshua Watt
2019-01-07  6:52               ` Alejandro Hernandez
2019-01-07 16:16               ` akuster808
2019-01-07 16:40                 ` Joshua Watt
2018-12-19  3:10         ` [PATCH v5 4/8] bitbake: runqueue: Pass unique hash to task Joshua Watt
2018-12-19  3:10         ` [PATCH v5 5/8] bitbake: runqueue: Pass unique hash to hash validate Joshua Watt
2018-12-19  3:10         ` [PATCH v5 6/8] classes/sstate: Handle unihash in hash check Joshua Watt
2018-12-19  3:10         ` [PATCH v5 7/8] bitbake: hashserv: Add hash equivalence reference server Joshua Watt
2018-12-19  3:10         ` [PATCH v5 8/8] sstate: Implement hash equivalence sstate Joshua Watt
2018-12-19  3:33       ` ✗ patchtest: failure for Hash Equivalency Server (rev3) Patchwork
2019-01-04  2:42       ` [PATCH v6 0/3] Hash Equivalency Server Joshua Watt
2019-01-04  2:42         ` [PATCH v6 1/3] classes/sstate: Handle unihash in hash check Joshua Watt
2019-01-04  7:01           ` [bitbake-devel] " Richard Purdie
2019-01-04  2:42         ` [PATCH v6 2/3] bitbake: hashserv: Add hash equivalence reference server Joshua Watt
2019-01-04  2:42         ` [PATCH v6 3/3] sstate: Implement hash equivalence sstate Joshua Watt
2019-01-04 16:20         ` [PATCH v7 0/3] Hash Equivalency Server Joshua Watt
2019-01-04 16:20           ` [PATCH v7 1/3] classes/sstate: Handle unihash in hash check Joshua Watt
2019-01-04 16:20           ` [PATCH v7 2/3] bitbake: hashserv: Add hash equivalence reference server Joshua Watt
2019-01-04 16:20           ` [PATCH v7 3/3] sstate: Implement hash equivalence sstate Joshua Watt
2019-01-08  6:29             ` [bitbake-devel] " Jacob Kroon
2019-01-09 17:09               ` Joshua Watt
2019-01-11 20:39                 ` Peter Kjellerstedt
2019-01-04 16:33         ` ✗ patchtest: failure for Hash Equivalency Server (rev5) Patchwork
2019-01-04  3:03       ` ✗ patchtest: failure for Hash Equivalency Server (rev4) Patchwork
2018-12-18 16:03     ` ✗ patchtest: failure for Hash Equivalency Server (rev2) Patchwork
2018-12-04  4:05   ` ✗ patchtest: failure for Hash Equivalency Server Patchwork

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20180809220840.26697-13-JPEWhacker@gmail.com \
    --to=jpewhacker@gmail.com \
    --cc=bitbake-devel@lists.openembedded.org \
    --cc=openembedded-core@lists.openembedded.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox