[PATCH v2 5/5] contrib: add python service and systemd unit to run shared jobserver

Martin Hundebøll
2024-04-04T11:38:23Z

For CI setups that might end up building multiple yocto builds in
parallel, a shared jobserver can reduce the total load of the system.
Setting up such a jobserver is simple, but it does require a process
hanging around to keep the jobserver fifo open (to avoid blocking token
requests).

Add a simple python script that creates such a jobserver fifo and waits
forever. Also add a systemd unit file to start the python service at
boot.

The systemd unit can be installed in $HOME/.config/systemd/user/, but
one might need to add a droplet config (i.e. `systemctl --user edit
jobserver.service`) to setup the PYTHONPATH variable to make the python
script loadable.

Signed-off-by: Martin Hundebøll <martin@geanix.com>
---
contrib/jobserver/jobserver.py | 78 +++++++++++++++++++++++++++++
contrib/jobserver/jobserver.service | 10 ++++
2 files changed, 88 insertions(+)
create mode 100644 contrib/jobserver/jobserver.py
create mode 100644 contrib/jobserver/jobserver.service

diff --git a/contrib/jobserver/jobserver.py b/contrib/jobserver/jobserver.py
new file mode 100644
index 0000000000..52f9711277
--- /dev/null
+++ b/contrib/jobserver/jobserver.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+
+from pathlib import Path
+from threading import Event
+import argparse
+import os
+import shutil
+import signal
+
+resumed = Event()
+runtime_dir = os.environ.get("XDG_RUNTIME_DIR", "/run")
+
+def signal_handler(signum, _frame):
+ """Wait for an external signal to exit the process gracefully."""
+ resumed.set()
+
+
+def main(path, user, group, mode, jobs):
+ """Setup a fifo to use as jobserver shared between builds."""
+ try:
+ path.unlink(missing_ok=True)
+ os.mkfifo(path)
+ shutil.chown(path, user, group)
+ os.chmod(path, mode)
+ except (FileNotFoundError, PermissionError) as exc:
+ raise SystemExit(f"failed to create fifo: {path}: {exc.strerror}")
+
+ print(f"jobserver: {path}: {jobs} jobs")
+ fifo = os.open(path, os.O_RDWR)
+ os.write(fifo, b"+" * jobs)
+
+ print("jobserver: ready; waiting indefinitely")
+ signal.signal(signal.SIGTERM, signal_handler)
+ signal.signal(signal.SIGINT, signal_handler)
+ resumed.wait()
+
+ print("jobserver: exiting")
+ path.unlink()
+ os.close(fifo)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(
+ prog='Make jobserver',
+ description='Simple application to instantiate a jobserver fifo and hang around',
+ )
+ parser.add_argument(
+ "--mode",
+ help="Permission to apply to jobserver fifo",
+ type=lambda v: int(v, 8),
+ default=0o0666,
+ )
+ parser.add_argument(
+ "--user",
+ help="Username or id to assign ownership of fifo to",
+ default=os.getuid(),
+ )
+ parser.add_argument(
+ "--group",
+ help="Groupname or id to assign ownership of fifo to",
+ default=os.getgid(),
+ )
+ parser.add_argument(
+ "path",
+ help="Path to jobserver fifo",
+ type=Path,
+ nargs='?',
+ default=f"{runtime_dir}/jobserver",
+ )
+ parser.add_argument(
+ "jobs",
+ help="Number of tokens to load jobserver with",
+ type=int,
+ nargs='?',
+ default=os.cpu_count(),
+ )
+ args = parser.parse_args()
+ main(args.path, args.user, args.group, args.mode, args.jobs)
diff --git a/contrib/jobserver/jobserver.service b/contrib/jobserver/jobserver.service
new file mode 100644
index 0000000000..bbc7167ac0
--- /dev/null
+++ b/contrib/jobserver/jobserver.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Shared jobserver fifo
+
+[Service]
+Type=simple
+Environment=PYTHONUNBUFFERED=1
+ExecStart=python jobserver.py
+
+[Install]
+WantedBy=multi-user.target
--
2.44.0