* [PATCH 0/2] hashserv: tighten unihash handling
@ 2026-06-05 4:58 Anders Heimer
2026-06-05 4:58 ` [PATCH 1/2] hashserv/tests: use valid 64-character unihashes Anders Heimer
2026-06-05 4:58 ` [PATCH 2/2] hashserv: validate unihash values Anders Heimer
0 siblings, 2 replies; 3+ messages in thread
From: Anders Heimer @ 2026-06-05 4:58 UTC (permalink / raw)
To: bitbake-devel
This series tightens unihash handling so hashserv accepts and stores only
valid lowercase 64-character hexadecimal unihash values, and BitBake treats
BB_UNIHASH as literal hash data when reading it from the datastore.
The first patch updates existing hashserv test fixtures to use valid
64-character unihash-shaped values.
The second patch adds unihash validation and focused coverage for hashserv
report paths and BitBake siggen handling. Invalid unihashes reported to
hashserv, or returned by hashserv to BitBake, are rejected before use.
Validation:
- py_compile on touched Python files
- focused hashserv/siggen tests
- broader hashserv and siggen regression tests
Anders Heimer (2):
hashserv/tests: use valid 64-character unihashes
hashserv: validate unihash values
lib/bb/siggen.py | 9 +-
lib/bb/tests/siggen.py | 47 ++++++++++
lib/hashserv/__init__.py | 7 ++
lib/hashserv/server.py | 27 +++++-
lib/hashserv/tests.py | 194 ++++++++++++++++++++++++++++-----------
5 files changed, 224 insertions(+), 60 deletions(-)
--
2.34.1
^ permalink raw reply [flat|nested] 3+ messages in thread
* [PATCH 1/2] hashserv/tests: use valid 64-character unihashes
2026-06-05 4:58 [PATCH 0/2] hashserv: tighten unihash handling Anders Heimer
@ 2026-06-05 4:58 ` Anders Heimer
2026-06-05 4:58 ` [PATCH 2/2] hashserv: validate unihash values Anders Heimer
1 sibling, 0 replies; 3+ messages in thread
From: Anders Heimer @ 2026-06-05 4:58 UTC (permalink / raw)
To: bitbake-devel
Signed-off-by: Anders Heimer <anders.heimer@est.tech>
---
lib/hashserv/tests.py | 164 ++++++++++++++++++++++++++++--------------
1 file changed, 110 insertions(+), 54 deletions(-)
diff --git a/lib/hashserv/tests.py b/lib/hashserv/tests.py
index 124d8aa00..83ce0c5ca 100644
--- a/lib/hashserv/tests.py
+++ b/lib/hashserv/tests.py
@@ -28,6 +28,7 @@ from pathlib import Path
THIS_DIR = Path(__file__).parent
BIN_DIR = THIS_DIR.parent.parent / "bin"
+
def server_prefunc(server, idx):
logging.basicConfig(level=logging.DEBUG, filename='bbhashserv-%d.log' % idx, filemode='w',
format='%(levelname)s %(filename)s:%(lineno)d %(message)s')
@@ -124,7 +125,7 @@ class HashEquivalenceTestSetup(object):
# Simple test that hashes can be created
taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
- unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+ unihash = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9'
self.assertClientGetHash(client, taskhash, None)
@@ -178,7 +179,7 @@ class HashEquivalenceCommonTests(object):
# assigned the same unihash
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
- unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+ unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
@@ -186,7 +187,7 @@ class HashEquivalenceCommonTests(object):
# Report a different task with the same outhash. The returned unihash
# should match the first task
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
- unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+ unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
result = self.client.report_unihash(taskhash2, self.METHOD, outhash, unihash2)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
@@ -196,19 +197,19 @@ class HashEquivalenceCommonTests(object):
# taskhash
taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a'
outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e'
- unihash = '218e57509998197d570e2c98512d0105985dffc9'
+ unihash = '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380'
self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertClientGetHash(self.client, taskhash, unihash)
outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d'
- unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'
+ unihash2 = 'a37541b54fd22440e292f617eb30ba07455e88fb0b9f0952eca229b6356290e3'
self.client.report_unihash(taskhash, self.METHOD, outhash2, unihash2)
self.assertClientGetHash(self.client, taskhash, unihash)
outhash3 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
- unihash3 = '9217a7d6398518e5dc002ed58f2cbbbc78696603'
+ unihash3 = '6842f1f2daccd96ddef15c9154d4e41ac8a2300d781ac9a9db7f8afeb8a96808'
self.client.report_unihash(taskhash, self.METHOD, outhash3, unihash3)
self.assertClientGetHash(self.client, taskhash, unihash)
@@ -268,7 +269,7 @@ class HashEquivalenceCommonTests(object):
# Simple test that hashes can be created
taskhash = 'c665584ee6817aa99edfc77a44dd853828279370'
outhash = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44'
- unihash = '90e9bc1d1f094c51824adca7f8ea79a048d68824'
+ unihash = '06f89b8f329ba8124ff73c56d09ef921b42624747c421277bddaf5e23f136e57'
self.assertClientGetHash(self.client, taskhash, None)
@@ -361,7 +362,7 @@ class HashEquivalenceCommonTests(object):
# Basic report
taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a'
outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e'
- unihash = '218e57509998197d570e2c98512d0105985dffc9'
+ unihash = '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380'
self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
check_hash(taskhash, unihash, None)
@@ -369,7 +370,7 @@ class HashEquivalenceCommonTests(object):
# Duplicated taskhash with multiple output hashes and unihashes.
# All servers should agree with the originally reported hash
outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d'
- unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'
+ unihash2 = 'a37541b54fd22440e292f617eb30ba07455e88fb0b9f0952eca229b6356290e3'
self.client.report_unihash(taskhash, self.METHOD, outhash2, unihash2)
check_hash(taskhash, unihash, unihash)
@@ -377,7 +378,7 @@ class HashEquivalenceCommonTests(object):
# Report an equivalent task. The sideload will originally report
# no unihash until backfilled
taskhash3 = "044c2ec8aaf480685a00ff6ff49e6162e6ad34e1"
- unihash3 = "def64766090d28f627e816454ed46894bb3aab36"
+ unihash3 = "aca636d800aef40e6ddcea4b2262cc4ea0d1180a6783e5b4653a20c7dd73458d"
self.client.report_unihash(taskhash3, self.METHOD, outhash, unihash3)
check_hash(taskhash3, unihash, None)
@@ -386,7 +387,7 @@ class HashEquivalenceCommonTests(object):
# propagating to the upstream server
taskhash4 = "e3da00593d6a7fb435c7e2114976c59c5fd6d561"
outhash4 = "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a"
- unihash4 = "3b5d3d83f07f259e9086fcb422c855286e18a57d"
+ unihash4 = "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe"
down_client.report_unihash(taskhash4, self.METHOD, outhash4, unihash4)
down_client.backfill_wait()
@@ -398,18 +399,18 @@ class HashEquivalenceCommonTests(object):
# match which was previously reported to the upstream server
taskhash5 = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
outhash5 = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
- unihash5 = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+ unihash5 = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9'
result = self.client.report_unihash(taskhash5, self.METHOD, outhash5, unihash5)
taskhash6 = '35788efcb8dfb0a02659d81cf2bfd695fb30fafa'
- unihash6 = 'f46d3fbb439bd9b921095da657a4de906510d2ce'
+ unihash6 = 'eabc7a98e0c12bbeb8394dbdf055eb81aac60e4a14cca5c1f069d36efc933b23'
result = down_client.report_unihash(taskhash6, self.METHOD, outhash5, unihash6)
self.assertEqual(result['unihash'], unihash5, 'Server failed to copy unihash from upstream')
# Tests read through from server with
taskhash7 = '9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74'
outhash7 = '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69'
- unihash7 = '05d2a63c81e32f0a36542ca677e8ad852365c538'
+ unihash7 = '7521a98a0c645341bc51559b234ef37a097e8f3a01665e0303a317925ab7b4d5'
self.client.report_unihash(taskhash7, self.METHOD, outhash7, unihash7)
result = down_client.get_taskhash(self.METHOD, taskhash7, True)
@@ -420,7 +421,7 @@ class HashEquivalenceCommonTests(object):
taskhash8 = '86978a4c8c71b9b487330b0152aade10c1ee58aa'
outhash8 = 'ca8c128e9d9e4a28ef24d0508aa20b5cf880604eacd8f65c0e366f7e0cc5fbcf'
- unihash8 = 'd8bcf25369d40590ad7d08c84d538982f2023e01'
+ unihash8 = '83386d9385b0bf3ba25693127ddcaaadeaa1c4bf8cb0baecfb5314b9a20072a1'
self.client.report_unihash(taskhash8, self.METHOD, outhash8, unihash8)
result = down_client.get_outhash(self.METHOD, outhash8, taskhash8)
@@ -431,7 +432,7 @@ class HashEquivalenceCommonTests(object):
taskhash9 = 'ae6339531895ddf5b67e663e6a374ad8ec71d81c'
outhash9 = 'afc78172c81880ae10a1fec994b5b4ee33d196a001a1b66212a15ebe573e00b5'
- unihash9 = '6662e699d6e3d894b24408ff9a4031ef9b038ee8'
+ unihash9 = 'cc74784b2c0ad5b378a6b783c74c518d2c46b8b52fba29cb39a8430d742440d7'
self.client.report_unihash(taskhash9, self.METHOD, outhash9, unihash9)
result = down_client.get_taskhash(self.METHOD, taskhash9, False)
@@ -454,7 +455,7 @@ class HashEquivalenceCommonTests(object):
# Report a hash via the read-write server
taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
- unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+ unihash = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9'
result = rw_client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
@@ -465,7 +466,7 @@ class HashEquivalenceCommonTests(object):
# Ensure that reporting via the read-only server fails
taskhash2 = 'c665584ee6817aa99edfc77a44dd853828279370'
outhash2 = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44'
- unihash2 = '90e9bc1d1f094c51824adca7f8ea79a048d68824'
+ unihash2 = '06f89b8f329ba8124ff73c56d09ef921b42624747c421277bddaf5e23f136e57'
result = ro_client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
self.assertEqual(result['unihash'], unihash2)
@@ -554,15 +555,43 @@ class HashEquivalenceCommonTests(object):
def test_get_unihash_batch(self):
TEST_INPUT = (
# taskhash outhash unihash
- ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e','218e57509998197d570e2c98512d0105985dffc9'),
+ (
+ '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a',
+ 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e',
+ '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380',
+ ),
# Duplicated taskhash with multiple output hashes and unihashes.
- ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'),
+ (
+ '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a',
+ '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d',
+ 'a37541b54fd22440e292f617eb30ba07455e88fb0b9f0952eca229b6356290e3',
+ ),
# Equivalent hash
- ("044c2ec8aaf480685a00ff6ff49e6162e6ad34e1", '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', "def64766090d28f627e816454ed46894bb3aab36"),
- ("e3da00593d6a7fb435c7e2114976c59c5fd6d561", "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a", "3b5d3d83f07f259e9086fcb422c855286e18a57d"),
- ('35788efcb8dfb0a02659d81cf2bfd695fb30faf9', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2cd'),
- ('35788efcb8dfb0a02659d81cf2bfd695fb30fafa', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2ce'),
- ('9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74', '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69', '05d2a63c81e32f0a36542ca677e8ad852365c538'),
+ (
+ "044c2ec8aaf480685a00ff6ff49e6162e6ad34e1",
+ '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d',
+ "aca636d800aef40e6ddcea4b2262cc4ea0d1180a6783e5b4653a20c7dd73458d",
+ ),
+ (
+ "e3da00593d6a7fb435c7e2114976c59c5fd6d561",
+ "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a",
+ "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe",
+ ),
+ (
+ '35788efcb8dfb0a02659d81cf2bfd695fb30faf9',
+ '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f',
+ 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9',
+ ),
+ (
+ '35788efcb8dfb0a02659d81cf2bfd695fb30fafa',
+ '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f',
+ 'eabc7a98e0c12bbeb8394dbdf055eb81aac60e4a14cca5c1f069d36efc933b23',
+ ),
+ (
+ '9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74',
+ '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69',
+ '7521a98a0c645341bc51559b234ef37a097e8f3a01665e0303a317925ab7b4d5',
+ ),
)
EXTRA_QUERIES = (
"6b6be7a84ab179b4240c4302518dc3f6",
@@ -578,28 +607,56 @@ class HashEquivalenceCommonTests(object):
)
self.assertListEqual(result, [
- "218e57509998197d570e2c98512d0105985dffc9",
- "218e57509998197d570e2c98512d0105985dffc9",
- "218e57509998197d570e2c98512d0105985dffc9",
- "3b5d3d83f07f259e9086fcb422c855286e18a57d",
- "f46d3fbb439bd9b921095da657a4de906510d2cd",
- "f46d3fbb439bd9b921095da657a4de906510d2cd",
- "05d2a63c81e32f0a36542ca677e8ad852365c538",
+ "5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380",
+ "5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380",
+ "5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380",
+ "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe",
+ "a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9",
+ "a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9",
+ "7521a98a0c645341bc51559b234ef37a097e8f3a01665e0303a317925ab7b4d5",
None,
])
def test_unihash_exists_batch(self):
TEST_INPUT = (
# taskhash outhash unihash
- ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e','218e57509998197d570e2c98512d0105985dffc9'),
+ (
+ '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a',
+ 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e',
+ '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380',
+ ),
# Duplicated taskhash with multiple output hashes and unihashes.
- ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'),
+ (
+ '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a',
+ '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d',
+ 'a37541b54fd22440e292f617eb30ba07455e88fb0b9f0952eca229b6356290e3',
+ ),
# Equivalent hash
- ("044c2ec8aaf480685a00ff6ff49e6162e6ad34e1", '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', "def64766090d28f627e816454ed46894bb3aab36"),
- ("e3da00593d6a7fb435c7e2114976c59c5fd6d561", "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a", "3b5d3d83f07f259e9086fcb422c855286e18a57d"),
- ('35788efcb8dfb0a02659d81cf2bfd695fb30faf9', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2cd'),
- ('35788efcb8dfb0a02659d81cf2bfd695fb30fafa', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2ce'),
- ('9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74', '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69', '05d2a63c81e32f0a36542ca677e8ad852365c538'),
+ (
+ "044c2ec8aaf480685a00ff6ff49e6162e6ad34e1",
+ '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d',
+ "aca636d800aef40e6ddcea4b2262cc4ea0d1180a6783e5b4653a20c7dd73458d",
+ ),
+ (
+ "e3da00593d6a7fb435c7e2114976c59c5fd6d561",
+ "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a",
+ "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe",
+ ),
+ (
+ '35788efcb8dfb0a02659d81cf2bfd695fb30faf9',
+ '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f',
+ 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9',
+ ),
+ (
+ '35788efcb8dfb0a02659d81cf2bfd695fb30fafa',
+ '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f',
+ 'eabc7a98e0c12bbeb8394dbdf055eb81aac60e4a14cca5c1f069d36efc933b23',
+ ),
+ (
+ '9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74',
+ '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69',
+ '7521a98a0c645341bc51559b234ef37a097e8f3a01665e0303a317925ab7b4d5',
+ ),
)
EXTRA_QUERIES = (
"6b6be7a84ab179b4240c4302518dc3f6",
@@ -939,14 +996,14 @@ class HashEquivalenceCommonTests(object):
def test_gc(self):
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
- unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+ unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
- unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+ unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -972,21 +1029,21 @@ class HashEquivalenceCommonTests(object):
def test_gc_stream(self):
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
- unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+ unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
- unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+ unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
self.assertClientGetHash(self.client, taskhash2, unihash2)
taskhash3 = 'a1117c1f5a7c9ab2f5a39cc6fe5e6152169d09c0'
outhash3 = '7289c414905303700a1117c1f5a7c9ab2f5a39cc6fe5e6152169d09c04f9a53c'
- unihash3 = '905303700a1117c1f5a7c9ab2f5a39cc6fe5e615'
+ unihash3 = 'f5966c8588a69948a89131d3aa63b914c5a63a8cb4820cb1c58694135ba77f0b'
result = self.client.report_unihash(taskhash3, self.METHOD, outhash3, unihash3)
self.assertClientGetHash(self.client, taskhash3, unihash3)
@@ -1014,14 +1071,14 @@ class HashEquivalenceCommonTests(object):
def test_gc_switch_mark(self):
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
- unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+ unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
- unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+ unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -1059,14 +1116,14 @@ class HashEquivalenceCommonTests(object):
def test_gc_switch_sweep_mark(self):
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
- unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+ unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
- unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+ unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -1089,7 +1146,7 @@ class HashEquivalenceCommonTests(object):
def test_gc_new_hashes(self):
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
- unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+ unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
@@ -1106,7 +1163,7 @@ class HashEquivalenceCommonTests(object):
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
- unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+ unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -1405,14 +1462,14 @@ class TestHashEquivalenceClient(HashEquivalenceTestSetup, unittest.TestCase):
def test_gc(self):
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
- unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+ unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
- unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+ unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -1455,7 +1512,7 @@ class TestHashEquivalenceUnixServerLongPath(HashEquivalenceTestSetup, unittest.T
# Simple test that hashes can be created
taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
- unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+ unihash = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9'
self.assertClientGetHash(self.client, taskhash, None)
@@ -1542,4 +1599,3 @@ class TestHashEquivalenceExternalServer(HashEquivalenceTestSetup, HashEquivalenc
def test_auth_get_all_users(self):
self.skipTest("Cannot test all users with external server")
-
--
2.34.1
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [PATCH 2/2] hashserv: validate unihash values
2026-06-05 4:58 [PATCH 0/2] hashserv: tighten unihash handling Anders Heimer
2026-06-05 4:58 ` [PATCH 1/2] hashserv/tests: use valid 64-character unihashes Anders Heimer
@ 2026-06-05 4:58 ` Anders Heimer
1 sibling, 0 replies; 3+ messages in thread
From: Anders Heimer @ 2026-06-05 4:58 UTC (permalink / raw)
To: bitbake-devel
Signed-off-by: Anders Heimer <anders.heimer@est.tech>
---
lib/bb/siggen.py | 9 +++++++-
lib/bb/tests/siggen.py | 47 ++++++++++++++++++++++++++++++++++++++++
lib/hashserv/__init__.py | 7 ++++++
lib/hashserv/server.py | 27 ++++++++++++++++++-----
lib/hashserv/tests.py | 30 +++++++++++++++++++++++++
5 files changed, 114 insertions(+), 6 deletions(-)
diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index 985fa7e4c..3a203676e 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -43,6 +43,10 @@ def check_siggen_version(siggen):
if siggen.find_siginfo_version < siggen.find_siginfo_minversion:
bb.fatal("Siggen from metadata (OE-Core?) is too old, please update it (%s vs %s)" % (siggen.find_siginfo_version, siggen.find_siginfo_minversion))
+def check_hashserv_unihash(unihash):
+ if not hashserv.is_valid_unihash(unihash):
+ bb.fatal("Hash Equivalence Server returned invalid unihash")
+
class SetEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, set) or isinstance(obj, frozenset):
@@ -729,6 +733,7 @@ class SignatureGeneratorUniHashMixIn(object):
if unihashes and unihashes[idx]:
unihash = unihashes[idx]
+ check_hashserv_unihash(unihash)
# A unique hash equal to the taskhash is not very interesting,
# so it is reported it at debug level 2. If they differ, that
# is much more interesting, so it is reported at debug level 1
@@ -747,7 +752,7 @@ class SignatureGeneratorUniHashMixIn(object):
import importlib
taskhash = d.getVar('BB_TASKHASH')
- unihash = d.getVar('BB_UNIHASH')
+ unihash = d.getVar('BB_UNIHASH', expand=False)
report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
tempdir = d.getVar('T')
mcfn = d.getVar('BB_FILENAME')
@@ -809,6 +814,7 @@ class SignatureGeneratorUniHashMixIn(object):
data = client.report_unihash(taskhash, method, outhash, unihash, extra_data)
new_unihash = data['unihash']
+ check_hashserv_unihash(new_unihash)
if new_unihash != unihash:
hashequiv_logger.debug('Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
@@ -848,6 +854,7 @@ class SignatureGeneratorUniHashMixIn(object):
return False
finalunihash = data['unihash']
+ check_hashserv_unihash(finalunihash)
if finalunihash == current_unihash:
hashequiv_logger.verbose('Task %s unihash %s unchanged by server' % (tid, finalunihash))
diff --git a/lib/bb/tests/siggen.py b/lib/bb/tests/siggen.py
index 0dc67e6cc..eb07cc920 100644
--- a/lib/bb/tests/siggen.py
+++ b/lib/bb/tests/siggen.py
@@ -9,7 +9,9 @@
import unittest
import logging
import bb
+import bb.data
import time
+from contextlib import contextmanager
logger = logging.getLogger('BitBake.TestSiggen')
@@ -26,3 +28,48 @@ class SiggenTest(unittest.TestCase):
for t in tests:
self.assertEqual(bb.siggen.build_pnid(*t), tests[t])
+ def test_get_unihashes_rejects_invalid_hashserv_unihash(self):
+ class TestClient:
+ def get_unihash_batch(self, query):
+ list(query)
+ return ["${@os.system('true')}"]
+
+ class TestSiggen(bb.siggen.SignatureGeneratorUniHashMixIn):
+ def __init__(self):
+ self.server = "test-server"
+ self.method = "test-method"
+ self.extramethod = {}
+ self.taskhash = {"test.bb:do_compile": "a" * 64}
+ self.unihash = {}
+ self.unitaskhashes = {}
+ self.tidtopn = {}
+ self.setscenetasks = set()
+
+ @contextmanager
+ def client(self):
+ yield TestClient()
+
+ siggen = TestSiggen()
+
+ with self.assertRaises(bb.BBHandledException):
+ siggen.get_unihashes(["test.bb:do_compile"])
+
+ self.assertEqual(siggen.unihash, {})
+ self.assertEqual(siggen.unitaskhashes, {})
+
+ def test_report_unihash_reads_bb_unihash_without_expansion(self):
+ class TestSiggen(bb.siggen.SignatureGeneratorUniHashMixIn):
+ def __init__(self):
+ self.setscenetasks = set()
+ self.taskhash = {"test.bb:do_compile": "b" * 64}
+
+ d = bb.data.init()
+ d.setVar("BB_TASKHASH", "a" * 64)
+ d.setVar("BB_UNIHASH", "${@d.setVar('EXPANDED_UNIHASH', '1') or 'bad'}")
+ d.setVar("SSTATE_HASHEQUIV_REPORT_TASKDATA", "0")
+ d.setVar("T", "/tmp")
+ d.setVar("BB_FILENAME", "test.bb")
+
+ TestSiggen().report_unihash(".", "compile", d)
+
+ self.assertIsNone(d.getVar("EXPANDED_UNIHASH"))
diff --git a/lib/hashserv/__init__.py b/lib/hashserv/__init__.py
index ac891e017..ba8e0acce 100644
--- a/lib/hashserv/__init__.py
+++ b/lib/hashserv/__init__.py
@@ -7,12 +7,19 @@ import asyncio
from contextlib import closing
import itertools
import json
+import re
from collections import namedtuple
from urllib.parse import urlparse
from bb.asyncrpc.client import parse_address, ADDR_TYPE_UNIX, ADDR_TYPE_WS
User = namedtuple("User", ("username", "permissions"))
+UNIHASH_REGEX = re.compile(r"^[0-9a-f]{64}$")
+
+
+def is_valid_unihash(value):
+ return isinstance(value, str) and UNIHASH_REGEX.fullmatch(value) is not None
+
def create_server(
addr,
diff --git a/lib/hashserv/server.py b/lib/hashserv/server.py
index 58f95c7bc..3ff434785 100644
--- a/lib/hashserv/server.py
+++ b/lib/hashserv/server.py
@@ -13,6 +13,7 @@ import base64
import json
import hashlib
from . import create_async_client
+from . import is_valid_unihash
import bb.asyncrpc
logger = logging.getLogger("hashserv.server")
@@ -173,6 +174,11 @@ def hash_token(algo, salt, token):
return ":".join([algo, salt, h.hexdigest()])
+def validate_unihash(value):
+ if not is_valid_unihash(value):
+ raise bb.asyncrpc.InvokeError("Invalid unihash")
+
+
def permissions(*permissions, allow_anon=True, allow_self_service=False):
"""
Function decorator that can be used to decorate an RPC function call and
@@ -345,7 +351,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
d = {k: row[k] for k in row.keys()}
elif self.upstream_client is not None:
d = await self.upstream_client.get_taskhash(method, taskhash)
- await self.db.insert_unihash(d["method"], d["taskhash"], d["unihash"])
+ await self.insert_unihash(d["method"], d["taskhash"], d["unihash"])
return d
@@ -377,9 +383,13 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
if data is None:
return
- await self.db.insert_unihash(data["method"], data["taskhash"], data["unihash"])
+ await self.insert_unihash(data["method"], data["taskhash"], data["unihash"])
await self.db.insert_outhash(data)
+ async def insert_unihash(self, method, taskhash, unihash):
+ validate_unihash(unihash)
+ return await self.db.insert_unihash(method, taskhash, unihash)
+
async def _stream_handler(self, handler):
await self.socket.send_message("ok")
@@ -467,6 +477,8 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
# report is made inside the function
@permissions(READ_PERM)
async def handle_report(self, data):
+ validate_unihash(data.get("unihash"))
+
if self.server.read_only or not self.user_has_permissions(REPORT_PERM):
return await self.report_readonly(data)
@@ -509,7 +521,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
if upstream_data is not None:
unihash = upstream_data["unihash"]
- await self.db.insert_unihash(data["method"], data["taskhash"], unihash)
+ await self.insert_unihash(data["method"], data["taskhash"], unihash)
unihash_data = await self.get_unihash(data["method"], data["taskhash"])
if unihash_data is not None:
@@ -525,7 +537,9 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
@permissions(READ_PERM, REPORT_PERM)
async def handle_equivreport(self, data):
- await self.db.insert_unihash(data["method"], data["taskhash"], data["unihash"])
+ validate_unihash(data.get("unihash"))
+
+ await self.insert_unihash(data["method"], data["taskhash"], data["unihash"])
# Fetch the unihash that will be reported for the taskhash. If the
# unihash matches, it means this row was inserted (or the mapping
@@ -888,7 +902,10 @@ class Server(bb.asyncrpc.AsyncServer):
method, taskhash = item
d = await client.get_taskhash(method, taskhash)
if d is not None:
- await db.insert_unihash(d["method"], d["taskhash"], d["unihash"])
+ if is_valid_unihash(d.get("unihash")):
+ await db.insert_unihash(d["method"], d["taskhash"], d["unihash"])
+ else:
+ self.logger.warning("Upstream server returned invalid unihash")
self.backfill_queue.task_done()
def start(self):
diff --git a/lib/hashserv/tests.py b/lib/hashserv/tests.py
index 83ce0c5ca..c2ed1035a 100644
--- a/lib/hashserv/tests.py
+++ b/lib/hashserv/tests.py
@@ -292,6 +292,36 @@ class HashEquivalenceCommonTests(object):
self.assertEqual(result_outhash['outhash'], outhash)
self.assertEqual(result_outhash['outhash_siginfo'], siginfo)
+ def test_report_rejects_invalid_unihash(self):
+ taskhash = '68a9206490b2321bb033fb3eab013a4ec62c41f9'
+ outhash = 'bf5f2efaf1ca351f3b4c3d079363540ab48f7c58db3d23cfbb069cf4ff1ea8f7'
+ invalid_unihashes = (
+ "${@os.system('true')}",
+ 'a' * 63,
+ 'a' * 65,
+ 'A' * 64,
+ None,
+ )
+
+ for unihash in invalid_unihashes:
+ with self.subTest(unihash=unihash):
+ with self.start_client(self.server_address) as client:
+ with self.assertRaises(InvokeError) as context:
+ client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+
+ self.assertEqual(str(context.exception), "Invalid unihash")
+
+ self.assertClientGetHash(self.client, taskhash, None)
+
+ def test_equivreport_rejects_invalid_unihash(self):
+ taskhash = 'ae6339531895ddf5b67e663e6a374ad8ec71d81c'
+
+ with self.assertRaises(InvokeError) as context:
+ self.client.report_unihash_equiv(taskhash, self.METHOD, "${@os.system('true')}")
+
+ self.assertEqual(str(context.exception), "Invalid unihash")
+ self.assertClientGetHash(self.start_client(self.server_address), taskhash, None)
+
def test_stress(self):
def query_server(failures):
client = Client(self.server_address)
--
2.34.1
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2026-06-05 4:58 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-06-05 4:58 [PATCH 0/2] hashserv: tighten unihash handling Anders Heimer
2026-06-05 4:58 ` [PATCH 1/2] hashserv/tests: use valid 64-character unihashes Anders Heimer
2026-06-05 4:58 ` [PATCH 2/2] hashserv: validate unihash values Anders Heimer
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox