* [PATCH 1/2] hashserv/tests: use valid 64-character unihashes
2026-06-05 4:58 [PATCH 0/2] hashserv: tighten unihash handling Anders Heimer
@ 2026-06-05 4:58 ` Anders Heimer
2026-06-05 4:58 ` [PATCH 2/2] hashserv: validate unihash values Anders Heimer
1 sibling, 0 replies; 3+ messages in thread
From: Anders Heimer @ 2026-06-05 4:58 UTC (permalink / raw)
To: bitbake-devel
Signed-off-by: Anders Heimer <anders.heimer@est.tech>
---
lib/hashserv/tests.py | 164 ++++++++++++++++++++++++++++--------------
1 file changed, 110 insertions(+), 54 deletions(-)
diff --git a/lib/hashserv/tests.py b/lib/hashserv/tests.py
index 124d8aa00..83ce0c5ca 100644
--- a/lib/hashserv/tests.py
+++ b/lib/hashserv/tests.py
@@ -28,6 +28,7 @@ from pathlib import Path
THIS_DIR = Path(__file__).parent
BIN_DIR = THIS_DIR.parent.parent / "bin"
+
def server_prefunc(server, idx):
logging.basicConfig(level=logging.DEBUG, filename='bbhashserv-%d.log' % idx, filemode='w',
format='%(levelname)s %(filename)s:%(lineno)d %(message)s')
@@ -124,7 +125,7 @@ class HashEquivalenceTestSetup(object):
# Simple test that hashes can be created
taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
- unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+ unihash = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9'
self.assertClientGetHash(client, taskhash, None)
@@ -178,7 +179,7 @@ class HashEquivalenceCommonTests(object):
# assigned the same unihash
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
- unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+ unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
@@ -186,7 +187,7 @@ class HashEquivalenceCommonTests(object):
# Report a different task with the same outhash. The returned unihash
# should match the first task
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
- unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+ unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
result = self.client.report_unihash(taskhash2, self.METHOD, outhash, unihash2)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
@@ -196,19 +197,19 @@ class HashEquivalenceCommonTests(object):
# taskhash
taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a'
outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e'
- unihash = '218e57509998197d570e2c98512d0105985dffc9'
+ unihash = '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380'
self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertClientGetHash(self.client, taskhash, unihash)
outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d'
- unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'
+ unihash2 = 'a37541b54fd22440e292f617eb30ba07455e88fb0b9f0952eca229b6356290e3'
self.client.report_unihash(taskhash, self.METHOD, outhash2, unihash2)
self.assertClientGetHash(self.client, taskhash, unihash)
outhash3 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
- unihash3 = '9217a7d6398518e5dc002ed58f2cbbbc78696603'
+ unihash3 = '6842f1f2daccd96ddef15c9154d4e41ac8a2300d781ac9a9db7f8afeb8a96808'
self.client.report_unihash(taskhash, self.METHOD, outhash3, unihash3)
self.assertClientGetHash(self.client, taskhash, unihash)
@@ -268,7 +269,7 @@ class HashEquivalenceCommonTests(object):
# Simple test that hashes can be created
taskhash = 'c665584ee6817aa99edfc77a44dd853828279370'
outhash = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44'
- unihash = '90e9bc1d1f094c51824adca7f8ea79a048d68824'
+ unihash = '06f89b8f329ba8124ff73c56d09ef921b42624747c421277bddaf5e23f136e57'
self.assertClientGetHash(self.client, taskhash, None)
@@ -361,7 +362,7 @@ class HashEquivalenceCommonTests(object):
# Basic report
taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a'
outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e'
- unihash = '218e57509998197d570e2c98512d0105985dffc9'
+ unihash = '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380'
self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
check_hash(taskhash, unihash, None)
@@ -369,7 +370,7 @@ class HashEquivalenceCommonTests(object):
# Duplicated taskhash with multiple output hashes and unihashes.
# All servers should agree with the originally reported hash
outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d'
- unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'
+ unihash2 = 'a37541b54fd22440e292f617eb30ba07455e88fb0b9f0952eca229b6356290e3'
self.client.report_unihash(taskhash, self.METHOD, outhash2, unihash2)
check_hash(taskhash, unihash, unihash)
@@ -377,7 +378,7 @@ class HashEquivalenceCommonTests(object):
# Report an equivalent task. The sideload will originally report
# no unihash until backfilled
taskhash3 = "044c2ec8aaf480685a00ff6ff49e6162e6ad34e1"
- unihash3 = "def64766090d28f627e816454ed46894bb3aab36"
+ unihash3 = "aca636d800aef40e6ddcea4b2262cc4ea0d1180a6783e5b4653a20c7dd73458d"
self.client.report_unihash(taskhash3, self.METHOD, outhash, unihash3)
check_hash(taskhash3, unihash, None)
@@ -386,7 +387,7 @@ class HashEquivalenceCommonTests(object):
# propagating to the upstream server
taskhash4 = "e3da00593d6a7fb435c7e2114976c59c5fd6d561"
outhash4 = "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a"
- unihash4 = "3b5d3d83f07f259e9086fcb422c855286e18a57d"
+ unihash4 = "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe"
down_client.report_unihash(taskhash4, self.METHOD, outhash4, unihash4)
down_client.backfill_wait()
@@ -398,18 +399,18 @@ class HashEquivalenceCommonTests(object):
# match which was previously reported to the upstream server
taskhash5 = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
outhash5 = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
- unihash5 = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+ unihash5 = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9'
result = self.client.report_unihash(taskhash5, self.METHOD, outhash5, unihash5)
taskhash6 = '35788efcb8dfb0a02659d81cf2bfd695fb30fafa'
- unihash6 = 'f46d3fbb439bd9b921095da657a4de906510d2ce'
+ unihash6 = 'eabc7a98e0c12bbeb8394dbdf055eb81aac60e4a14cca5c1f069d36efc933b23'
result = down_client.report_unihash(taskhash6, self.METHOD, outhash5, unihash6)
self.assertEqual(result['unihash'], unihash5, 'Server failed to copy unihash from upstream')
# Tests read through from server with
taskhash7 = '9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74'
outhash7 = '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69'
- unihash7 = '05d2a63c81e32f0a36542ca677e8ad852365c538'
+ unihash7 = '7521a98a0c645341bc51559b234ef37a097e8f3a01665e0303a317925ab7b4d5'
self.client.report_unihash(taskhash7, self.METHOD, outhash7, unihash7)
result = down_client.get_taskhash(self.METHOD, taskhash7, True)
@@ -420,7 +421,7 @@ class HashEquivalenceCommonTests(object):
taskhash8 = '86978a4c8c71b9b487330b0152aade10c1ee58aa'
outhash8 = 'ca8c128e9d9e4a28ef24d0508aa20b5cf880604eacd8f65c0e366f7e0cc5fbcf'
- unihash8 = 'd8bcf25369d40590ad7d08c84d538982f2023e01'
+ unihash8 = '83386d9385b0bf3ba25693127ddcaaadeaa1c4bf8cb0baecfb5314b9a20072a1'
self.client.report_unihash(taskhash8, self.METHOD, outhash8, unihash8)
result = down_client.get_outhash(self.METHOD, outhash8, taskhash8)
@@ -431,7 +432,7 @@ class HashEquivalenceCommonTests(object):
taskhash9 = 'ae6339531895ddf5b67e663e6a374ad8ec71d81c'
outhash9 = 'afc78172c81880ae10a1fec994b5b4ee33d196a001a1b66212a15ebe573e00b5'
- unihash9 = '6662e699d6e3d894b24408ff9a4031ef9b038ee8'
+ unihash9 = 'cc74784b2c0ad5b378a6b783c74c518d2c46b8b52fba29cb39a8430d742440d7'
self.client.report_unihash(taskhash9, self.METHOD, outhash9, unihash9)
result = down_client.get_taskhash(self.METHOD, taskhash9, False)
@@ -454,7 +455,7 @@ class HashEquivalenceCommonTests(object):
# Report a hash via the read-write server
taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
- unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+ unihash = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9'
result = rw_client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
@@ -465,7 +466,7 @@ class HashEquivalenceCommonTests(object):
# Ensure that reporting via the read-only server fails
taskhash2 = 'c665584ee6817aa99edfc77a44dd853828279370'
outhash2 = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44'
- unihash2 = '90e9bc1d1f094c51824adca7f8ea79a048d68824'
+ unihash2 = '06f89b8f329ba8124ff73c56d09ef921b42624747c421277bddaf5e23f136e57'
result = ro_client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
self.assertEqual(result['unihash'], unihash2)
@@ -554,15 +555,43 @@ class HashEquivalenceCommonTests(object):
def test_get_unihash_batch(self):
TEST_INPUT = (
# taskhash outhash unihash
- ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e','218e57509998197d570e2c98512d0105985dffc9'),
+ (
+ '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a',
+ 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e',
+ '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380',
+ ),
# Duplicated taskhash with multiple output hashes and unihashes.
- ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'),
+ (
+ '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a',
+ '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d',
+ 'a37541b54fd22440e292f617eb30ba07455e88fb0b9f0952eca229b6356290e3',
+ ),
# Equivalent hash
- ("044c2ec8aaf480685a00ff6ff49e6162e6ad34e1", '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', "def64766090d28f627e816454ed46894bb3aab36"),
- ("e3da00593d6a7fb435c7e2114976c59c5fd6d561", "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a", "3b5d3d83f07f259e9086fcb422c855286e18a57d"),
- ('35788efcb8dfb0a02659d81cf2bfd695fb30faf9', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2cd'),
- ('35788efcb8dfb0a02659d81cf2bfd695fb30fafa', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2ce'),
- ('9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74', '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69', '05d2a63c81e32f0a36542ca677e8ad852365c538'),
+ (
+ "044c2ec8aaf480685a00ff6ff49e6162e6ad34e1",
+ '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d',
+ "aca636d800aef40e6ddcea4b2262cc4ea0d1180a6783e5b4653a20c7dd73458d",
+ ),
+ (
+ "e3da00593d6a7fb435c7e2114976c59c5fd6d561",
+ "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a",
+ "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe",
+ ),
+ (
+ '35788efcb8dfb0a02659d81cf2bfd695fb30faf9',
+ '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f',
+ 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9',
+ ),
+ (
+ '35788efcb8dfb0a02659d81cf2bfd695fb30fafa',
+ '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f',
+ 'eabc7a98e0c12bbeb8394dbdf055eb81aac60e4a14cca5c1f069d36efc933b23',
+ ),
+ (
+ '9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74',
+ '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69',
+ '7521a98a0c645341bc51559b234ef37a097e8f3a01665e0303a317925ab7b4d5',
+ ),
)
EXTRA_QUERIES = (
"6b6be7a84ab179b4240c4302518dc3f6",
@@ -578,28 +607,56 @@ class HashEquivalenceCommonTests(object):
)
self.assertListEqual(result, [
- "218e57509998197d570e2c98512d0105985dffc9",
- "218e57509998197d570e2c98512d0105985dffc9",
- "218e57509998197d570e2c98512d0105985dffc9",
- "3b5d3d83f07f259e9086fcb422c855286e18a57d",
- "f46d3fbb439bd9b921095da657a4de906510d2cd",
- "f46d3fbb439bd9b921095da657a4de906510d2cd",
- "05d2a63c81e32f0a36542ca677e8ad852365c538",
+ "5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380",
+ "5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380",
+ "5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380",
+ "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe",
+ "a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9",
+ "a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9",
+ "7521a98a0c645341bc51559b234ef37a097e8f3a01665e0303a317925ab7b4d5",
None,
])
def test_unihash_exists_batch(self):
TEST_INPUT = (
# taskhash outhash unihash
- ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e','218e57509998197d570e2c98512d0105985dffc9'),
+ (
+ '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a',
+ 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e',
+ '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380',
+ ),
# Duplicated taskhash with multiple output hashes and unihashes.
- ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'),
+ (
+ '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a',
+ '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d',
+ 'a37541b54fd22440e292f617eb30ba07455e88fb0b9f0952eca229b6356290e3',
+ ),
# Equivalent hash
- ("044c2ec8aaf480685a00ff6ff49e6162e6ad34e1", '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', "def64766090d28f627e816454ed46894bb3aab36"),
- ("e3da00593d6a7fb435c7e2114976c59c5fd6d561", "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a", "3b5d3d83f07f259e9086fcb422c855286e18a57d"),
- ('35788efcb8dfb0a02659d81cf2bfd695fb30faf9', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2cd'),
- ('35788efcb8dfb0a02659d81cf2bfd695fb30fafa', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2ce'),
- ('9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74', '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69', '05d2a63c81e32f0a36542ca677e8ad852365c538'),
+ (
+ "044c2ec8aaf480685a00ff6ff49e6162e6ad34e1",
+ '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d',
+ "aca636d800aef40e6ddcea4b2262cc4ea0d1180a6783e5b4653a20c7dd73458d",
+ ),
+ (
+ "e3da00593d6a7fb435c7e2114976c59c5fd6d561",
+ "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a",
+ "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe",
+ ),
+ (
+ '35788efcb8dfb0a02659d81cf2bfd695fb30faf9',
+ '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f',
+ 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9',
+ ),
+ (
+ '35788efcb8dfb0a02659d81cf2bfd695fb30fafa',
+ '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f',
+ 'eabc7a98e0c12bbeb8394dbdf055eb81aac60e4a14cca5c1f069d36efc933b23',
+ ),
+ (
+ '9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74',
+ '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69',
+ '7521a98a0c645341bc51559b234ef37a097e8f3a01665e0303a317925ab7b4d5',
+ ),
)
EXTRA_QUERIES = (
"6b6be7a84ab179b4240c4302518dc3f6",
@@ -939,14 +996,14 @@ class HashEquivalenceCommonTests(object):
def test_gc(self):
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
- unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+ unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
- unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+ unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -972,21 +1029,21 @@ class HashEquivalenceCommonTests(object):
def test_gc_stream(self):
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
- unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+ unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
- unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+ unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
self.assertClientGetHash(self.client, taskhash2, unihash2)
taskhash3 = 'a1117c1f5a7c9ab2f5a39cc6fe5e6152169d09c0'
outhash3 = '7289c414905303700a1117c1f5a7c9ab2f5a39cc6fe5e6152169d09c04f9a53c'
- unihash3 = '905303700a1117c1f5a7c9ab2f5a39cc6fe5e615'
+ unihash3 = 'f5966c8588a69948a89131d3aa63b914c5a63a8cb4820cb1c58694135ba77f0b'
result = self.client.report_unihash(taskhash3, self.METHOD, outhash3, unihash3)
self.assertClientGetHash(self.client, taskhash3, unihash3)
@@ -1014,14 +1071,14 @@ class HashEquivalenceCommonTests(object):
def test_gc_switch_mark(self):
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
- unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+ unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
- unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+ unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -1059,14 +1116,14 @@ class HashEquivalenceCommonTests(object):
def test_gc_switch_sweep_mark(self):
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
- unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+ unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
- unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+ unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -1089,7 +1146,7 @@ class HashEquivalenceCommonTests(object):
def test_gc_new_hashes(self):
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
- unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+ unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
@@ -1106,7 +1163,7 @@ class HashEquivalenceCommonTests(object):
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
- unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+ unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -1405,14 +1462,14 @@ class TestHashEquivalenceClient(HashEquivalenceTestSetup, unittest.TestCase):
def test_gc(self):
taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
- unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+ unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
- unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+ unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -1455,7 +1512,7 @@ class TestHashEquivalenceUnixServerLongPath(HashEquivalenceTestSetup, unittest.T
# Simple test that hashes can be created
taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
- unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+ unihash = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9'
self.assertClientGetHash(self.client, taskhash, None)
@@ -1542,4 +1599,3 @@ class TestHashEquivalenceExternalServer(HashEquivalenceTestSetup, HashEquivalenc
def test_auth_get_all_users(self):
self.skipTest("Cannot test all users with external server")
-
--
2.34.1
^ permalink raw reply related [flat|nested] 3+ messages in thread* [PATCH 2/2] hashserv: validate unihash values
2026-06-05 4:58 [PATCH 0/2] hashserv: tighten unihash handling Anders Heimer
2026-06-05 4:58 ` [PATCH 1/2] hashserv/tests: use valid 64-character unihashes Anders Heimer
@ 2026-06-05 4:58 ` Anders Heimer
1 sibling, 0 replies; 3+ messages in thread
From: Anders Heimer @ 2026-06-05 4:58 UTC (permalink / raw)
To: bitbake-devel
Signed-off-by: Anders Heimer <anders.heimer@est.tech>
---
lib/bb/siggen.py | 9 +++++++-
lib/bb/tests/siggen.py | 47 ++++++++++++++++++++++++++++++++++++++++
lib/hashserv/__init__.py | 7 ++++++
lib/hashserv/server.py | 27 ++++++++++++++++++-----
lib/hashserv/tests.py | 30 +++++++++++++++++++++++++
5 files changed, 114 insertions(+), 6 deletions(-)
diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index 985fa7e4c..3a203676e 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -43,6 +43,10 @@ def check_siggen_version(siggen):
if siggen.find_siginfo_version < siggen.find_siginfo_minversion:
bb.fatal("Siggen from metadata (OE-Core?) is too old, please update it (%s vs %s)" % (siggen.find_siginfo_version, siggen.find_siginfo_minversion))
+def check_hashserv_unihash(unihash):
+ if not hashserv.is_valid_unihash(unihash):
+ bb.fatal("Hash Equivalence Server returned invalid unihash")
+
class SetEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, set) or isinstance(obj, frozenset):
@@ -729,6 +733,7 @@ class SignatureGeneratorUniHashMixIn(object):
if unihashes and unihashes[idx]:
unihash = unihashes[idx]
+ check_hashserv_unihash(unihash)
# A unique hash equal to the taskhash is not very interesting,
# so it is reported it at debug level 2. If they differ, that
# is much more interesting, so it is reported at debug level 1
@@ -747,7 +752,7 @@ class SignatureGeneratorUniHashMixIn(object):
import importlib
taskhash = d.getVar('BB_TASKHASH')
- unihash = d.getVar('BB_UNIHASH')
+ unihash = d.getVar('BB_UNIHASH', expand=False)
report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
tempdir = d.getVar('T')
mcfn = d.getVar('BB_FILENAME')
@@ -809,6 +814,7 @@ class SignatureGeneratorUniHashMixIn(object):
data = client.report_unihash(taskhash, method, outhash, unihash, extra_data)
new_unihash = data['unihash']
+ check_hashserv_unihash(new_unihash)
if new_unihash != unihash:
hashequiv_logger.debug('Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
@@ -848,6 +854,7 @@ class SignatureGeneratorUniHashMixIn(object):
return False
finalunihash = data['unihash']
+ check_hashserv_unihash(finalunihash)
if finalunihash == current_unihash:
hashequiv_logger.verbose('Task %s unihash %s unchanged by server' % (tid, finalunihash))
diff --git a/lib/bb/tests/siggen.py b/lib/bb/tests/siggen.py
index 0dc67e6cc..eb07cc920 100644
--- a/lib/bb/tests/siggen.py
+++ b/lib/bb/tests/siggen.py
@@ -9,7 +9,9 @@
import unittest
import logging
import bb
+import bb.data
import time
+from contextlib import contextmanager
logger = logging.getLogger('BitBake.TestSiggen')
@@ -26,3 +28,48 @@ class SiggenTest(unittest.TestCase):
for t in tests:
self.assertEqual(bb.siggen.build_pnid(*t), tests[t])
+ def test_get_unihashes_rejects_invalid_hashserv_unihash(self):
+ class TestClient:
+ def get_unihash_batch(self, query):
+ list(query)
+ return ["${@os.system('true')}"]
+
+ class TestSiggen(bb.siggen.SignatureGeneratorUniHashMixIn):
+ def __init__(self):
+ self.server = "test-server"
+ self.method = "test-method"
+ self.extramethod = {}
+ self.taskhash = {"test.bb:do_compile": "a" * 64}
+ self.unihash = {}
+ self.unitaskhashes = {}
+ self.tidtopn = {}
+ self.setscenetasks = set()
+
+ @contextmanager
+ def client(self):
+ yield TestClient()
+
+ siggen = TestSiggen()
+
+ with self.assertRaises(bb.BBHandledException):
+ siggen.get_unihashes(["test.bb:do_compile"])
+
+ self.assertEqual(siggen.unihash, {})
+ self.assertEqual(siggen.unitaskhashes, {})
+
+ def test_report_unihash_reads_bb_unihash_without_expansion(self):
+ class TestSiggen(bb.siggen.SignatureGeneratorUniHashMixIn):
+ def __init__(self):
+ self.setscenetasks = set()
+ self.taskhash = {"test.bb:do_compile": "b" * 64}
+
+ d = bb.data.init()
+ d.setVar("BB_TASKHASH", "a" * 64)
+ d.setVar("BB_UNIHASH", "${@d.setVar('EXPANDED_UNIHASH', '1') or 'bad'}")
+ d.setVar("SSTATE_HASHEQUIV_REPORT_TASKDATA", "0")
+ d.setVar("T", "/tmp")
+ d.setVar("BB_FILENAME", "test.bb")
+
+ TestSiggen().report_unihash(".", "compile", d)
+
+ self.assertIsNone(d.getVar("EXPANDED_UNIHASH"))
diff --git a/lib/hashserv/__init__.py b/lib/hashserv/__init__.py
index ac891e017..ba8e0acce 100644
--- a/lib/hashserv/__init__.py
+++ b/lib/hashserv/__init__.py
@@ -7,12 +7,19 @@ import asyncio
from contextlib import closing
import itertools
import json
+import re
from collections import namedtuple
from urllib.parse import urlparse
from bb.asyncrpc.client import parse_address, ADDR_TYPE_UNIX, ADDR_TYPE_WS
User = namedtuple("User", ("username", "permissions"))
+UNIHASH_REGEX = re.compile(r"^[0-9a-f]{64}$")
+
+
+def is_valid_unihash(value):
+ return isinstance(value, str) and UNIHASH_REGEX.fullmatch(value) is not None
+
def create_server(
addr,
diff --git a/lib/hashserv/server.py b/lib/hashserv/server.py
index 58f95c7bc..3ff434785 100644
--- a/lib/hashserv/server.py
+++ b/lib/hashserv/server.py
@@ -13,6 +13,7 @@ import base64
import json
import hashlib
from . import create_async_client
+from . import is_valid_unihash
import bb.asyncrpc
logger = logging.getLogger("hashserv.server")
@@ -173,6 +174,11 @@ def hash_token(algo, salt, token):
return ":".join([algo, salt, h.hexdigest()])
+def validate_unihash(value):
+ if not is_valid_unihash(value):
+ raise bb.asyncrpc.InvokeError("Invalid unihash")
+
+
def permissions(*permissions, allow_anon=True, allow_self_service=False):
"""
Function decorator that can be used to decorate an RPC function call and
@@ -345,7 +351,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
d = {k: row[k] for k in row.keys()}
elif self.upstream_client is not None:
d = await self.upstream_client.get_taskhash(method, taskhash)
- await self.db.insert_unihash(d["method"], d["taskhash"], d["unihash"])
+ await self.insert_unihash(d["method"], d["taskhash"], d["unihash"])
return d
@@ -377,9 +383,13 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
if data is None:
return
- await self.db.insert_unihash(data["method"], data["taskhash"], data["unihash"])
+ await self.insert_unihash(data["method"], data["taskhash"], data["unihash"])
await self.db.insert_outhash(data)
+ async def insert_unihash(self, method, taskhash, unihash):
+ validate_unihash(unihash)
+ return await self.db.insert_unihash(method, taskhash, unihash)
+
async def _stream_handler(self, handler):
await self.socket.send_message("ok")
@@ -467,6 +477,8 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
# report is made inside the function
@permissions(READ_PERM)
async def handle_report(self, data):
+ validate_unihash(data.get("unihash"))
+
if self.server.read_only or not self.user_has_permissions(REPORT_PERM):
return await self.report_readonly(data)
@@ -509,7 +521,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
if upstream_data is not None:
unihash = upstream_data["unihash"]
- await self.db.insert_unihash(data["method"], data["taskhash"], unihash)
+ await self.insert_unihash(data["method"], data["taskhash"], unihash)
unihash_data = await self.get_unihash(data["method"], data["taskhash"])
if unihash_data is not None:
@@ -525,7 +537,9 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
@permissions(READ_PERM, REPORT_PERM)
async def handle_equivreport(self, data):
- await self.db.insert_unihash(data["method"], data["taskhash"], data["unihash"])
+ validate_unihash(data.get("unihash"))
+
+ await self.insert_unihash(data["method"], data["taskhash"], data["unihash"])
# Fetch the unihash that will be reported for the taskhash. If the
# unihash matches, it means this row was inserted (or the mapping
@@ -888,7 +902,10 @@ class Server(bb.asyncrpc.AsyncServer):
method, taskhash = item
d = await client.get_taskhash(method, taskhash)
if d is not None:
- await db.insert_unihash(d["method"], d["taskhash"], d["unihash"])
+ if is_valid_unihash(d.get("unihash")):
+ await db.insert_unihash(d["method"], d["taskhash"], d["unihash"])
+ else:
+ self.logger.warning("Upstream server returned invalid unihash")
self.backfill_queue.task_done()
def start(self):
diff --git a/lib/hashserv/tests.py b/lib/hashserv/tests.py
index 83ce0c5ca..c2ed1035a 100644
--- a/lib/hashserv/tests.py
+++ b/lib/hashserv/tests.py
@@ -292,6 +292,36 @@ class HashEquivalenceCommonTests(object):
self.assertEqual(result_outhash['outhash'], outhash)
self.assertEqual(result_outhash['outhash_siginfo'], siginfo)
+ def test_report_rejects_invalid_unihash(self):
+ taskhash = '68a9206490b2321bb033fb3eab013a4ec62c41f9'
+ outhash = 'bf5f2efaf1ca351f3b4c3d079363540ab48f7c58db3d23cfbb069cf4ff1ea8f7'
+ invalid_unihashes = (
+ "${@os.system('true')}",
+ 'a' * 63,
+ 'a' * 65,
+ 'A' * 64,
+ None,
+ )
+
+ for unihash in invalid_unihashes:
+ with self.subTest(unihash=unihash):
+ with self.start_client(self.server_address) as client:
+ with self.assertRaises(InvokeError) as context:
+ client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+
+ self.assertEqual(str(context.exception), "Invalid unihash")
+
+ self.assertClientGetHash(self.client, taskhash, None)
+
+ def test_equivreport_rejects_invalid_unihash(self):
+ taskhash = 'ae6339531895ddf5b67e663e6a374ad8ec71d81c'
+
+ with self.assertRaises(InvokeError) as context:
+ self.client.report_unihash_equiv(taskhash, self.METHOD, "${@os.system('true')}")
+
+ self.assertEqual(str(context.exception), "Invalid unihash")
+ self.assertClientGetHash(self.start_client(self.server_address), taskhash, None)
+
def test_stress(self):
def query_server(failures):
client = Client(self.server_address)
--
2.34.1
^ permalink raw reply related [flat|nested] 3+ messages in thread