[PATCH BlueZ v1 2/8] shared/util: Make strnlenutf8 reject ill-formed sequences

[Luiz Augusto von Dentz <luiz.dentz@gmail.com>]

From: Luiz Augusto von Dentz <luiz.von.dentz@intel.com>

strnlenutf8() only checks the shape of the lead byte and that the
following bytes are continuation bytes, so it accepts sequences that are
not well-formed UTF-8:

  C0 80        overlong encoding of U+0000
  C0 AF        overlong encoding of '/'
  ED A0 80     UTF-16 surrogate U+D800
  F5 80 80 80  past the U+10FFFF limit

strisutf8() and strtoutf8() are built on it, so a remote name containing
any of those is considered valid and passed on unchanged, for instance
to D-Bus, which does validate UTF-8 strictly and rejects them.

Validate the sequences as defined by table 3-7 of the Unicode Standard
instead, which constrains the range of the second byte for the E0, ED,
F0 and F4 lead bytes and rejects the C0, C1 and F5 to FF ones outright.

The decoding is split out into a helper that also reports the size of
the maximal subpart of an ill-formed sequence, so that callers can skip
over it, as recommended by section 3.9 of the Unicode Standard.

Assisted-by: Claude:claude-opus-5
---
 src/shared/util.c | 90 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 62 insertions(+), 28 deletions(-)

diff --git a/src/shared/util.c b/src/shared/util.c
index 62dd1369b70d..e946214edbb9 100644
--- a/src/shared/util.c
+++ b/src/shared/util.c
@@ -2211,44 +2211,78 @@ char *strstrip(char *str)
 	return str;
 }
 
-size_t strnlenutf8(const char *str, size_t len)
+/*
+ * Decode the UTF-8 sequence at str, as defined by table 3-7 of the Unicode
+ * Standard, and return its size, or 0 if it is ill-formed.
+ *
+ * sublen is set to the size of the maximal subpart of the sequence, that is
+ * the number of leading bytes that could still have formed a well-formed
+ * sequence, which is what the caller needs to skip over.
+ */
+static size_t utf8_seqlen(const unsigned char *str, size_t len, size_t *sublen)
+{
+	unsigned char lo = 0x80, hi = 0xbf;
+	size_t size, i;
 
+	if (str[0] <= 0x7f) {
+		*sublen = 1;
+		return 1;
+	}
+
+	if (str[0] >= 0xc2 && str[0] <= 0xdf) {
+		size = 2;
+	} else if (str[0] >= 0xe0 && str[0] <= 0xef) {
+		size = 3;
+		/* Reject the overlong encodings and the UTF-16 surrogates */
+		if (str[0] == 0xe0)
+			lo = 0xa0;
+		else if (str[0] == 0xed)
+			hi = 0x9f;
+	} else if (str[0] >= 0xf0 && str[0] <= 0xf4) {
+		size = 4;
+		/* Reject the overlong encodings and anything past U+10FFFF */
+		if (str[0] == 0xf0)
+			lo = 0x90;
+		else if (str[0] == 0xf4)
+			hi = 0x8f;
+	} else {
+		/* C0 and C1 are overlong, F5 to FF are out of range, and a
+		 * continuation byte cannot start a sequence.
+		 */
+		*sublen = 1;
+		return 0;
+	}
+
+	for (i = 1; i < size; i++) {
+		if (i >= len || str[i] < lo || str[i] > hi) {
+			*sublen = i;
+			return 0;
+		}
+
+		/* Only the second byte has a restricted range */
+		lo = 0x80;
+		hi = 0xbf;
+	}
+
+	*sublen = size;
+	return size;
+}
+
+size_t strnlenutf8(const char *str, size_t len)
 {
 	size_t i = 0;
 
 	while (i < len) {
-		unsigned char c = str[i];
-		size_t size = 0;
+		size_t sublen;
 
-		/* Check the first byte to determine the number of bytes in the
-		 * UTF-8 character.
-		 */
-		if ((c & 0x80) == 0x00)
-			size = 1;
-		else if ((c & 0xE0) == 0xC0)
-			size = 2;
-		else if ((c & 0xF0) == 0xE0)
-			size = 3;
-		else if ((c & 0xF8) == 0xF0)
-			size = 4;
-		else
-			/* Invalid UTF-8 sequence */
-			goto done;
-
-		/* Check the following bytes to ensure they have the correct
-		 * format.
-		 */
-		for (size_t j = 1; j < size; ++j) {
-			if (i + j >= len || (str[i + j] & 0xC0) != 0x80)
-				/* Invalid UTF-8 sequence */
-				goto done;
-		}
+		if (!utf8_seqlen((const unsigned char *) str + i, len - i,
+								&sublen))
+			break;
 
 		/* Move to the next character */
-		i += size;
+		i += sublen;
 	}
 
-done:
 	return i;
 }
 
-- 
2.54.0