[PATCH 0/6] Add alphabet support to SMS

[PATCH 0/6] Add alphabet support to SMS

[Aki Niemi]

[-- Attachment #1: Type: text/plain, Size: 570 bytes --]

Hi,

This is a set resurrected from last year, adding support for different
GSM alphabets or dialects when sending SMSs.

The first two patches do some refactoring that allows buidling also
custom GSM alphabets such as a combination of single shift and locking
shift tables of different GSM alphabet dialects, or even completely
custom conversion tables such as one that transliterates a range of
unicode to the default GSM alphabet.

The current algorithm for picking an optimal encoding, however, only
uses tables from a single dialect.

Cheers,
Aki

[PATCH 1/6] util: Refactor conversion table handling

[Aki Niemi]

[-- Attachment #1: Type: text/plain, Size: 13407 bytes --]

---
 src/util.c |  205 +++++++++++++++++++++++++++++++++++++-----------------------
 src/util.h |    1 -
 2 files changed, 126 insertions(+), 80 deletions(-)

diff --git a/src/util.c b/src/util.c
index f63ce67..547f38c 100644
--- a/src/util.c
+++ b/src/util.c
@@ -3,6 +3,7 @@
  *  oFono - Open Source Telephony
  *
  *  Copyright (C) 2008-2010  Intel Corporation. All rights reserved.
+ *  Copyright (C) 2009-2010  Nokia Corporation and/or its subsidiary(-ies).
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License version 2 as
@@ -62,8 +63,7 @@
 #define GUND			0xFFFF
 
 #define UTF8_LENGTH(c) \
-	((c) < 0x80 ? 1 : \
-	 ((c) < 0x800 ? 2 : 3))
+	((c) < 0x80 ? 1 : ((c) < 0x800 ? 2 : 3))
 
 #define TABLE_SIZE(t) \
 	(sizeof((t)) / sizeof(struct codepoint))
@@ -73,13 +73,21 @@ struct codepoint {
 	unsigned short to;
 };
 
-struct alphabet_conversion_table {
-	const unsigned short *togsm_locking_shift;
-	const struct codepoint *togsm_single_shift;
-	unsigned int togsm_single_shift_len;
-	const struct codepoint *tounicode_locking_shift;
-	const struct codepoint *tounicode_single_shift;
-	unsigned int tounicode_single_shift_len;
+struct conversion_table {
+	/* To unicode locking shift table */
+	const struct codepoint *locking_u;
+	unsigned int locking_len_u;
+
+	/* To unicode single shift table */
+	const struct codepoint *single_u;
+	unsigned int single_len_u;
+
+	/* To GSM locking shift table, fixed size */
+	const unsigned short *locking_g;
+
+	/* To GSM single shift table */
+	const struct codepoint *single_g;
+	unsigned int single_len_g;
 };
 
 /* GSM to Unicode extension table, for GSM sequences starting with 0x1B */
@@ -300,6 +308,7 @@ static const unsigned short def_gsm[] = {
 	0x0078, 0x0079, 0x007A, 0x00E4, 0x00F6, 0x00F1, 0x00FC, 0x00E0
 };
 
+/* Used for conversion of Unicode to GSM */
 static const struct codepoint def_unicode[] = {
 	{ 0x000A, 0x0A }, { 0x000D, 0x0D }, { 0x0020, 0x20 }, { 0x0021, 0x21 },
 	{ 0x0022, 0x22 }, { 0x0023, 0x23 }, { 0x0024, 0x02 }, { 0x0025, 0x25 },
@@ -445,21 +454,6 @@ static const struct codepoint por_unicode[] = {
 	{ 0x00FC, 0x7E }, { 0x0394, 0x10 }, { 0x20AC, 0x18 }, { 0x221E, 0x15 }
 };
 
-static const struct alphabet_conversion_table alphabet_lookup[] = {
-	/* Default GSM 7 bit */
-	{ def_gsm, def_ext_gsm, TABLE_SIZE(def_ext_gsm),
-		def_unicode, def_ext_unicode, TABLE_SIZE(def_ext_unicode) },
-	/* Turkish GSM dialect */
-	{ tur_gsm, tur_ext_gsm, TABLE_SIZE(tur_ext_gsm),
-		tur_unicode, tur_ext_unicode, TABLE_SIZE(tur_ext_unicode) },
-	/* Spanish GSM dialect, note that this one only has extension table */
-	{ def_gsm, spa_ext_gsm, TABLE_SIZE(spa_ext_gsm),
-		def_unicode, spa_ext_unicode, TABLE_SIZE(spa_ext_unicode)  },
-	/* Portuguese GSM dialect */
-	{ por_gsm, por_ext_gsm, TABLE_SIZE(por_ext_gsm),
-		por_unicode, por_ext_unicode, TABLE_SIZE(por_ext_unicode) },
-};
-
 static int compare_codepoints(const void *a, const void *b)
 {
 	const struct codepoint *ca = (const struct codepoint *) a;
@@ -480,48 +474,102 @@ static unsigned short codepoint_lookup(struct codepoint *key,
 	return result ? result->to : GUND;
 }
 
-static unsigned short gsm_locking_shift_lookup(unsigned char k,
-						unsigned char lang)
+static unsigned short gsm_locking_shift_lookup(struct conversion_table *t,
+						unsigned char k)
 {
-	return alphabet_lookup[lang].togsm_locking_shift[k];
+	return t->locking_g[k];
 }
 
-static unsigned short gsm_single_shift_lookup(unsigned char k,
-						unsigned char lang)
+static unsigned short gsm_single_shift_lookup(struct conversion_table *t,
+						unsigned char k)
 {
 	struct codepoint key = { k, 0 };
-	const struct codepoint *table;
-	unsigned int len;
-
-	table = alphabet_lookup[lang].togsm_single_shift;
-	len = alphabet_lookup[lang].togsm_single_shift_len;
-
-	return codepoint_lookup(&key, table, len);
+	return codepoint_lookup(&key, t->single_g, t->single_len_g);
 }
 
-static unsigned short unicode_locking_shift_lookup(unsigned short k,
-							unsigned char lang)
+static unsigned short unicode_locking_shift_lookup(struct conversion_table *t,
+							unsigned short k)
 {
 	struct codepoint key = { k, 0 };
-	const struct codepoint *table;
-	unsigned int len = 128;
+	return codepoint_lookup(&key, t->locking_u, t->locking_len_u);
+}
 
-	table = alphabet_lookup[lang].tounicode_locking_shift;
+static unsigned short unicode_single_shift_lookup(struct conversion_table *t,
+							unsigned short k)
+{
+	struct codepoint key = { k, 0 };
+	return codepoint_lookup(&key, t->single_u, t->single_len_u);
+}
 
-	return codepoint_lookup(&key, table, len);
+static int populate_locking_shift(struct conversion_table *t,
+					enum gsm_dialect lang)
+{
+	switch (lang) {
+	case GSM_DIALECT_DEFAULT:
+	case GSM_DIALECT_SPANISH:
+		t->locking_g = def_gsm;
+		t->locking_u = def_unicode;
+		t->locking_len_u = TABLE_SIZE(def_unicode);
+		return 1;
+
+	case GSM_DIALECT_TURKISH:
+		t->locking_g = tur_gsm;
+		t->locking_u = tur_unicode;
+		t->locking_len_u = TABLE_SIZE(tur_unicode);
+		return 1;
+
+	case GSM_DIALECT_PORTUGUESE:
+		t->locking_g = por_gsm;
+		t->locking_u = por_unicode;
+		t->locking_len_u = TABLE_SIZE(por_unicode);
+		return 1;
+	}
+	return 0;
 }
 
-static unsigned short unicode_single_shift_lookup(unsigned short k,
-							unsigned char lang)
+static int populate_single_shift(struct conversion_table *t,
+					enum gsm_dialect lang)
 {
-	struct codepoint key = { k, 0 };
-	const struct codepoint *table;
-	unsigned int len;
+	switch (lang) {
+	case GSM_DIALECT_DEFAULT:
+		t->single_g = def_ext_gsm;
+		t->single_len_g = TABLE_SIZE(def_ext_gsm);
+		t->single_u = def_ext_unicode;
+		t->single_len_u = TABLE_SIZE(def_ext_unicode);
+		return 1;
+
+	case GSM_DIALECT_TURKISH:
+		t->single_g = tur_ext_gsm;
+		t->single_len_g = TABLE_SIZE(tur_ext_gsm);
+		t->single_u = tur_ext_unicode;
+		t->single_len_u = TABLE_SIZE(tur_ext_unicode);
+		return 1;
+
+	case GSM_DIALECT_SPANISH:
+		t->single_g = spa_ext_gsm;
+		t->single_len_g = TABLE_SIZE(spa_ext_gsm);
+		t->single_u = spa_ext_unicode;
+		t->single_len_u = TABLE_SIZE(spa_ext_unicode);
+		return 1;
+
+	case GSM_DIALECT_PORTUGUESE:
+		t->single_g = por_ext_gsm;
+		t->single_len_g = TABLE_SIZE(por_ext_gsm);
+		t->single_u = por_ext_unicode;
+		t->single_len_u = TABLE_SIZE(por_ext_unicode);
+		return 1;
+	}
+	return 0;
+}
 
-	table = alphabet_lookup[lang].tounicode_single_shift;
-	len = alphabet_lookup[lang].tounicode_single_shift_len;
+static int conversion_table_init(struct conversion_table *t,
+					enum gsm_dialect locking,
+					enum gsm_dialect single)
+{
+	memset(t, 0, sizeof(struct conversion_table));
 
-	return codepoint_lookup(&key, table, len);
+	return populate_locking_shift(t, locking) &&
+			populate_single_shift(t, single);
 }
 
 /*!
@@ -548,10 +596,9 @@ char *convert_gsm_to_utf8_with_lang(const unsigned char *text, long len,
 	long i = 0;
 	long res_length;
 
-	if (locking_lang >= GSM_DIALECT_INVALID)
-		return NULL;
+	struct conversion_table t;
 
-	if (single_lang >= GSM_DIALECT_INVALID)
+	if (!conversion_table_init(&t, locking_lang, single_lang))
 		return NULL;
 
 	if (len < 0 && !terminator)
@@ -577,12 +624,12 @@ char *convert_gsm_to_utf8_with_lang(const unsigned char *text, long len,
 			if (i >= len)
 				goto error;
 
-			c = gsm_single_shift_lookup(text[i], single_lang);
+			c = gsm_single_shift_lookup(&t, text[i]);
 
 			if (c == GUND)
 				goto error;
 		} else {
-			c = gsm_locking_shift_lookup(text[i], locking_lang);
+			c = gsm_locking_shift_lookup(&t, text[i]);
 		}
 
 		res_length += UTF8_LENGTH(c);
@@ -599,9 +646,9 @@ char *convert_gsm_to_utf8_with_lang(const unsigned char *text, long len,
 		unsigned short c;
 
 		if (text[i] == 0x1b)
-			c = gsm_single_shift_lookup(text[++i], single_lang);
+			c = gsm_single_shift_lookup(&t, text[++i]);
 		else
-			c = gsm_locking_shift_lookup(text[i], locking_lang);
+			c = gsm_locking_shift_lookup(&t, text[i]);
 
 		out += g_unichar_to_utf8(c, out);
 
@@ -647,6 +694,7 @@ unsigned char *convert_utf8_to_gsm_with_lang(const char *text, long len,
 					enum gsm_dialect locking_lang,
 					enum gsm_dialect single_lang)
 {
+	struct conversion_table t;
 	long nchars = 0;
 	const char *in;
 	unsigned char *out;
@@ -654,10 +702,7 @@ unsigned char *convert_utf8_to_gsm_with_lang(const char *text, long len,
 	long res_len;
 	long i;
 
-	if (locking_lang >= GSM_DIALECT_INVALID)
-		return NULL;
-
-	if (single_lang >= GSM_DIALECT_INVALID)
+	if (!conversion_table_init(&t, locking_lang, single_lang))
 		return NULL;
 
 	in = text;
@@ -674,10 +719,10 @@ unsigned char *convert_utf8_to_gsm_with_lang(const char *text, long len,
 		if (c > 0xffff)
 			goto err_out;
 
-		converted = unicode_locking_shift_lookup(c, locking_lang);
+		converted = unicode_locking_shift_lookup(&t, c);
 
 		if (converted == GUND)
-			converted = unicode_single_shift_lookup(c, single_lang);
+			converted = unicode_single_shift_lookup(&t, c);
 
 		if (converted == GUND)
 			goto err_out;
@@ -702,10 +747,10 @@ unsigned char *convert_utf8_to_gsm_with_lang(const char *text, long len,
 
 		gunichar c = g_utf8_get_char(in);
 
-		converted = unicode_locking_shift_lookup(c, locking_lang);
+		converted = unicode_locking_shift_lookup(&t, c);
 
 		if (converted == GUND)
-			converted = unicode_single_shift_lookup(c, single_lang);
+			converted = unicode_single_shift_lookup(&t, c);
 
 		if (converted & 0x1b00) {
 			*out = 0x1b;
@@ -1087,6 +1132,7 @@ unsigned char *pack_7bit(const unsigned char *in, long len, int byte_offset,
 
 char *sim_string_to_utf8(const unsigned char *buffer, int length)
 {
+	struct conversion_table t;
 	int i;
 	int j;
 	int num_chars;
@@ -1096,6 +1142,9 @@ char *sim_string_to_utf8(const unsigned char *buffer, int length)
 	char *utf8 = NULL;
 	char *out;
 
+	if (!conversion_table_init(&t, GSM_DIALECT_DEFAULT, GSM_DIALECT_DEFAULT))
+		return NULL;
+
 	if (length < 1)
 		return NULL;
 
@@ -1172,14 +1221,14 @@ char *sim_string_to_utf8(const unsigned char *buffer, int length)
 			if (i >= length)
 				return NULL;
 
-			c = gsm_single_shift_lookup(buffer[i++], 0);
+			c = gsm_single_shift_lookup(&t, buffer[i++]);
 
 			if (c == 0)
 				return NULL;
 
 			j += 2;
 		} else {
-			c = gsm_locking_shift_lookup(buffer[i++], 0);
+			c = gsm_locking_shift_lookup(&t, buffer[i++]);
 			j += 1;
 		}
 
@@ -1208,9 +1257,9 @@ char *sim_string_to_utf8(const unsigned char *buffer, int length)
 			c = (buffer[i++] & 0x7f) + ucs2_offset;
 		else if (buffer[i] == 0x1b) {
 			++i;
-			c = gsm_single_shift_lookup(buffer[i++], 0);
+			c = gsm_single_shift_lookup(&t, buffer[i++]);
 		} else
-			c = gsm_locking_shift_lookup(buffer[i++], 0);
+			c = gsm_locking_shift_lookup(&t, buffer[i++]);
 
 		out += g_unichar_to_utf8(c, out);
 	}
@@ -1220,8 +1269,8 @@ char *sim_string_to_utf8(const unsigned char *buffer, int length)
 	return utf8;
 }
 
-unsigned char *utf8_to_sim_string(const char *utf,
-					int max_length, int *out_length)
+unsigned char *utf8_to_sim_string(const char *utf, int max_length,
+					int *out_length)
 {
 	unsigned char *result;
 	unsigned char *ucs2;
@@ -1282,6 +1331,7 @@ unsigned char *convert_ucs2_to_gsm_with_lang(const unsigned char *text,
 					enum gsm_dialect locking_lang,
 					enum gsm_dialect single_lang)
 {
+	struct conversion_table t;
 	long nchars = 0;
 	const unsigned char *in;
 	unsigned char *out;
@@ -1289,10 +1339,7 @@ unsigned char *convert_ucs2_to_gsm_with_lang(const unsigned char *text,
 	long res_len;
 	long i;
 
-	if (locking_lang >= GSM_DIALECT_INVALID)
-		return NULL;
-
-	if (single_lang >= GSM_DIALECT_INVALID)
+	if (!conversion_table_init(&t, locking_lang, single_lang))
 		return NULL;
 
 	if (len < 1 || len % 2)
@@ -1308,10 +1355,10 @@ unsigned char *convert_ucs2_to_gsm_with_lang(const unsigned char *text,
 		if (c > 0xffff)
 			goto err_out;
 
-		converted = unicode_locking_shift_lookup(c, locking_lang);
+		converted = unicode_locking_shift_lookup(&t, c);
 
 		if (converted == GUND)
-			converted = unicode_single_shift_lookup(c, single_lang);
+			converted = unicode_single_shift_lookup(&t, c);
 
 		if (converted == GUND)
 			goto err_out;
@@ -1335,10 +1382,10 @@ unsigned char *convert_ucs2_to_gsm_with_lang(const unsigned char *text,
 		gunichar c = (in[i] << 8) | in[i + 1];
 		unsigned short converted = GUND;
 
-		converted = unicode_locking_shift_lookup(c, locking_lang);
+		converted = unicode_locking_shift_lookup(&t, c);
 
 		if (converted == GUND)
-			converted = unicode_single_shift_lookup(c, single_lang);
+			converted = unicode_single_shift_lookup(&t, c);
 
 		if (converted & 0x1b00) {
 			*out = 0x1b;
diff --git a/src/util.h b/src/util.h
index d09d496..8e4b7f3 100644
--- a/src/util.h
+++ b/src/util.h
@@ -24,7 +24,6 @@ enum gsm_dialect {
 	GSM_DIALECT_TURKISH,
 	GSM_DIALECT_SPANISH,
 	GSM_DIALECT_PORTUGUESE,
-	GSM_DIALECT_INVALID
 };
 
 char *convert_gsm_to_utf8(const unsigned char *text, long len, long *items_read,
-- 
1.7.1

Re: [PATCH 1/6] util: Refactor conversion table handling

[Denis Kenzior]

[-- Attachment #1: Type: text/plain, Size: 323 bytes --]

Hi Aki,

On 02/03/2011 07:20 AM, Aki Niemi wrote:
> ---
>  src/util.c |  205 +++++++++++++++++++++++++++++++++++++-----------------------
>  src/util.h |    1 -
>  2 files changed, 126 insertions(+), 80 deletions(-)
> 

Patch has been applied, but I pushed some minor style issues thereafter.

Regards,
-Denis

[PATCH 2/6] smsutil: Remove dialect checks from caller

[Aki Niemi]

[-- Attachment #1: Type: text/plain, Size: 790 bytes --]

---
 src/smsutil.c |   10 ----------
 1 files changed, 0 insertions(+), 10 deletions(-)

diff --git a/src/smsutil.c b/src/smsutil.c
index 1762391..5c1ae4c 100644
--- a/src/smsutil.c
+++ b/src/smsutil.c
@@ -2267,16 +2267,6 @@ char *sms_decode_text(GSList *sms_list)
 			sms_extract_language_variant(sms, &locking_shift,
 								&single_shift);
 
-			/*
-			 * If language is not defined in 3GPP TS 23.038,
-			 * implementations are instructed to ignore it
-			 */
-			if (locking_shift >= GSM_DIALECT_INVALID)
-				locking_shift = GSM_DIALECT_DEFAULT;
-
-			if (single_shift >= GSM_DIALECT_INVALID)
-				single_shift = GSM_DIALECT_DEFAULT;
-
 			converted = convert_gsm_to_utf8_with_lang(buf, written,
 								NULL, NULL, 0,
 								locking_shift,
-- 
1.7.1

Re: [PATCH 2/6] smsutil: Remove dialect checks from caller

[Denis Kenzior]

[-- Attachment #1: Type: text/plain, Size: 1103 bytes --]

Hi Aki,

On 02/03/2011 07:20 AM, Aki Niemi wrote:
> ---
>  src/smsutil.c |   10 ----------
>  1 files changed, 0 insertions(+), 10 deletions(-)
> 
> diff --git a/src/smsutil.c b/src/smsutil.c
> index 1762391..5c1ae4c 100644
> --- a/src/smsutil.c
> +++ b/src/smsutil.c
> @@ -2267,16 +2267,6 @@ char *sms_decode_text(GSList *sms_list)
>  			sms_extract_language_variant(sms, &locking_shift,
>  								&single_shift);
>  
> -			/*
> -			 * If language is not defined in 3GPP TS 23.038,
> -			 * implementations are instructed to ignore it
> -			 */
> -			if (locking_shift >= GSM_DIALECT_INVALID)
> -				locking_shift = GSM_DIALECT_DEFAULT;
> -
> -			if (single_shift >= GSM_DIALECT_INVALID)
> -				single_shift = GSM_DIALECT_DEFAULT;
> -
>  			converted = convert_gsm_to_utf8_with_lang(buf, written,
>  								NULL, NULL, 0,
>  								locking_shift,

I'm pretty sure you want to leave this check in, otherwise you will not
be compliant with 23.040.  If the dialect used is not understood, we
have to fall-back to the default, not ignore the message.

Regards,
-Denis

Re: [PATCH 2/6] smsutil: Remove dialect checks from caller

[Aki Niemi]

[-- Attachment #1: Type: text/plain, Size: 869 bytes --]

Hi Denis,

2011/2/3 Denis Kenzior <denkenz@gmail.com>:
>>                       converted = convert_gsm_to_utf8_with_lang(buf, written,
>>                                                               NULL, NULL, 0,
>>                                                               locking_shift,
>
> I'm pretty sure you want to leave this check in, otherwise you will not
> be compliant with 23.040.  If the dialect used is not understood, we
> have to fall-back to the default, not ignore the message.

Uh, you're right. My intention was to implement the fallback inside
convert_gsm_to_utf8_with_lang(), but then ended up factoring that
fallback behavior out with conversion_table_init()...

So maybe it's better done here anyway, and this patch can be ignored.

Cheers,
Aki

[PATCH 3/6] util: Add best dialect picker API

[Aki Niemi]

[-- Attachment #1: Type: text/plain, Size: 3431 bytes --]

---
 src/util.c |   64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/util.h |    7 ++++++
 2 files changed, 71 insertions(+), 0 deletions(-)

diff --git a/src/util.c b/src/util.c
index 547f38c..e1e8a0a 100644
--- a/src/util.c
+++ b/src/util.c
@@ -788,6 +788,70 @@ unsigned char *convert_utf8_to_gsm(const char *text, long len,
 }
 
 /*!
+ * Converts UTF-8 encoded text to GSM alphabet. It finds an encoding
+ * that uses the minimum set of GSM dialects based on the hint given.
+ *
+ * It first attempts to use the default dialect's single shift and
+ * locking shift tables. It then tries with only the single shift
+ * table of the hinted dialect, and finally with both the single shift
+ * and locking shift tables of the hinted dialect.
+ *
+ * Returns the encoded data or NULL if no suitable encoding could be
+ * found. The data must be freed by the caller. If items_read is not
+ * NULL, it contains the actual number of bytes read. If items_written
+ * is not NULL, it contains the number of bytes written. If
+ * used_locking and used_single are not NULL, they will contain the
+ * dialects used for the locking shift and single shift tables.
+ */
+unsigned char *convert_utf8_to_gsm_best_lang(const char *utf8, long len,
+					long *items_read, long *items_written,
+					unsigned char terminator,
+					enum gsm_dialect hint,
+					enum gsm_dialect *used_locking,
+					enum gsm_dialect *used_single)
+{
+	enum gsm_dialect locking = GSM_DIALECT_DEFAULT;
+	enum gsm_dialect single = GSM_DIALECT_DEFAULT;
+	unsigned char *encoded;
+
+	encoded = convert_utf8_to_gsm_with_lang(utf8, len, items_read,
+						items_written, terminator,
+						locking, single);
+	if (encoded != NULL)
+		return encoded;
+
+	if (hint == GSM_DIALECT_DEFAULT)
+		return NULL;
+
+	single = hint;
+	encoded = convert_utf8_to_gsm_with_lang(utf8, len, items_read,
+						items_written, terminator,
+						locking, single);
+	if (encoded != NULL)
+		return encoded;
+
+	/* Spanish dialect uses the default locking shift table */
+	if (hint == GSM_DIALECT_SPANISH)
+		return NULL;
+
+	locking = hint;
+	encoded = convert_utf8_to_gsm_with_lang(utf8, len, items_read,
+						items_written, terminator,
+						locking, single);
+
+	if (encoded == NULL)
+		return NULL;
+
+	if (used_locking != NULL)
+		*used_locking = locking;
+
+	if (used_single != NULL)
+		*used_single = single;
+
+	return encoded;
+}
+
+/*!
  * Decodes the hex encoded data and converts to a byte array.  If terminator
  * is not 0, the terminator character is appended to the end of the result.
  * This might be useful for converting GSM encoded data if the CSCS is set
diff --git a/src/util.h b/src/util.h
index 8e4b7f3..a1d11b4 100644
--- a/src/util.h
+++ b/src/util.h
@@ -44,6 +44,13 @@ unsigned char *convert_utf8_to_gsm_with_lang(const char *text, long len,
 					enum gsm_dialect locking_shift_lang,
 					enum gsm_dialect single_shift_lang);
 
+unsigned char *convert_utf8_to_gsm_best_lang(const char *utf8, long len,
+					long *items_read, long *items_written,
+					unsigned char terminator,
+					enum gsm_dialect hint,
+					enum gsm_dialect *used_locking,
+					enum gsm_dialect *used_single);
+
 unsigned char *decode_hex_own_buf(const char *in, long len, long *items_written,
 					unsigned char terminator,
 					unsigned char *buf);
-- 
1.7.1

Re: [PATCH 3/6] util: Add best dialect picker API

[Denis Kenzior]

[-- Attachment #1: Type: text/plain, Size: 1775 bytes --]

Hi Aki,

On 02/03/2011 07:20 AM, Aki Niemi wrote:
> ---
>  src/util.c |   64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
>  src/util.h |    7 ++++++
>  2 files changed, 71 insertions(+), 0 deletions(-)

<snip>

> +unsigned char *convert_utf8_to_gsm_best_lang(const char *utf8, long len,
> +					long *items_read, long *items_written,
> +					unsigned char terminator,
> +					enum gsm_dialect hint,
> +					enum gsm_dialect *used_locking,
> +					enum gsm_dialect *used_single)
> +{
> +	enum gsm_dialect locking = GSM_DIALECT_DEFAULT;
> +	enum gsm_dialect single = GSM_DIALECT_DEFAULT;
> +	unsigned char *encoded;
> +
> +	encoded = convert_utf8_to_gsm_with_lang(utf8, len, items_read,
> +						items_written, terminator,
> +						locking, single);
> +	if (encoded != NULL)
> +		return encoded;

So this was actually wrong since you were not setting used_locking or
used_single.

> +
> +	if (hint == GSM_DIALECT_DEFAULT)
> +		return NULL;
> +
> +	single = hint;
> +	encoded = convert_utf8_to_gsm_with_lang(utf8, len, items_read,
> +						items_written, terminator,
> +						locking, single);
> +	if (encoded != NULL)
> +		return encoded;

Same here

> +
> +	/* Spanish dialect uses the default locking shift table */
> +	if (hint == GSM_DIALECT_SPANISH)
> +		return NULL;
> +
> +	locking = hint;
> +	encoded = convert_utf8_to_gsm_with_lang(utf8, len, items_read,
> +						items_written, terminator,
> +						locking, single);
> +
> +	if (encoded == NULL)
> +		return NULL;
> +
> +	if (used_locking != NULL)
> +		*used_locking = locking;
> +
> +	if (used_single != NULL)
> +		*used_single = single;
> +
> +	return encoded;
> +}
> +

Applied and fixed in a follow on patch.

Regards,
-Denis

[PATCH 4/6] smsutil: Add alphabet support to encoding

[Aki Niemi]

[-- Attachment #1: Type: text/plain, Size: 4522 bytes --]

---
 src/smsutil.c |   53 ++++++++++++++++++++++++++++++++++++++++++++++-------
 src/smsutil.h |   12 ++++++++++++
 2 files changed, 58 insertions(+), 7 deletions(-)

diff --git a/src/smsutil.c b/src/smsutil.c
index 5c1ae4c..3fd3397 100644
--- a/src/smsutil.c
+++ b/src/smsutil.c
@@ -3514,9 +3514,10 @@ GSList *sms_datagram_prepare(const char *to,
  * @use_delivery_reports: value for the Status-Report-Request field
  *     (23.040 3.2.9, 9.2.2.2)
  */
-GSList *sms_text_prepare(const char *to, const char *utf8, guint16 ref,
-				gboolean use_16bit,
-				gboolean use_delivery_reports)
+GSList *sms_text_prepare_with_alphabet(const char *to, const char *utf8,
+					guint16 ref, gboolean use_16bit,
+					gboolean use_delivery_reports,
+					enum sms_alphabet alphabet)
 {
 	struct sms template;
 	int offset = 0;
@@ -3526,6 +3527,8 @@ GSList *sms_text_prepare(const char *to, const char *utf8, guint16 ref,
 	long left;
 	guint8 seq;
 	GSList *r = NULL;
+	enum gsm_dialect used_locking;
+	enum gsm_dialect used_single;
 
 	memset(&template, 0, sizeof(struct sms));
 	template.type = SMS_TYPE_SUBMIT;
@@ -3537,8 +3540,13 @@ GSList *sms_text_prepare(const char *to, const char *utf8, guint16 ref,
 	template.submit.vp.relative = 0xA7; /* 24 Hours */
 	sms_address_from_string(&template.submit.daddr, to);
 
-	/* UDHI, UDL, UD and DCS actually depend on what we have in the text */
-	gsm_encoded = convert_utf8_to_gsm(utf8, -1, NULL, &written, 0);
+	/*
+	 * UDHI, UDL, UD and DCS actually depend on the contents of
+	 * the text, and also on the GSM dialect we use to encode it.
+	 */
+	gsm_encoded = convert_utf8_to_gsm_best_lang(utf8, -1, NULL, &written, 0,
+							alphabet, &used_locking,
+							&used_single);
 	if (gsm_encoded == NULL) {
 		gsize converted;
 
@@ -3550,13 +3558,35 @@ GSList *sms_text_prepare(const char *to, const char *utf8, guint16 ref,
 	if (gsm_encoded == NULL && ucs2_encoded == NULL)
 		return NULL;
 
-	if (gsm_encoded)
+	if (gsm_encoded != NULL)
 		template.submit.dcs = 0x00; /* Class Unspecified, 7 Bit */
 	else
 		template.submit.dcs = 0x08; /* Class Unspecified, UCS2 */
 
+	if (gsm_encoded != NULL && used_single != GSM_DIALECT_DEFAULT) {
+		if (!offset)
+			offset = 1;
+
+		template.submit.ud[0] += 3;
+		template.submit.ud[offset] = SMS_IEI_NATIONAL_LANGUAGE_SINGLE_SHIFT;
+		template.submit.ud[offset + 1] = 1;
+		template.submit.ud[offset + 2] = used_single;
+		offset += 3;
+	}
+
+	if (gsm_encoded != NULL && used_locking != GSM_DIALECT_DEFAULT) {
+		if (!offset)
+			offset = 1;
+
+		template.submit.ud[0] += 3;
+		template.submit.ud[offset] = SMS_IEI_NATIONAL_LANGUAGE_LOCKING_SHIFT;
+		template.submit.ud[offset + 1] = 1;
+		template.submit.ud[offset + 2] = used_locking;
+		offset += 3;
+	}
+
 	if (offset != 0)
-		template.submit.udhi = FALSE;
+		template.submit.udhi = TRUE;
 
 	if (gsm_encoded && (written <= sms_text_capacity_gsm(160, offset))) {
 		template.submit.udl = written + (offset * 8 + 6) / 7;
@@ -3668,6 +3698,15 @@ GSList *sms_text_prepare(const char *to, const char *utf8, guint16 ref,
 	return r;
 }
 
+GSList *sms_text_prepare(const char *to, const char *utf8, guint16 ref,
+				gboolean use_16bit,
+				gboolean use_delivery_reports)
+{
+	return sms_text_prepare_with_alphabet(to, utf8, ref, use_16bit,
+						use_delivery_reports,
+						SMS_ALPHABET_DEFAULT);
+}
+
 gboolean cbs_dcs_decode(guint8 dcs, gboolean *udhi, enum sms_class *cls,
 			enum sms_charset *charset, gboolean *compressed,
 			enum cbs_language *language, gboolean *iso639)
diff --git a/src/smsutil.h b/src/smsutil.h
index 16b5350..fc81eb1 100644
--- a/src/smsutil.h
+++ b/src/smsutil.h
@@ -154,6 +154,13 @@ enum sms_charset {
 	SMS_CHARSET_UCS2 = 2,
 };
 
+enum sms_alphabet {
+	SMS_ALPHABET_DEFAULT = 0,
+	SMS_ALPHABET_TURKISH,
+	SMS_ALPHABET_SPANISH,
+	SMS_ALPHABET_PORTUGUESE,
+};
+
 enum sms_mwi_type {
 	SMS_MWI_TYPE_VOICE = 0,
 	SMS_MWI_TYPE_FAX = 1,
@@ -538,6 +545,11 @@ GSList *sms_text_prepare(const char *to, const char *utf8, guint16 ref,
 				gboolean use_16bit,
 				gboolean use_delivery_reports);
 
+GSList *sms_text_prepare_with_alphabet(const char *to, const char *utf8,
+				guint16 ref, gboolean use_16bit,
+				gboolean use_delivery_reports,
+				enum sms_alphabet alphabet);
+
 GSList *sms_datagram_prepare(const char *to,
 				const unsigned char *data, unsigned int len,
 				guint16 ref, gboolean use_16bit_ref,
-- 
1.7.1

Re: [PATCH 4/6] smsutil: Add alphabet support to encoding

[Denis Kenzior]

[-- Attachment #1: Type: text/plain, Size: 290 bytes --]

Hi Aki,

On 02/03/2011 07:20 AM, Aki Niemi wrote:
> ---
>  src/smsutil.c |   53 ++++++++++++++++++++++++++++++++++++++++++++++-------
>  src/smsutil.h |   12 ++++++++++++
>  2 files changed, 58 insertions(+), 7 deletions(-)
> 

Patch has been applied, thanks.

Regards,
-Denis

[PATCH 5/6] sms: Add alphabet support to SMS atom

[Aki Niemi]

[-- Attachment #1: Type: text/plain, Size: 5038 bytes --]

---
 src/sms.c |   91 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 89 insertions(+), 2 deletions(-)

diff --git a/src/sms.c b/src/sms.c
index 73e067e..bb94b3d 100644
--- a/src/sms.c
+++ b/src/sms.c
@@ -77,6 +77,7 @@ struct ofono_sms {
 	GKeyFile *settings;
 	char *imsi;
 	int bearer;
+	enum sms_alphabet alphabet;
 	const struct ofono_sms_driver *driver;
 	void *driver_data;
 	struct ofono_atom *atom;
@@ -157,6 +158,38 @@ static int sms_bearer_from_string(const char *str)
 	return -1;
 }
 
+static const char *sms_alphabet_to_string(enum sms_alphabet alphabet)
+{
+	switch (alphabet) {
+	case SMS_ALPHABET_TURKISH:
+		return "turkish";
+	case SMS_ALPHABET_SPANISH:
+		return "spanish";
+	case SMS_ALPHABET_PORTUGUESE:
+		return "portuguese";
+	case SMS_ALPHABET_DEFAULT:
+		return "default";
+	}
+	return NULL;
+}
+
+static gboolean sms_alphabet_from_string(const char *str,
+						enum sms_alphabet *alphabet)
+{
+	if (g_str_equal(str, "default"))
+		*alphabet = SMS_ALPHABET_DEFAULT;
+	else if (g_str_equal(str, "turkish"))
+		*alphabet = SMS_ALPHABET_TURKISH;
+	else if (g_str_equal(str, "spanish"))
+		*alphabet = SMS_ALPHABET_SPANISH;
+	else if (g_str_equal(str, "portuguese"))
+		*alphabet = SMS_ALPHABET_PORTUGUESE;
+	else
+		return FALSE;
+
+	return TRUE;
+}
+
 static unsigned int add_sms_handler(struct ofono_watchlist *watchlist,
 					int dst, int src, void *notify,
 					void *data, ofono_destroy_func destroy)
@@ -253,6 +286,25 @@ static void set_bearer(struct ofono_sms *sms, int bearer)
 						DBUS_TYPE_STRING, &value);
 }
 
+static void set_alphabet(struct ofono_sms *sms, enum sms_alphabet alphabet)
+{
+	DBusConnection *conn = ofono_dbus_get_connection();
+	const char *path = __ofono_atom_get_path(sms->atom);
+	const char *value;
+
+	if (sms->alphabet == alphabet)
+		return;
+
+	sms->alphabet = alphabet;
+
+	value = sms_alphabet_to_string(sms->alphabet);
+
+	ofono_dbus_signal_property_changed(conn, path,
+						OFONO_MESSAGE_MANAGER_INTERFACE,
+						"Alphabet",
+						DBUS_TYPE_STRING, &value);
+}
+
 static void set_sca(struct ofono_sms *sms,
 			const struct ofono_phone_number *sca)
 {
@@ -284,6 +336,7 @@ static DBusMessage *generate_get_properties_reply(struct ofono_sms *sms,
 	DBusMessageIter dict;
 	const char *sca;
 	const char *bearer;
+	const char *alphabet;
 
 	reply = dbus_message_new_method_return(msg);
 	if (reply == NULL)
@@ -306,6 +359,9 @@ static DBusMessage *generate_get_properties_reply(struct ofono_sms *sms,
 	bearer = sms_bearer_to_string(sms->bearer);
 	ofono_dbus_dict_append(&dict, "Bearer", DBUS_TYPE_STRING, &bearer);
 
+	alphabet = sms_alphabet_to_string(sms->alphabet);
+	ofono_dbus_dict_append(&dict, "Alphabet", DBUS_TYPE_STRING, &alphabet);
+
 	dbus_message_iter_close_container(&iter, &dict);
 
 	return reply;
@@ -515,6 +571,24 @@ static DBusMessage *sms_set_property(DBusConnection *conn, DBusMessage *msg,
 		return NULL;
 	}
 
+	if (!strcmp(property, "Alphabet")) {
+		const char *value;
+		enum sms_alphabet alphabet;
+
+		if (dbus_message_iter_get_arg_type(&var) != DBUS_TYPE_STRING)
+			return __ofono_error_invalid_args(msg);
+
+		dbus_message_iter_get_basic(&var, &value);
+
+		if (!sms_alphabet_from_string(value, &alphabet))
+			return __ofono_error_invalid_format(msg);
+
+		set_alphabet(sms, alphabet);
+
+		g_dbus_send_reply(conn, msg, DBUS_TYPE_INVALID);
+		return NULL;
+	}
+
 	return __ofono_error_invalid_args(msg);
 }
 
@@ -867,8 +941,10 @@ static DBusMessage *sms_send_message(DBusConnection *conn, DBusMessage *msg,
 	if (valid_phone_number_format(to) == FALSE)
 		return __ofono_error_invalid_format(msg);
 
-	msg_list = sms_text_prepare(to, text, sms->ref, use_16bit_ref,
-						sms->use_delivery_reports);
+	msg_list = sms_text_prepare_with_alphabet(to, text, sms->ref,
+						use_16bit_ref,
+						sms->use_delivery_reports,
+						sms->alphabet);
 
 	if (msg_list == NULL)
 		return __ofono_error_invalid_format(msg);
@@ -1585,6 +1661,8 @@ static void sms_remove(struct ofono_atom *atom)
 					sms->use_delivery_reports);
 		g_key_file_set_integer(sms->settings, SETTINGS_GROUP,
 					"Bearer", sms->bearer);
+		g_key_file_set_integer(sms->settings, SETTINGS_GROUP,
+					"Alphabet", sms->alphabet);
 
 		storage_close(sms->imsi, SETTINGS_STORE, sms->settings, TRUE);
 
@@ -1689,6 +1767,15 @@ static void sms_load_settings(struct ofono_sms *sms, const char *imsi)
 							"Bearer", &error);
 	if (error)
 		sms->bearer = 3; /* Default to CS then PS */
+
+	sms->alphabet = g_key_file_get_integer(sms->settings, SETTINGS_GROUP,
+						"Alphabet", NULL);
+	if (sms_alphabet_to_string(sms->alphabet) != NULL)
+		return;
+
+	sms->alphabet = SMS_ALPHABET_DEFAULT;
+	g_key_file_set_integer(sms->settings, SETTINGS_GROUP, "Alphabet",
+				sms->alphabet);
 }
 
 static void bearer_init_callback(const struct ofono_error *error, void *data)
-- 
1.7.1

Re: [PATCH 5/6] sms: Add alphabet support to SMS atom

[Denis Kenzior]

[-- Attachment #1: Type: text/plain, Size: 303 bytes --]

Hi Aki,

On 02/03/2011 07:20 AM, Aki Niemi wrote:
> ---
>  src/sms.c |   91 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
>  1 files changed, 89 insertions(+), 2 deletions(-)
> 

So I went ahead and applied this patch.  I pushed some minor fixes
afterwards.

Regards,
-Denis

[PATCH 6/6] doc: Add documentation for Alphabet property

[Aki Niemi]

[-- Attachment #1: Type: text/plain, Size: 749 bytes --]

---
 doc/message-api.txt |   14 ++++++++++++++
 1 files changed, 14 insertions(+), 0 deletions(-)

diff --git a/doc/message-api.txt b/doc/message-api.txt
index 1c68aee..d5a7b15 100644
--- a/doc/message-api.txt
+++ b/doc/message-api.txt
@@ -24,3 +24,17 @@ Properties	string State
 				"pending",
 				"sent",
 				"failed"
+
+		string Alphabet
+
+			Contains the alphabet setting for outgoing SMSs.
+			Possible values are:
+
+				"default" - Default GSM alphabet
+				"turkish" - Turkish alphabet
+				"spanish" - Spanish alphabet
+				"portuguese" - Portuguese alphabet
+
+			The standard, language-specific alphabets are defined in
+			3GPP TS23.038, Annex A.  By default, oFono uses "default"
+			setting.
-- 
1.7.1

Re: [PATCH 6/6] doc: Add documentation for Alphabet property

[Denis Kenzior]

[-- Attachment #1: Type: text/plain, Size: 992 bytes --]

Hi Aki,

On 02/03/2011 07:20 AM, Aki Niemi wrote:
> ---
>  doc/message-api.txt |   14 ++++++++++++++
>  1 files changed, 14 insertions(+), 0 deletions(-)
> 
> diff --git a/doc/message-api.txt b/doc/message-api.txt
> index 1c68aee..d5a7b15 100644
> --- a/doc/message-api.txt
> +++ b/doc/message-api.txt
> @@ -24,3 +24,17 @@ Properties	string State
>  				"pending",
>  				"sent",
>  				"failed"
> +
> +		string Alphabet
> +
> +			Contains the alphabet setting for outgoing SMSs.
> +			Possible values are:
> +
> +				"default" - Default GSM alphabet
> +				"turkish" - Turkish alphabet
> +				"spanish" - Spanish alphabet
> +				"portuguese" - Portuguese alphabet
> +
> +			The standard, language-specific alphabets are defined in
> +			3GPP TS23.038, Annex A.  By default, oFono uses "default"
> +			setting.

Before I look at patch 5, are you sure you want this as a property? I
thought you were a proponent of hiding this in main.conf?

Regards,
-Denis

Re: [PATCH 6/6] doc: Add documentation for Alphabet property

[Denis Kenzior]

[-- Attachment #1: Type: text/plain, Size: 278 bytes --]

Hi Aki,

On 02/03/2011 07:20 AM, Aki Niemi wrote:
> ---
>  doc/message-api.txt |   14 ++++++++++++++
>  1 files changed, 14 insertions(+), 0 deletions(-)
> 

So this patch was actually against the wrong file.  I went ahead and
fixed it up for you.

Regards,
-Denis