1 From cc5681e3d07023e8684c5da962c4fb5fcecfd385 Mon Sep 17 00:00:00 2001
2 From: "Reynaldo H. Verdejo Pinochet" <reynaldo@osg.samsung.com>
3 Date: Fri, 28 Nov 2014 13:26:13 -0300
4 Subject: [PATCH] subparse: avoid false negatives dealing with UTF-8
6 g_utf8_validate() chokes at any NUL among max_len
7 bytes so we should avoid passing null character
8 terminators if present. Additionally, only part of
9 the available data might be valid UTF-8. For example
10 a byte at the end might be the start of a valid UTF-8
11 run (ie: d0) but not be a valid UTF-8 character by
12 itself. In this case, we consume only the valid portion
15 https://bugzilla.gnome.org/show_bug.cgi?id=740784
17 gst/subparse/gstsubparse.c | 23 +++++++++++++++++++++--
18 1 file changed, 21 insertions(+), 2 deletions(-)
20 diff --git a/gst/subparse/gstsubparse.c b/gst/subparse/gstsubparse.c
21 index 11b76c3..2719fd7 100644
22 --- a/gst/subparse/gstsubparse.c
23 +++ b/gst/subparse/gstsubparse.c
24 @@ -437,6 +437,9 @@ convert_encoding (GstSubParse * self, const gchar * str, gsize len,
25 const gchar *encoding;
29 + gsize valid_utf8_len;
30 + const gchar *invalid_utf8_start;
34 @@ -457,11 +460,27 @@ convert_encoding (GstSubParse * self, const gchar * str, gsize len,
36 /* Otherwise check if it's UTF8 */
37 if (self->valid_utf8) {
38 - if (g_utf8_validate (str, len, NULL)) {
39 + /* Trim NUL terminator(s) if present */
40 + while (len > 0 && str[len - 1] == '\0') {
45 + /* Consume whole byte run if all valid UTF-8 */
46 + if (g_utf8_validate (str, len, &invalid_utf8_start)) {
47 GST_LOG_OBJECT (self, "valid UTF-8, no conversion needed");
49 + *consumed = len + nuls;
50 return g_strndup (str, len);
53 + /* Consume initial data as far as we have at least 1 valid code point */
54 + valid_utf8_len = invalid_utf8_start - str;
55 + if (valid_utf8_len) {
56 + GST_WARNING_OBJECT (self, "At least some of the data was invalid UTF-8");
57 + *consumed = valid_utf8_len;
58 + return g_strndup (str, valid_utf8_len);
61 GST_INFO_OBJECT (self, "invalid UTF-8!");
62 self->valid_utf8 = FALSE;