dovecot-2.2: lib-fts: Fixed assert-crash in fts-tokenizer-generic

[dovecot at dovecot.org]

details:   http://hg.dovecot.org/dovecot-2.2/rev/aa3374b9ce0f
changeset: 18634:aa3374b9ce0f
user:      Timo Sirainen <tss at iki.fi>
date:      Mon May 11 14:42:18 2015 +0300
description:
lib-fts: Fixed assert-crash in fts-tokenizer-generic

diffstat:

 src/lib-fts/fts-tokenizer-generic.c |  20 +++++++++++---------
 1 files changed, 11 insertions(+), 9 deletions(-)

diffs (60 lines):

diff -r 2483039db977 -r aa3374b9ce0f src/lib-fts/fts-tokenizer-generic.c
--- a/src/lib-fts/fts-tokenizer-generic.c	Mon May 11 14:35:49 2015 +0300
+++ b/src/lib-fts/fts-tokenizer-generic.c	Mon May 11 14:42:18 2015 +0300
@@ -91,6 +91,7 @@
 
 	/* if input is truncated with a partial UTF-8 character, drop it */
 	(void)uni_utf8_partial_strlen_n(data, size, &pos);
+	i_assert(pos > 0);
 	return t_strndup(data, pos);
 }
 
@@ -495,7 +496,7 @@
 
 	return FALSE;
 }
-static void
+static bool
 fts_tokenizer_generic_tr29_current_token(struct generic_fts_tokenizer *tok,
                                          const char **token_r)
 {
@@ -505,12 +506,13 @@
 	if (is_one_past_end(tok))
 		end_skip = tok->last_size;
 
-	len = tok->token->used - end_skip;
-	i_assert(len > 0);
-	*token_r = fts_uni_strndup(tok->token->data, len);
-	buffer_set_used_size(tok->token, 0);
 	tok->prev_prev_letter = LETTER_TYPE_NONE;
 	tok->prev_letter = LETTER_TYPE_NONE;
+
+	len = tok->token->used - end_skip;
+	*token_r = len == 0 ? "" : fts_uni_strndup(tok->token->data, len);
+	buffer_set_used_size(tok->token, 0);
+	return len > 0;
 }
 
 struct letter_fn {
@@ -594,8 +596,8 @@
 			tok_append_truncated(tok, data + start_skip,
 					     char_start_i - start_skip);
 			*skip_r = i + 1;
-			fts_tokenizer_generic_tr29_current_token(tok, token_r);
-			return 1;
+			if (fts_tokenizer_generic_tr29_current_token(tok, token_r))
+				return 1;
 		}
 	}
 	i_assert(i >= start_skip && size >= start_skip);
@@ -605,8 +607,8 @@
 	if (size == 0 && tok->token->used > 0) {
 		/* return the last token */
 		*skip_r = 0;
-		fts_tokenizer_generic_tr29_current_token(tok, token_r);
-		return 1;
+		if (fts_tokenizer_generic_tr29_current_token(tok, token_r))
+			return 1;
 	}
 	return 0;
 }