dovecot-2.2: fts: Reset tokenizers before using them

[dovecot at dovecot.org]

details:   http://hg.dovecot.org/dovecot-2.2/rev/d2e0e9aed81c
changeset: 18581:d2e0e9aed81c
user:      Timo Sirainen <tss at iki.fi>
date:      Sat May 09 13:57:41 2015 +0300
description:
fts: Reset tokenizers before using them
This is mainly needed if the previous tokenization had failed.

diffstat:

 src/plugins/fts/fts-build-mail.c  |  10 +++++++++-
 src/plugins/fts/fts-search-args.c |   3 +++
 2 files changed, 12 insertions(+), 1 deletions(-)

diffs (34 lines):

diff -r 2dca6925bd88 -r d2e0e9aed81c src/plugins/fts/fts-build-mail.c
--- a/src/plugins/fts/fts-build-mail.c	Sat May 09 13:52:37 2015 +0300
+++ b/src/plugins/fts/fts-build-mail.c	Sat May 09 13:57:41 2015 +0300
@@ -456,8 +456,16 @@
 	memset(&ctx, 0, sizeof(ctx));
 	ctx.update_ctx = update_ctx;
 	ctx.mail = mail;
-	if ((update_ctx->backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0)
+	if ((update_ctx->backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) {
 		ctx.pending_input = buffer_create_dynamic(default_pool, 128);
+		/* reset tokenizer between mails - just to be sure no state
+		   leaks between mails (especially if previous indexing had
+		   failed) */
+		struct fts_tokenizer *tokenizer;
+
+		tokenizer = fts_user_get_index_tokenizer(update_ctx->backend->ns->user);
+		fts_tokenizer_reset(tokenizer);
+	}
 
 	prev_part = NULL;
 	parser = message_parser_init(pool_datastack_create(), input,
diff -r 2dca6925bd88 -r d2e0e9aed81c src/plugins/fts/fts-search-args.c
--- a/src/plugins/fts/fts-search-args.c	Sat May 09 13:52:37 2015 +0300
+++ b/src/plugins/fts/fts-search-args.c	Sat May 09 13:57:41 2015 +0300
@@ -114,6 +114,9 @@
 	and_arg->match_not = orig_arg->match_not;
 	and_arg->next = orig_arg->next;
 
+	/* reset tokenizer between search args in case there's any state left
+	   from some previous failure */
+	fts_tokenizer_reset(tokenizer);
 	while (fts_tokenizer_next(tokenizer,
 	                          (const void *)orig_token,
 	                          orig_token_len, &token) > 0) {