dovecot: FTS API changes and Squat rewrite. Squat is still missi...

dovecot at dovecot.org dovecot at dovecot.org
Sun Dec 2 23:51:51 EET 2007


details:   http://hg.dovecot.org/dovecot/rev/e739cffd05ef
changeset: 6898:e739cffd05ef
user:      Timo Sirainen <tss at iki.fi>
date:      Sun Dec 02 23:51:46 2007 +0200
description:
FTS API changes and Squat rewrite. Squat is still missing expunge handling.

diffstat:

14 files changed, 3285 insertions(+), 3618 deletions(-)
src/plugins/fts-lucene/fts-backend-lucene.c |   25 
src/plugins/fts-lucene/lucene-wrapper.cc    |    4 
src/plugins/fts-squat/fts-backend-squat.c   |  127 -
src/plugins/fts-squat/squat-test.c          |  127 -
src/plugins/fts-squat/squat-trie-private.h  |  168 +
src/plugins/fts-squat/squat-trie.c          | 3254 ++++++++++-----------------
src/plugins/fts-squat/squat-trie.h          |   55 
src/plugins/fts-squat/squat-uidlist.c       | 2312 ++++++++++---------
src/plugins/fts-squat/squat-uidlist.h       |   95 
src/plugins/fts/Makefile.am                 |    4 
src/plugins/fts/fts-api-private.h           |   26 
src/plugins/fts/fts-api.c                   |  177 +
src/plugins/fts/fts-api.h                   |   23 
src/plugins/fts/fts-storage.c               |  506 +---

diffs (truncated from 7662 to 300 lines):

diff -r 0a3186f44dff -r e739cffd05ef src/plugins/fts-lucene/fts-backend-lucene.c
--- a/src/plugins/fts-lucene/fts-backend-lucene.c	Sun Dec 02 23:22:28 2007 +0200
+++ b/src/plugins/fts-lucene/fts-backend-lucene.c	Sun Dec 02 23:51:46 2007 +0200
@@ -101,23 +101,26 @@ fts_backend_lucene_get_last_uid(struct f
 	return lucene_index_get_last_uid(backend->lstorage->index, last_uid_r);
 }
 
-static struct fts_backend_build_context *
-fts_backend_lucene_build_init(struct fts_backend *_backend, uint32_t *last_uid_r)
+static int
+fts_backend_lucene_build_init(struct fts_backend *_backend,
+			      uint32_t *last_uid_r,
+			      struct fts_backend_build_context **ctx_r)
 {
 	struct lucene_fts_backend *backend =
 		(struct lucene_fts_backend *)_backend;
 	struct fts_backend_build_context *ctx;
 
 	fts_backend_select(backend);
+	if (lucene_index_build_init(backend->lstorage->index,
+				    &backend->last_uid) < 0)
+		return -1;
 
 	ctx = i_new(struct fts_backend_build_context, 1);
 	ctx->backend = _backend;
-	if (lucene_index_build_init(backend->lstorage->index,
-				    &backend->last_uid) < 0)
-		ctx->failed = TRUE;
 
 	*last_uid_r = backend->last_uid;
-	return ctx;
+	*ctx_r = ctx;
+	return 0;
 }
 
 static int
@@ -182,22 +185,24 @@ fts_backend_lucene_unlock(struct fts_bac
 
 static int
 fts_backend_lucene_lookup(struct fts_backend *_backend,
-			  enum fts_lookup_flags flags,
-			  const char *key, ARRAY_TYPE(seq_range) *result)
+			  const char *key, enum fts_lookup_flags flags,
+			  ARRAY_TYPE(seq_range) *definite_uids,
+			  ARRAY_TYPE(seq_range) *maybe_uids)
 {
 	struct lucene_fts_backend *backend =
 		(struct lucene_fts_backend *)_backend;
 
 	i_assert((flags & FTS_LOOKUP_FLAG_INVERT) == 0);
 
+	array_clear(maybe_uids);
 	fts_backend_select(backend);
 	return lucene_index_lookup(backend->lstorage->index,
-				   flags, key, result);
+				   flags, key, definite_uids);
 }
 
 struct fts_backend fts_backend_lucene = {
 	MEMBER(name) "lucene",
-	MEMBER(flags) FTS_BACKEND_FLAG_DEFINITE_LOOKUPS,
+	MEMBER(flags) 0,
 
 	{
 		fts_backend_lucene_init,
diff -r 0a3186f44dff -r e739cffd05ef src/plugins/fts-lucene/lucene-wrapper.cc
--- a/src/plugins/fts-lucene/lucene-wrapper.cc	Sun Dec 02 23:22:28 2007 +0200
+++ b/src/plugins/fts-lucene/lucene-wrapper.cc	Sun Dec 02 23:51:46 2007 +0200
@@ -549,7 +549,7 @@ int lucene_index_lookup(struct lucene_in
 	const char *quoted_key;
 	int ret = 0;
 
-	i_assert((flags & (FTS_LOOKUP_FLAG_HEADERS|FTS_LOOKUP_FLAG_BODY)) != 0);
+	i_assert((flags & (FTS_LOOKUP_FLAG_HEADER|FTS_LOOKUP_FLAG_BODY)) != 0);
 
 	if (lucene_index_open_search(index) <= 0)
 		return -1;
@@ -567,7 +567,7 @@ int lucene_index_lookup(struct lucene_in
 	BooleanQuery lookup_query;
 	Query *content_query1 = NULL, *content_query2 = NULL;
 	try {
-		if ((flags & FTS_LOOKUP_FLAG_HEADERS) != 0) {
+		if ((flags & FTS_LOOKUP_FLAG_HEADER) != 0) {
 			content_query1 = QueryParser::parse(tkey, _T("headers"),
 							    index->analyzer);
 			lookup_query.add(content_query1, false, false);
diff -r 0a3186f44dff -r e739cffd05ef src/plugins/fts-squat/fts-backend-squat.c
--- a/src/plugins/fts-squat/fts-backend-squat.c	Sun Dec 02 23:22:28 2007 +0200
+++ b/src/plugins/fts-squat/fts-backend-squat.c	Sun Dec 02 23:51:46 2007 +0200
@@ -16,7 +16,7 @@ struct squat_fts_backend {
 
 struct squat_fts_backend_build_context {
 	struct fts_backend_build_context ctx;
-	struct squat_trie_build_context *trie_ctx;
+	struct squat_trie_build_context *build_ctx;
 };
 
 static struct fts_backend *fts_backend_squat_init(struct mailbox *box)
@@ -43,7 +43,7 @@ static struct fts_backend *fts_backend_s
 	backend = i_new(struct squat_fts_backend, 1);
 	backend->backend = fts_backend_squat;
 	backend->trie =
-		squat_trie_open(t_strconcat(path, "/"SQUAT_FILE_PREFIX, NULL),
+		squat_trie_init(t_strconcat(path, "/"SQUAT_FILE_PREFIX, NULL),
 				status.uidvalidity, storage->lock_method,
 				mmap_disable);
 	return &backend->backend;
@@ -54,7 +54,7 @@ static void fts_backend_squat_deinit(str
 	struct squat_fts_backend *backend =
 		(struct squat_fts_backend *)_backend;
 
-	squat_trie_close(backend->trie);
+	squat_trie_deinit(&backend->trie);
 	i_free(backend);
 }
 
@@ -67,28 +67,39 @@ static int fts_backend_squat_get_last_ui
 	return squat_trie_get_last_uid(backend->trie, last_uid_r);
 }
 
-static struct fts_backend_build_context *
-fts_backend_squat_build_init(struct fts_backend *_backend, uint32_t *last_uid_r)
+static int
+fts_backend_squat_build_init(struct fts_backend *_backend, uint32_t *last_uid_r,
+			     struct fts_backend_build_context **ctx_r)
 {
 	struct squat_fts_backend *backend =
 		(struct squat_fts_backend *)_backend;
 	struct squat_fts_backend_build_context *ctx;
+	struct squat_trie_build_context *build_ctx;
+
+	if (squat_trie_build_init(backend->trie, last_uid_r, &build_ctx) < 0)
+		return -1;
 
 	ctx = i_new(struct squat_fts_backend_build_context, 1);
 	ctx->ctx.backend = _backend;
-	ctx->trie_ctx = squat_trie_build_init(backend->trie, last_uid_r);
-	return &ctx->ctx;
+	ctx->build_ctx = build_ctx;
+
+	*ctx_r = &ctx->ctx;
+	return 0;
 }
 
 static int
 fts_backend_squat_build_more(struct fts_backend_build_context *_ctx,
 			     uint32_t uid, const unsigned char *data,
-			     size_t size, bool headers ATTR_UNUSED)
+			     size_t size, bool headers)
 {
 	struct squat_fts_backend_build_context *ctx =
 		(struct squat_fts_backend_build_context *)_ctx;
+	enum squat_index_type squat_type;
 
-	return squat_trie_build_more(ctx->trie_ctx, uid, data, size);
+	squat_type = headers ? SQUAT_INDEX_TYPE_HEADER :
+		SQUAT_INDEX_TYPE_BODY;
+	return squat_trie_build_more(ctx->build_ctx, uid, squat_type,
+				     data, size);
 }
 
 static int
@@ -98,7 +109,7 @@ fts_backend_squat_build_deinit(struct ft
 		(struct squat_fts_backend_build_context *)_ctx;
 	int ret;
 
-	ret = squat_trie_build_deinit(ctx->trie_ctx);
+	ret = squat_trie_build_deinit(&ctx->build_ctx);
 	i_free(ctx);
 	return ret;
 }
@@ -109,55 +120,11 @@ fts_backend_squat_expunge(struct fts_bac
 {
 }
 
-static int get_uids(struct mailbox *box, ARRAY_TYPE(seq_range) *uids,
-		    unsigned int *message_count_r)
-{
-	struct mail_search_arg search_arg;
-        struct mailbox_transaction_context *t;
-	struct mail_search_context *ctx;
-	struct mail *mail;
-	unsigned int count = 0;
-	int ret;
-
-	memset(&search_arg, 0, sizeof(search_arg));
-	search_arg.type = SEARCH_ALL;
-
-	t = mailbox_transaction_begin(box, 0);
-	ctx = mailbox_search_init(t, NULL, &search_arg, NULL);
-
-	mail = mail_alloc(t, 0, NULL);
-	while (mailbox_search_next(ctx, mail) > 0) {
-		seq_range_array_add(uids, 0, mail->uid);
-		count++;
-	}
-	mail_free(&mail);
-
-	ret = mailbox_search_deinit(&ctx);
-	mailbox_transaction_rollback(&t);
-
-	*message_count_r = count;
-	return ret;
-}
-
 static void
 fts_backend_squat_expunge_finish(struct fts_backend *_backend,
 				 struct mailbox *box, bool committed)
 {
-	struct squat_fts_backend *backend =
-		(struct squat_fts_backend *)_backend;
-	ARRAY_TYPE(seq_range) uids = ARRAY_INIT;
-	unsigned int count;
-
-	if (!committed)
-		return;
-
-	t_push();
-	t_array_init(&uids, 128);
-	if (get_uids(box, &uids, &count) == 0) {
-		(void)squat_trie_mark_having_expunges(backend->trie, &uids,
-						      count);
-	}
-	t_pop();
+	/* FIXME */
 }
 
 static int fts_backend_squat_lock(struct fts_backend *_backend)
@@ -165,45 +132,39 @@ static int fts_backend_squat_lock(struct
 	struct squat_fts_backend *backend =
 		(struct squat_fts_backend *)_backend;
 
-	return squat_trie_lock(backend->trie, F_RDLCK);
+	squat_trie_refresh(backend->trie);
+	return 1;
 }
 
-static void fts_backend_squat_unlock(struct fts_backend *_backend)
+static void fts_backend_squat_unlock(struct fts_backend *_backend ATTR_UNUSED)
+{
+}
+
+static int
+fts_backend_squat_lookup(struct fts_backend *_backend, const char *key,
+			 enum fts_lookup_flags flags,
+			 ARRAY_TYPE(seq_range) *definite_uids,
+			 ARRAY_TYPE(seq_range) *maybe_uids)
 {
 	struct squat_fts_backend *backend =
 		(struct squat_fts_backend *)_backend;
-
-	squat_trie_unlock(backend->trie);
-}
-
-static int
-fts_backend_squat_lookup(struct fts_backend *_backend,
-			 enum fts_lookup_flags flags,
-			 const char *key, ARRAY_TYPE(seq_range) *result)
-{
-	struct squat_fts_backend *backend =
-		(struct squat_fts_backend *)_backend;
-
-	i_assert((flags & FTS_LOOKUP_FLAG_INVERT) == 0);
-	return squat_trie_lookup(backend->trie, result, key);
-}
-
-static int
-fts_backend_squat_filter(struct fts_backend *_backend,
-			 enum fts_lookup_flags flags,
-			 const char *key, ARRAY_TYPE(seq_range) *result)
-{
-	struct squat_fts_backend *backend =
-		(struct squat_fts_backend *)_backend;
+	enum squat_index_type squat_type = 0;
 
 	i_assert((flags & FTS_LOOKUP_FLAG_INVERT) == 0);
 
-	return squat_trie_filter(backend->trie, result, key);
+	if ((flags & FTS_LOOKUP_FLAG_HEADER) != 0)
+		squat_type |= SQUAT_INDEX_TYPE_HEADER;
+	if ((flags & FTS_LOOKUP_FLAG_BODY) != 0)
+		squat_type |= SQUAT_INDEX_TYPE_BODY;
+	i_assert(squat_type != 0);
+
+	return squat_trie_lookup(backend->trie, key, flags,
+				 definite_uids, maybe_uids);
 }
 
 struct fts_backend fts_backend_squat = {
 	MEMBER(name) "squat",
-	MEMBER(flags) FTS_BACKEND_FLAG_EXACT_LOOKUPS,
+	MEMBER(flags) FTS_BACKEND_FLAG_SUBSTRING_LOOKUPS,
 
 	{
 		fts_backend_squat_init,
@@ -217,6 +178,6 @@ struct fts_backend fts_backend_squat = {
 		fts_backend_squat_lock,
 		fts_backend_squat_unlock,
 		fts_backend_squat_lookup,
-		fts_backend_squat_filter


More information about the dovecot-cvs mailing list