dovecot-1.2: Virtual mailboxes: Speed up initial search result b...
dovecot at dovecot.org
dovecot at dovecot.org
Wed Jun 18 12:09:57 EEST 2008
details: http://hg.dovecot.org/dovecot-1.2/rev/29d6c17f2009
changeset: 7898:29d6c17f2009
user: Timo Sirainen <tss at iki.fi>
date: Wed Jun 18 12:09:48 2008 +0300
description:
Virtual mailboxes: Speed up initial search result building using modseqs.
diffstat:
7 files changed, 455 insertions(+), 23 deletions(-)
src/lib-storage/index/index-search-result.c | 3
src/lib-storage/mailbox-search-result-private.h | 1
src/lib-storage/mailbox-search-result.c | 3
src/plugins/virtual/virtual-config.c | 2
src/plugins/virtual/virtual-storage.c | 20 +
src/plugins/virtual/virtual-storage.h | 38 ++
src/plugins/virtual/virtual-sync.c | 411 ++++++++++++++++++++++-
diffs (truncated from 732 to 300 lines):
diff -r 02bef9a155d7 -r 29d6c17f2009 src/lib-storage/index/index-search-result.c
--- a/src/lib-storage/index/index-search-result.c Wed Jun 18 08:09:56 2008 +0300
+++ b/src/lib-storage/index/index-search-result.c Wed Jun 18 12:09:48 2008 +0300
@@ -118,6 +118,9 @@ int index_search_result_update_flags(str
{
struct mail_search_arg search_arg;
int ret;
+
+ if (array_count(changes) == 0)
+ return 0;
/* add a temporary search parameter to limit the search only to
the changed messages */
diff -r 02bef9a155d7 -r 29d6c17f2009 src/lib-storage/mailbox-search-result-private.h
--- a/src/lib-storage/mailbox-search-result-private.h Wed Jun 18 08:09:56 2008 +0300
+++ b/src/lib-storage/mailbox-search-result-private.h Wed Jun 18 12:09:48 2008 +0300
@@ -24,6 +24,7 @@ mailbox_search_result_alloc(struct mailb
enum mailbox_search_result_flags flags);
/* called when initial search is done. */
+void mailbox_search_result_initial_done(struct mail_search_result *result);
void mailbox_search_results_initial_done(struct mail_search_context *ctx);
void mailbox_search_result_add(struct mail_search_result *result, uint32_t uid);
diff -r 02bef9a155d7 -r 29d6c17f2009 src/lib-storage/mailbox-search-result.c
--- a/src/lib-storage/mailbox-search-result.c Wed Jun 18 08:09:56 2008 +0300
+++ b/src/lib-storage/mailbox-search-result.c Wed Jun 18 12:09:48 2008 +0300
@@ -95,8 +95,7 @@ mailbox_search_result_save(struct mail_s
return result;
}
-static void
-mailbox_search_result_initial_done(struct mail_search_result *result)
+void mailbox_search_result_initial_done(struct mail_search_result *result)
{
if ((result->flags & MAILBOX_SEARCH_RESULT_FLAG_QUEUE_SYNC) != 0) {
i_array_init(&result->removed_uids, 32);
diff -r 02bef9a155d7 -r 29d6c17f2009 src/plugins/virtual/virtual-config.c
--- a/src/plugins/virtual/virtual-config.c Wed Jun 18 08:09:56 2008 +0300
+++ b/src/plugins/virtual/virtual-config.c Wed Jun 18 12:09:48 2008 +0300
@@ -17,7 +17,6 @@ struct virtual_parse_context {
pool_t pool;
string_t *rule;
- unsigned int mailbox_id;
unsigned int rule_idx;
};
@@ -99,7 +98,6 @@ virtual_config_parse_line(struct virtual
/* new mailbox */
bbox = p_new(ctx->pool, struct virtual_backend_box, 1);
- bbox->mailbox_id = ++ctx->mailbox_id;
bbox->name = p_strdup(ctx->pool, line);
array_append(&ctx->mbox->backend_boxes, &bbox, 1);
return 0;
diff -r 02bef9a155d7 -r 29d6c17f2009 src/plugins/virtual/virtual-storage.c
--- a/src/plugins/virtual/virtual-storage.c Wed Jun 18 08:09:56 2008 +0300
+++ b/src/plugins/virtual/virtual-storage.c Wed Jun 18 12:09:48 2008 +0300
@@ -32,6 +32,7 @@ static int
static int
virtual_list_iter_is_mailbox(struct mailbox_list_iterate_context *ctx,
const char *dir, const char *fname,
+ const char *mailbox_name,
enum mailbox_list_file_type type,
enum mailbox_info_flags *flags);
@@ -120,16 +121,30 @@ static int virtual_create(struct mail_st
}
struct virtual_backend_box *
-virtual_backend_box_lookup(struct virtual_mailbox *mbox, uint32_t mailbox_id)
+virtual_backend_box_lookup_name(struct virtual_mailbox *mbox, const char *name)
{
struct virtual_backend_box *const *bboxes;
unsigned int i, count;
+ bboxes = array_get(&mbox->backend_boxes, &count);
+ for (i = 0; i < count; i++) {
+ if (strcmp(bboxes[i]->name, name) == 0)
+ return bboxes[i];
+ }
+ return NULL;
+}
+
+struct virtual_backend_box *
+virtual_backend_box_lookup(struct virtual_mailbox *mbox, uint32_t mailbox_id)
+{
+ struct virtual_backend_box *const *bboxes;
+ unsigned int i, count;
+
if (mailbox_id == 0)
return NULL;
bboxes = array_get(&mbox->backend_boxes, &count);
- for (i = mailbox_id-1; i < count; i++) {
+ for (i = 0; i < count; i++) {
if (bboxes[i]->mailbox_id == mailbox_id)
return bboxes[i];
}
@@ -390,6 +405,7 @@ virtual_list_iter_is_mailbox(struct mail
virtual_list_iter_is_mailbox(struct mailbox_list_iterate_context *ctx
ATTR_UNUSED,
const char *dir, const char *fname,
+ const char *mailbox_name ATTR_UNUSED,
enum mailbox_list_file_type type,
enum mailbox_info_flags *flags)
{
diff -r 02bef9a155d7 -r 29d6c17f2009 src/plugins/virtual/virtual-storage.h
--- a/src/plugins/virtual/virtual-storage.h Wed Jun 18 08:09:56 2008 +0300
+++ b/src/plugins/virtual/virtual-storage.h Wed Jun 18 12:09:48 2008 +0300
@@ -9,6 +9,31 @@
#define VIRTUAL_SUBSCRIPTION_FILE_NAME ".virtual-subscriptions"
#define VIRTUAL_CONFIG_FNAME "dovecot-virtual"
#define VIRTUAL_INDEX_PREFIX "dovecot.index"
+
+struct virtual_mail_index_header {
+ /* Increased by one each time the header is modified */
+ uint32_t change_counter;
+ /* Number of mailbox records following this header. Mailbox names
+ follow the mailbox records - they have neither NUL terminator nor
+ padding. */
+ uint32_t mailbox_count;
+ /* Highest used mailbox ID. IDs are never reused. */
+ uint32_t highest_mailbox_id;
+ uint32_t unused_padding;
+};
+
+struct virtual_mail_index_mailbox_record {
+ /* Unique mailbox ID used as mailbox_id in records. */
+ uint32_t id;
+ /* Length of this mailbox's name. */
+ uint32_t name_len;
+ /* Synced UID validity value */
+ uint32_t uid_validity;
+ /* Next unseen UID */
+ uint32_t next_uid;
+ /* Synced highest modseq value */
+ uint64_t highest_modseq;
+};
struct virtual_mail_index_record {
uint32_t mailbox_id;
@@ -27,8 +52,14 @@ struct virtual_backend_uidmap {
};
struct virtual_backend_box {
+ /* Initially zero, updated by syncing */
uint32_t mailbox_id;
const char *name;
+
+ unsigned int sync_mailbox_idx;
+ uint32_t sync_uid_validity;
+ uint32_t sync_next_uid;
+ uint64_t sync_highest_modseq;
struct mail_search_args *search_args;
struct mail_search_result *search_result;
@@ -52,10 +83,15 @@ struct virtual_mailbox {
const char *path;
uint32_t virtual_ext_id;
+ uint32_t prev_uid_validity;
+ uint32_t prev_change_counter;
+ uint32_t highest_mailbox_id;
+
/* Mailboxes this virtual mailbox consists of, sorted by mailbox_id */
ARRAY_DEFINE(backend_boxes, struct virtual_backend_box *);
unsigned int uids_mapped:1;
+ unsigned int sync_initialized:1;
};
extern struct mail_storage virtual_storage;
@@ -64,6 +100,8 @@ int virtual_config_read(struct virtual_m
int virtual_config_read(struct virtual_mailbox *mbox);
void virtual_config_free(struct virtual_mailbox *mbox);
+struct virtual_backend_box *
+virtual_backend_box_lookup_name(struct virtual_mailbox *mbox, const char *name);
struct virtual_backend_box *
virtual_backend_box_lookup(struct virtual_mailbox *mbox, uint32_t mailbox_id);
struct mailbox_transaction_context *
diff -r 02bef9a155d7 -r 29d6c17f2009 src/plugins/virtual/virtual-sync.c
--- a/src/plugins/virtual/virtual-sync.c Wed Jun 18 08:09:56 2008 +0300
+++ b/src/plugins/virtual/virtual-sync.c Wed Jun 18 12:09:48 2008 +0300
@@ -5,7 +5,10 @@
#include "bsearch-insert-pos.h"
#include "ioloop.h"
#include "str.h"
+#include "mail-index-modseq.h"
#include "mail-search-build.h"
+#include "mailbox-search-result-private.h"
+#include "index-search-result.h"
#include "virtual-storage.h"
#include <stdlib.h>
@@ -34,7 +37,11 @@ struct virtual_sync_context {
ARRAY_DEFINE(all_adds, struct virtual_add_record);
enum mailbox_sync_flags flags;
uint32_t uid_validity;
+
+ unsigned int ext_header_changed:1;
+ unsigned int ext_header_rewrite:1;
unsigned int expunge_removed:1;
+ unsigned int retry:1;
};
static void virtual_sync_set_uidvalidity(struct virtual_sync_context *ctx)
@@ -107,6 +114,192 @@ virtual_backend_box_sync_mail_unset(stru
mail_free(&bbox->sync_mail);
(void)mailbox_transaction_commit(&trans);
}
+}
+
+static int bbox_mailbox_id_cmp(const void *p1, const void *p2)
+{
+ const struct virtual_backend_box *b1 = p1, *b2 = p2;
+
+ if (b1->mailbox_id < b2->mailbox_id)
+ return -1;
+ if (b1->mailbox_id > b2->mailbox_id)
+ return 1;
+ return 0;
+}
+
+static bool virtual_sync_ext_header_read(struct virtual_sync_context *ctx)
+{
+ const struct virtual_mail_index_header *ext_hdr;
+ const struct mail_index_header *hdr;
+ const struct virtual_mail_index_mailbox_record *mailboxes;
+ struct virtual_backend_box *bbox, **bboxes;
+ const void *ext_data;
+ size_t ext_size;
+ unsigned int i, count, ext_name_offset, ext_mailbox_count;
+ uint32_t prev_mailbox_id;
+ bool ret;
+
+ hdr = mail_index_get_header(ctx->sync_view);
+ mail_index_get_header_ext(ctx->sync_view, ctx->mbox->virtual_ext_id,
+ &ext_data, &ext_size);
+ ext_hdr = ext_data;
+ if (ctx->mbox->sync_initialized &&
+ ctx->mbox->prev_uid_validity == hdr->uid_validity &&
+ ext_size >= sizeof(*ext_hdr) &&
+ ctx->mbox->prev_change_counter == ext_hdr->change_counter) {
+ /* fully refreshed */
+ return TRUE;
+ }
+
+ ctx->mbox->sync_initialized = TRUE;
+ ctx->mbox->prev_uid_validity = hdr->uid_validity;
+ if (ext_hdr == NULL) {
+ mailboxes = NULL;
+ ext_name_offset = 0;
+ ext_mailbox_count = 0;
+ } else {
+ ctx->mbox->prev_change_counter = ext_hdr->change_counter;
+ mailboxes = (const void *)(ext_hdr + 1);
+ ext_name_offset = sizeof(*ext_hdr) +
+ ext_hdr->mailbox_count * sizeof(*mailboxes);
+ if (ext_name_offset >= ext_size ||
+ ext_hdr->mailbox_count > INT_MAX/sizeof(*mailboxes)) {
+ i_error("virtual %s: Broken mailbox_count header",
+ ctx->mbox->path);
+ ext_mailbox_count = 0;
+ } else {
+ ext_mailbox_count = ext_hdr->mailbox_count;
+ }
+ }
+
+ /* update mailbox backends */
+ prev_mailbox_id = 0;
+ for (i = 0; i < ext_mailbox_count; i++) {
+ if (mailboxes[i].id > ext_hdr->highest_mailbox_id ||
+ mailboxes[i].id <= prev_mailbox_id) {
+ i_error("virtual %s: Broken mailbox id",
+ ctx->mbox->path);
+ break;
+ }
+ if (mailboxes[i].name_len == 0 ||
+ mailboxes[i].name_len > ext_size) {
+ i_error("virtual %s: Broken mailbox name_len",
+ ctx->mbox->path);
+ break;
+ }
+ if (ext_name_offset + mailboxes[i].name_len > ext_size) {
+ i_error("virtual %s: Broken mailbox list",
+ ctx->mbox->path);
+ break;
+ }
+ T_BEGIN {
+ const unsigned char *nameptr;
+ const char *name;
+
+ nameptr = CONST_PTR_OFFSET(ext_data, ext_name_offset);
+ name = t_strndup(nameptr, mailboxes[i].name_len);
+ bbox = virtual_backend_box_lookup_name(ctx->mbox, name);
+ } T_END;
+ if (bbox == NULL) {
More information about the dovecot-cvs
mailing list