dovecot-1.1: Keep track of dovecot-uidlist mtime+size in index f...
dovecot at dovecot.org
dovecot at dovecot.org
Thu May 15 07:01:46 EEST 2008
details: http://hg.dovecot.org/dovecot-1.1/rev/c14a2b0a3126
changeset: 7515:c14a2b0a3126
user: Timo Sirainen <tss at iki.fi>
date: Thu May 15 07:01:26 2008 +0300
description:
Keep track of dovecot-uidlist mtime+size in index file. If it's up-to-date
(and unless indexes weren't updated at some point, it always is) and we're
saving a new message, don't bother reading the uidlist contents.
diffstat:
7 files changed, 224 insertions(+), 103 deletions(-)
src/lib-storage/index/maildir/maildir-save.c | 8
src/lib-storage/index/maildir/maildir-storage.h | 1
src/lib-storage/index/maildir/maildir-sync-index.c | 80 +++----
src/lib-storage/index/maildir/maildir-sync.c | 14 -
src/lib-storage/index/maildir/maildir-sync.h | 2
src/lib-storage/index/maildir/maildir-uidlist.c | 216 ++++++++++++++------
src/lib-storage/index/maildir/maildir-uidlist.h | 6
diffs (truncated from 632 to 300 lines):
diff -r 3b818654abfa -r c14a2b0a3126 src/lib-storage/index/maildir/maildir-save.c
--- a/src/lib-storage/index/maildir/maildir-save.c Thu May 15 06:05:39 2008 +0300
+++ b/src/lib-storage/index/maildir/maildir-save.c Thu May 15 07:01:26 2008 +0300
@@ -609,6 +609,11 @@ maildir_transaction_save_commit_pre_sync
if (maildir_sync_index_begin(mbox, NULL, &ctx->sync_ctx) < 0)
return -1;
+ if (maildir_sync_header_refresh(mbox) < 0)
+ return -1;
+ if (maildir_uidlist_refresh_fast_init(mbox->uidlist) < 0)
+ return 1;
+
ctx->keywords_sync_ctx =
maildir_sync_get_keywords_sync_ctx(ctx->sync_ctx);
@@ -659,7 +664,8 @@ int maildir_transaction_save_commit_pre(
i_assert(ctx->output == NULL);
i_assert(ctx->finished);
- sync_flags = MAILDIR_UIDLIST_SYNC_PARTIAL;
+ sync_flags = MAILDIR_UIDLIST_SYNC_PARTIAL |
+ MAILDIR_UIDLIST_SYNC_NOREFRESH;
/* if we want to assign UIDs or keywords, we require uidlist lock */
if ((t->ictx.flags & MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS) == 0 &&
diff -r 3b818654abfa -r c14a2b0a3126 src/lib-storage/index/maildir/maildir-storage.h
--- a/src/lib-storage/index/maildir/maildir-storage.h Thu May 15 06:05:39 2008 +0300
+++ b/src/lib-storage/index/maildir/maildir-storage.h Thu May 15 07:01:26 2008 +0300
@@ -57,6 +57,7 @@ struct maildir_index_header {
struct maildir_index_header {
uint32_t new_check_time, new_mtime, new_mtime_nsecs;
uint32_t cur_check_time, cur_mtime, cur_mtime_nsecs;
+ uint32_t uidlist_mtime, uidlist_mtime_nsecs, uidlist_size;
};
struct maildir_list_index_record {
diff -r 3b818654abfa -r c14a2b0a3126 src/lib-storage/index/maildir/maildir-sync-index.c
--- a/src/lib-storage/index/maildir/maildir-sync-index.c Thu May 15 06:05:39 2008 +0300
+++ b/src/lib-storage/index/maildir/maildir-sync-index.c Thu May 15 07:01:26 2008 +0300
@@ -186,6 +186,46 @@ int maildir_sync_index_begin(struct mail
return 0;
}
+static bool
+maildir_index_header_has_changed(const struct maildir_index_header *old_hdr,
+ const struct maildir_index_header *new_hdr)
+{
+#define DIR_DELAYED_REFRESH(hdr, name) \
+ ((hdr)->name ## _check_time <= \
+ (hdr)->name ## _mtime + MAILDIR_SYNC_SECS)
+
+ if (old_hdr->new_mtime != new_hdr->new_mtime ||
+ old_hdr->new_mtime_nsecs != new_hdr->new_mtime_nsecs ||
+ old_hdr->cur_mtime != new_hdr->cur_mtime ||
+ old_hdr->cur_mtime_nsecs != new_hdr->cur_mtime_nsecs ||
+ old_hdr->uidlist_mtime != new_hdr->uidlist_mtime ||
+ old_hdr->uidlist_mtime_nsecs != new_hdr->uidlist_mtime_nsecs ||
+ old_hdr->uidlist_size != new_hdr->uidlist_size)
+ return TRUE;
+
+ return DIR_DELAYED_REFRESH(old_hdr, new) !=
+ DIR_DELAYED_REFRESH(new_hdr, new) ||
+ DIR_DELAYED_REFRESH(old_hdr, cur) !=
+ DIR_DELAYED_REFRESH(new_hdr, cur);
+}
+
+static void
+maildir_sync_index_update_ext_header(struct maildir_index_sync_context *ctx)
+{
+ struct maildir_mailbox *mbox = ctx->mbox;
+ const void *data;
+ size_t data_size;
+
+ mail_index_get_header_ext(mbox->ibox.view, mbox->maildir_ext_id,
+ &data, &data_size);
+ if (data_size != sizeof(mbox->maildir_hdr) ||
+ maildir_index_header_has_changed(data, &mbox->maildir_hdr)) {
+ mail_index_update_header_ext(ctx->trans, mbox->maildir_ext_id,
+ 0, &mbox->maildir_hdr,
+ sizeof(mbox->maildir_hdr));
+ }
+}
+
int maildir_sync_index_finish(struct maildir_index_sync_context **_ctx,
bool failed, bool cancel)
{
@@ -198,6 +238,8 @@ int maildir_sync_index_finish(struct mai
if (ret < 0 || cancel)
mail_index_sync_rollback(&ctx->sync_ctx);
else {
+ maildir_sync_index_update_ext_header(ctx);
+
/* Set syncing_commit=TRUE so that if any sync callbacks try
to access mails which got lost (eg. expunge callback trying
to open the file which was just unlinked) we don't try to
@@ -217,43 +259,6 @@ int maildir_sync_index_finish(struct mai
index_sync_changes_deinit(&ctx->sync_changes);
i_free(ctx);
return ret;
-}
-
-static bool
-maildir_index_header_has_changed(const struct maildir_index_header *old_hdr,
- const struct maildir_index_header *new_hdr)
-{
-#define DIR_DELAYED_REFRESH(hdr, name) \
- ((hdr)->name ## _check_time <= \
- (hdr)->name ## _mtime + MAILDIR_SYNC_SECS)
-
- if (old_hdr->new_mtime != new_hdr->new_mtime ||
- old_hdr->cur_mtime != new_hdr->cur_mtime ||
- old_hdr->new_mtime_nsecs != new_hdr->new_mtime_nsecs ||
- old_hdr->cur_mtime_nsecs != new_hdr->cur_mtime_nsecs)
- return TRUE;
-
- return DIR_DELAYED_REFRESH(old_hdr, new) !=
- DIR_DELAYED_REFRESH(new_hdr, new) ||
- DIR_DELAYED_REFRESH(old_hdr, cur) !=
- DIR_DELAYED_REFRESH(new_hdr, cur);
-}
-
-static void
-maildir_index_update_ext_header(struct maildir_mailbox *mbox,
- struct mail_index_transaction *trans)
-{
- const void *data;
- size_t data_size;
-
- mail_index_get_header_ext(mbox->ibox.view, mbox->maildir_ext_id,
- &data, &data_size);
- if (data_size != sizeof(mbox->maildir_hdr) ||
- maildir_index_header_has_changed(data, &mbox->maildir_hdr)) {
- mail_index_update_header_ext(trans, mbox->maildir_ext_id, 0,
- &mbox->maildir_hdr,
- sizeof(mbox->maildir_hdr));
- }
}
int maildir_sync_index(struct maildir_index_sync_context *ctx,
@@ -461,7 +466,6 @@ int maildir_sync_index(struct maildir_in
if (ctx->changed)
mbox->maildir_hdr.cur_mtime = time(NULL);
- maildir_index_update_ext_header(mbox, trans);
if (uid_validity == 0) {
uid_validity = hdr->uid_validity != 0 ?
diff -r 3b818654abfa -r c14a2b0a3126 src/lib-storage/index/maildir/maildir-sync.c
--- a/src/lib-storage/index/maildir/maildir-sync.c Thu May 15 06:05:39 2008 +0300
+++ b/src/lib-storage/index/maildir/maildir-sync.c Thu May 15 07:01:26 2008 +0300
@@ -512,7 +512,7 @@ static int maildir_scan_dir(struct maild
(move_count <= MAILDIR_RENAME_RESCAN_COUNT ? 0 : 1);
}
-static int maildir_header_refresh(struct maildir_mailbox *mbox)
+int maildir_sync_header_refresh(struct maildir_mailbox *mbox)
{
const void *data;
size_t data_size;
@@ -529,10 +529,8 @@ static int maildir_header_refresh(struct
return 0;
}
- if (data_size != sizeof(mbox->maildir_hdr))
- i_warning("Maildir %s: Invalid header record size", mbox->path);
- else
- memcpy(&mbox->maildir_hdr, data, sizeof(mbox->maildir_hdr));
+ memcpy(&mbox->maildir_hdr, data,
+ I_MIN(sizeof(mbox->maildir_hdr), data_size));
return 0;
}
@@ -562,7 +560,7 @@ static int maildir_sync_quick_check(stru
bool refreshed = FALSE, check_new = FALSE, check_cur = FALSE;
if (mbox->maildir_hdr.new_mtime == 0) {
- if (maildir_header_refresh(mbox) < 0)
+ if (maildir_sync_header_refresh(mbox) < 0)
return -1;
if (mbox->maildir_hdr.new_mtime == 0) {
/* first sync */
@@ -577,7 +575,7 @@ static int maildir_sync_quick_check(stru
if (DIR_DELAYED_REFRESH(hdr, new) ||
DIR_DELAYED_REFRESH(hdr, cur)) {
/* refresh index and try again */
- if (maildir_header_refresh(mbox) < 0)
+ if (maildir_sync_header_refresh(mbox) < 0)
return -1;
refreshed = TRUE;
@@ -610,7 +608,7 @@ static int maildir_sync_quick_check(stru
break;
/* refresh index and try again */
- if (maildir_header_refresh(mbox) < 0)
+ if (maildir_sync_header_refresh(mbox) < 0)
return -1;
refreshed = TRUE;
}
diff -r 3b818654abfa -r c14a2b0a3126 src/lib-storage/index/maildir/maildir-sync.h
--- a/src/lib-storage/index/maildir/maildir-sync.h Thu May 15 06:05:39 2008 +0300
+++ b/src/lib-storage/index/maildir/maildir-sync.h Thu May 15 07:01:26 2008 +0300
@@ -24,6 +24,8 @@ maildir_storage_sync_init(struct mailbox
maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);
int maildir_storage_sync_force(struct maildir_mailbox *mbox, uint32_t uid);
+int maildir_sync_header_refresh(struct maildir_mailbox *mbox);
+
int maildir_sync_index_begin(struct maildir_mailbox *mbox,
struct maildir_sync_context *maildir_sync_ctx,
struct maildir_index_sync_context **ctx_r);
diff -r 3b818654abfa -r c14a2b0a3126 src/lib-storage/index/maildir/maildir-uidlist.c
--- a/src/lib-storage/index/maildir/maildir-uidlist.c Thu May 15 06:05:39 2008 +0300
+++ b/src/lib-storage/index/maildir/maildir-uidlist.c Thu May 15 07:01:26 2008 +0300
@@ -49,6 +49,7 @@
/* how many seconds to wait before overriding uidlist.lock */
#define UIDLIST_LOCK_STALE_TIMEOUT (60*2)
+#define UIDLIST_VERSION 3
#define UIDLIST_COMPRESS_PERCENTAGE 75
#define UIDLIST_IS_LOCKED(uidlist) \
@@ -90,6 +91,7 @@ struct maildir_uidlist {
unsigned int recreate:1;
unsigned int initial_read:1;
+ unsigned int initial_hdr_read:1;
unsigned int initial_sync:1;
};
@@ -124,7 +126,7 @@ static bool maildir_uidlist_iter_next_re
struct maildir_uidlist_rec **rec_r);
static int maildir_uidlist_lock_timeout(struct maildir_uidlist *uidlist,
- bool nonblock)
+ bool nonblock, bool refresh)
{
struct mailbox *box = &uidlist->ibox->box;
const char *control_dir, *path;
@@ -170,22 +172,25 @@ static int maildir_uidlist_lock_timeout(
uidlist->lock_count++;
- /* make sure we have the latest changes before changing anything */
- if (maildir_uidlist_refresh(uidlist) < 0) {
- maildir_uidlist_unlock(uidlist);
- return -1;
+ if (refresh) {
+ /* make sure we have the latest changes before
+ changing anything */
+ if (maildir_uidlist_refresh(uidlist) < 0) {
+ maildir_uidlist_unlock(uidlist);
+ return -1;
+ }
}
return 1;
}
int maildir_uidlist_lock(struct maildir_uidlist *uidlist)
{
- return maildir_uidlist_lock_timeout(uidlist, FALSE);
+ return maildir_uidlist_lock_timeout(uidlist, FALSE, TRUE);
}
int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist)
{
- return maildir_uidlist_lock_timeout(uidlist, TRUE);
+ return maildir_uidlist_lock_timeout(uidlist, TRUE, TRUE);
}
int maildir_uidlist_lock_touch(struct maildir_uidlist *uidlist)
@@ -293,6 +298,32 @@ static int maildir_uid_cmp(const void *p
(*rec1)->uid > (*rec2)->uid ? 1 : 0;
}
+static void ATTR_FORMAT(2, 3)
+maildir_uidlist_set_corrupted(struct maildir_uidlist *uidlist,
+ const char *fmt, ...)
+{
+ struct mail_storage *storage = uidlist->ibox->box.storage;
+ va_list args;
+
+ va_start(args, fmt);
+ mail_storage_set_critical(storage, "Broken file %s line %u: %s",
+ uidlist->path, uidlist->read_line_count,
+ t_strdup_vprintf(fmt, args));
+ va_end(args);
+}
+
+static void maildir_uidlist_update_hdr(struct maildir_uidlist *uidlist,
+ const struct stat *st)
+{
+ struct maildir_index_header *mhdr = &uidlist->mbox->maildir_hdr;
+
+ mhdr->uidlist_mtime = st->st_mtime;
+#ifdef HAVE_STAT_TV_NSEC
+ mhdr->uidlist_mtime_nsecs = st->st_mtim.tv_nsec;
+#endif
+ mhdr->uidlist_size = st->st_size;
+}
More information about the dovecot-cvs
mailing list