[dovecot-cvs] dovecot/src/lib-storage/index/dbox Makefile.am, 1.1,
1.2 dbox-file.c, 1.1, 1.2 dbox-sync-full.c, NONE,
1.1 dbox-sync.c, 1.2, 1.3 dbox-sync.h, 1.2, 1.3 dbox-uidlist.c,
1.4, 1.5 dbox-uidlist.h, 1.1, 1.2
cras at dovecot.org
cras at dovecot.org
Wed Dec 21 20:43:19 EET 2005
Update of /var/lib/cvs/dovecot/src/lib-storage/index/dbox
In directory talvi:/tmp/cvs-serv30849
Modified Files:
Makefile.am dbox-file.c dbox-sync.c dbox-sync.h dbox-uidlist.c
dbox-uidlist.h
Added Files:
dbox-sync-full.c
Log Message:
When index files (dovecot.index and dbox index) aren't synced with each
others do a full sync.
Index: Makefile.am
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/Makefile.am,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- Makefile.am 27 Nov 2005 23:05:29 -0000 1.1
+++ Makefile.am 21 Dec 2005 18:43:16 -0000 1.2
@@ -15,6 +15,7 @@
dbox-save.c \
dbox-sync.c \
dbox-sync-expunge.c \
+ dbox-sync-full.c \
dbox-storage.c \
dbox-transaction.c \
dbox-uidlist.c
Index: dbox-file.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-file.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- dbox-file.c 27 Nov 2005 23:05:29 -0000 1.1
+++ dbox-file.c 21 Dec 2005 18:43:16 -0000 1.2
@@ -126,6 +126,9 @@
return -1;
}
+ if (offset == 0)
+ offset = mbox->file->header_size;
+
return dbox_file_read_mail_header(mbox, mbox->file, offset);
}
--- NEW FILE: dbox-sync-full.c ---
/* Copyright (C) 2005 Timo Sirainen */
#include "lib.h"
#include "array.h"
#include "seq-range-array.h"
#include "dbox-storage.h"
#include "dbox-uidlist.h"
#include "dbox-file.h"
#include "dbox-sync.h"
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <dirent.h>
static int dbox_sync_full_mail(struct dbox_sync_context *ctx, uint32_t *seq_r)
{
struct dbox_mailbox *mbox = ctx->mbox;
const struct dbox_mail_header *hdr = &mbox->file->seeked_mail_header;
enum mail_flags flags;
uint32_t seq;
uint64_t hdr_offset = mbox->file->seeked_offset;
/* FIXME: mails can be in two places at the same time if we crashed
during copying expunge */
i_assert(hdr->expunged != '1');
if (mbox->file->seeked_uid >= ctx->mail_index_next_uid) {
/* new mail. append it. */
mail_index_append(ctx->trans, mbox->file->seeked_uid, &seq);
} else {
if (mail_index_lookup_uid_range(ctx->sync_view,
mbox->file->seeked_uid,
mbox->file->seeked_uid,
&seq, &seq) < 0) {
mail_storage_set_index_error(&ctx->mbox->ibox);
return -1;
}
if (seq == 0) {
/* not found. it should have been there. */
mail_storage_set_critical(STORAGE(mbox->storage),
"dbox %s sync: "
"UID %u inserted in the middle of mailbox",
mbox->path, mbox->file->seeked_uid);
mail_index_mark_corrupted(mbox->ibox.index);
return -1;
}
}
flags = 0;
if (hdr->answered)
flags |= MAIL_ANSWERED;
if (hdr->flagged)
flags |= MAIL_FLAGGED;
if (hdr->deleted)
flags |= MAIL_DELETED;
if (hdr->seen)
flags |= MAIL_SEEN;
if (hdr->draft)
flags |= MAIL_DRAFT;
mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, flags);
// FIXME: keywords
mail_index_update_ext(ctx->trans, seq, mbox->dbox_file_ext_idx,
&mbox->file->file_seq, NULL);
mail_index_update_ext(ctx->trans, seq, mbox->dbox_offset_ext_idx,
&hdr_offset, NULL);
*seq_r = seq;
return 0;
}
static int dbox_sync_full_file(struct dbox_sync_context *ctx, uint32_t file_seq)
{
struct dbox_mailbox *mbox = ctx->mbox;
struct dbox_uidlist_entry entry;
uint32_t seq;
int ret;
if ((ret = dbox_file_seek(mbox, file_seq, 0)) < 0) {
/* error / broken file */
return -1;
}
if (ret == 0) {
/* broken file, but without any useful data in it */
if (unlink(mbox->file->path) < 0) {
mail_storage_set_critical(STORAGE(mbox->storage),
"unlink(%s) failed: %m", mbox->file->path);
return -1;
}
return 0;
}
memset(&entry, 0, sizeof(entry));
entry.file_seq = file_seq;
ARRAY_CREATE(&entry.uid_list, pool_datastack_create(),
struct seq_range, 64);
do {
if (dbox_sync_full_mail(ctx, &seq) < 0)
return -1;
seq_range_array_add(&entry.uid_list, 0, seq);
seq_range_array_add(&ctx->exists, 0, seq);
} while ((ret = dbox_file_seek_next_nonexpunged(mbox)) > 0);
dbox_uidlist_sync_append(ctx->uidlist_sync_ctx, &entry);
return ret;
}
static void dbox_sync_full_expunge_nonfound(struct dbox_sync_context *ctx)
{
const struct seq_range *exists;
const struct mail_index_header *hdr;
unsigned int i, count;
uint32_t seq = 1;
exists = array_get(&ctx->exists, &count);
for (i = 0; i < count; i++) {
/* expunge seq .. exists[i]-1 */
while (seq < exists[i].seq1) {
mail_index_expunge(ctx->trans, seq);
seq++;
}
seq = exists[i].seq2 + 1;
}
hdr = mail_index_get_header(ctx->sync_view);
while (seq < hdr->messages_count) {
mail_index_expunge(ctx->trans, seq);
seq++;
}
}
int dbox_sync_full(struct dbox_sync_context *ctx)
{
struct dbox_mailbox *mbox = ctx->mbox;
const struct mail_index_header *hdr;
unsigned int file_prefix_len = strlen(DBOX_MAIL_FILE_PREFIX);
const char *path;
uint32_t file_seq;
DIR *dirp;
struct dirent *dp;
int ret = 0;
/* go through msg.* files, sync them to index and based on it
write dbox's index file */
path = t_strconcat(mbox->path, "/"DBOX_MAILDIR_NAME, NULL);
dirp = opendir(path);
if (dirp == NULL) {
mail_storage_set_critical(STORAGE(mbox->storage),
"opendir(%s) failed: %m", path);
return -1;
}
hdr = mail_index_get_header(ctx->sync_view);
ctx->mail_index_next_uid = hdr->next_uid;
dbox_uidlist_sync_from_scratch(ctx->uidlist_sync_ctx);
ARRAY_CREATE(&ctx->exists, default_pool, struct seq_range, 128);
while ((dp = readdir(dirp)) != NULL) {
if (strncmp(dp->d_name, DBOX_MAIL_FILE_PREFIX,
file_prefix_len) != 0 ||
!is_numeric(dp->d_name + file_prefix_len, '\0'))
continue;
file_seq = (uint32_t)strtoul(dp->d_name + file_prefix_len,
NULL, 10);
t_push();
ret = dbox_sync_full_file(ctx, file_seq);
t_pop();
if (ret < 0)
break;
}
if (closedir(dirp) < 0) {
mail_storage_set_critical(STORAGE(mbox->storage),
"closedir(%s) failed: %m", path);
ret = -1;
}
if (ret == 0)
dbox_sync_full_expunge_nonfound(ctx);
array_free(&ctx->exists);
return ret;
}
Index: dbox-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-sync.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- dbox-sync.c 21 Dec 2005 14:15:37 -0000 1.2
+++ dbox-sync.c 21 Dec 2005 18:43:16 -0000 1.3
@@ -23,7 +23,7 @@
if (ret == 0) {
mail_storage_set_critical(STORAGE(ctx->mbox->storage),
"Unexpectedly lost seq %u in "
- "dbox file %s", seq, ctx->mbox->path);
+ "dbox %s", seq, ctx->mbox->path);
}
return -1;
}
@@ -80,8 +80,10 @@
if (mail_index_lookup_uid_range(ctx->sync_view,
sync_rec->uid1, sync_rec->uid2,
- &seq1, &seq2) < 0)
+ &seq1, &seq2) < 0) {
+ mail_storage_set_index_error(&ctx->mbox->ibox);
return -1;
+ }
if (seq1 == 0) {
/* already expunged everything. nothing to do. */
@@ -186,7 +188,8 @@
if (ret < 0) {
mail_storage_set_critical(
STORAGE(mbox->storage),
- "pwrite(%s) failed: %m", mbox->path);
+ "pwrite(%s) failed: %m",
+ mbox->file->path);
return -1;
}
}
@@ -240,13 +243,50 @@
return 0;
}
-static int dbox_sync_index(struct dbox_mailbox *mbox)
+static int dbox_sync_index(struct dbox_sync_context *ctx)
{
- struct dbox_sync_context ctx;
struct mail_index_sync_rec sync_rec;
struct hash_iterate_context *iter;
- const struct mail_index_header *hdr;
void *key, *value;
+ int ret;
+
+ /* read all changes and sort them to file_seq order */
+ ctx->pool = pool_alloconly_create("dbox sync pool", 10240);
+ ctx->syncs = hash_create(default_pool, ctx->pool, 0, NULL, NULL);
+ for (;;) {
+ ret = mail_index_sync_next(ctx->index_sync_ctx, &sync_rec);
+ if (ret <= 0) {
+ if (ret < 0)
+ mail_storage_set_index_error(&ctx->mbox->ibox);
+ break;
+ }
+ if (dbox_sync_add(ctx, &sync_rec) < 0) {
+ ret = -1;
+ break;
+ }
+ }
+
+ iter = hash_iterate_init(ctx->syncs);
+ while (hash_iterate(iter, &key, &value)) {
+ const struct dbox_sync_file_entry *entry = value;
+
+ if (dbox_sync_file(ctx, entry) < 0) {
+ ret = -1;
+ break;
+ }
+ }
+ hash_iterate_deinit(iter);
+
+ hash_destroy(ctx->syncs);
+ pool_unref(ctx->pool);
+
+ return ret;
+}
+
+int dbox_sync(struct dbox_mailbox *mbox, int force)
+{
+ struct dbox_sync_context ctx;
+ const struct mail_index_header *hdr;
uint32_t seq, uid_validity, next_uid;
uoff_t offset;
time_t mtime;
@@ -265,40 +305,26 @@
mail_storage_set_index_error(&mbox->ibox);
return ret;
}
- if (dbox_uidlist_sync_init(mbox->uidlist, &ctx.uidlist_sync_ctx) < 0) {
+ if (dbox_uidlist_sync_init(mbox->uidlist, &ctx.uidlist_sync_ctx,
+ &mtime) < 0) {
mail_index_sync_rollback(ctx.index_sync_ctx);
return -1;
}
ctx.trans = mail_index_transaction_begin(ctx.sync_view, FALSE, TRUE);
- /* read all changes and sort them to file_seq order */
- ctx.pool = pool_alloconly_create("dbox sync pool", 10240);
- ctx.syncs = hash_create(default_pool, ctx.pool, 0, NULL, NULL);
- while ((ret = mail_index_sync_next(ctx.index_sync_ctx,
- &sync_rec)) > 0) {
- if (dbox_sync_add(&ctx, &sync_rec) < 0) {
- ret = -1;
- break;
- }
- }
-
- iter = hash_iterate_init(ctx.syncs);
- while (hash_iterate(iter, &key, &value)) {
- const struct dbox_sync_file_entry *entry = value;
-
- if (dbox_sync_file(&ctx, entry) < 0) {
- ret = -1;
- break;
- }
+ hdr = mail_index_get_header(ctx.sync_view);
+ if ((uint32_t)mtime != hdr->sync_stamp) {
+ /* indexes aren't synced. we'll do a full sync. */
+ force = TRUE;
}
- hash_iterate_deinit(iter);
- hash_destroy(ctx.syncs);
- pool_unref(ctx.pool);
+ if (force)
+ ret = dbox_sync_full(&ctx);
+ else
+ ret = dbox_sync_index(&ctx);
if (ret < 0) {
- mail_storage_set_index_error(&mbox->ibox);
mail_index_sync_rollback(ctx.index_sync_ctx);
dbox_uidlist_sync_rollback(ctx.uidlist_sync_ctx);
return -1;
@@ -316,7 +342,7 @@
if (hdr->next_uid != next_uid) {
mail_index_update_header(ctx.trans,
offsetof(struct mail_index_header, next_uid),
- &next_uid, sizeof(next_uid), TRUE);
+ &next_uid, sizeof(next_uid), FALSE);
}
if (dbox_uidlist_sync_commit(ctx.uidlist_sync_ctx, &mtime) < 0) {
@@ -338,23 +364,19 @@
return -1;
}
- if (mail_index_sync_commit(ctx.index_sync_ctx) < 0) {
- mail_storage_set_index_error(&mbox->ibox);
- return -1;
+ if (force) {
+ mail_index_sync_rollback(ctx.index_sync_ctx);
+ /* now that indexes are ok, sync changes from the index */
+ return dbox_sync(mbox, FALSE);
+ } else {
+ if (mail_index_sync_commit(ctx.index_sync_ctx) < 0) {
+ mail_storage_set_index_error(&mbox->ibox);
+ return -1;
+ }
}
return 0;
}
-int dbox_sync(struct dbox_mailbox *mbox, int force)
-{
- if (!force) {
- /* just sync index */
- return dbox_sync_index(mbox);
- }
-
- return -1;
-}
-
struct mailbox_sync_context *
dbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
Index: dbox-sync.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-sync.h,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- dbox-sync.h 21 Dec 2005 14:15:37 -0000 1.2
+++ dbox-sync.h 21 Dec 2005 18:43:16 -0000 1.3
@@ -41,6 +41,10 @@
uint32_t prev_file_seq;
uint32_t dotlock_failed_file_seq;
+
+ /* full sync: */
+ uint32_t mail_index_next_uid;
+ array_t ARRAY_DEFINE(exists, struct seq_range);
};
int dbox_sync(struct dbox_mailbox *mbox, int force);
@@ -57,5 +61,6 @@
int dbox_sync_expunge(struct dbox_sync_context *ctx,
const struct dbox_sync_file_entry *entry,
unsigned int sync_idx);
+int dbox_sync_full(struct dbox_sync_context *ctx);
#endif
Index: dbox-uidlist.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-uidlist.c,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- dbox-uidlist.c 21 Dec 2005 17:47:49 -0000 1.4
+++ dbox-uidlist.c 21 Dec 2005 18:43:16 -0000 1.5
@@ -157,7 +157,7 @@
const unsigned int *file_seq = key;
struct dbox_uidlist_entry *const *entry = p;
- return (int)file_seq - (int)(*entry)->file_seq;
+ return (int)*file_seq - (int)(*entry)->file_seq;
}
static int dbox_uidlist_add_entry(struct dbox_uidlist *uidlist,
@@ -537,10 +537,14 @@
/* grow mtime by one if needed to make sure the last write is noticed */
lock_path = file_dotlock_get_lock_path(uidlist->dotlock);
- if (stat(uidlist->path, &st) < 0 && errno != ENOENT) {
- mail_storage_set_critical(STORAGE(uidlist->mbox->storage),
- "stat(%s) failed: %m", uidlist->path);
- return -1;
+ if (stat(uidlist->path, &st) < 0) {
+ if (errno != ENOENT) {
+ mail_storage_set_critical(
+ STORAGE(uidlist->mbox->storage),
+ "stat(%s) failed: %m", uidlist->path);
+ return -1;
+ }
+ st.st_mtime = 0;
}
if (fstat(uidlist->lock_fd, &st2) < 0) {
mail_storage_set_critical(STORAGE(uidlist->mbox->storage),
@@ -551,9 +555,9 @@
if (st2.st_mtime <= st.st_mtime) {
struct utimbuf ut;
- st.st_mtime++;
+ st2.st_mtime = ++st.st_mtime;
ut.actime = ioloop_time;
- ut.modtime = st.st_mtime;
+ ut.modtime = st2.st_mtime;
if (utime(lock_path, &ut) < 0) {
mail_storage_set_critical(
@@ -563,8 +567,8 @@
}
}
- uidlist->ino = st.st_ino;
- uidlist->mtime = st.st_mtime;
+ uidlist->ino = st2.st_ino;
+ uidlist->mtime = st2.st_mtime;
/* now, finish the uidlist update by renaming the lock file to
uidlist */
@@ -1022,10 +1026,12 @@
}
int dbox_uidlist_sync_init(struct dbox_uidlist *uidlist,
- struct dbox_uidlist_sync_ctx **ctx_r)
+ struct dbox_uidlist_sync_ctx **ctx_r,
+ time_t *mtime_r)
{
int ret;
+ *mtime_r = -1;
if (dbox_uidlist_lock(uidlist) < 0)
return -1;
@@ -1037,6 +1043,8 @@
if (ret == 0) {
/* file is deleted */
uidlist->need_full_rewrite = TRUE;
+ } else {
+ *mtime_r = uidlist->mtime;
}
*ctx_r = i_new(struct dbox_uidlist_sync_ctx, 1);
@@ -1071,6 +1079,16 @@
i_free(ctx);
}
+void dbox_uidlist_sync_from_scratch(struct dbox_uidlist_sync_ctx *ctx)
+{
+ array_clear(&ctx->uidlist->entries);
+ ctx->uidlist->ino = 0;
+ ctx->uidlist->mtime = 0;
+
+ ctx->modified = TRUE;
+ ctx->uidlist->need_full_rewrite = TRUE;
+}
+
void dbox_uidlist_sync_set_modified(struct dbox_uidlist_sync_ctx *ctx)
{
ctx->modified = TRUE;
Index: dbox-uidlist.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-uidlist.h,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- dbox-uidlist.h 27 Nov 2005 23:05:29 -0000 1.1
+++ dbox-uidlist.h 21 Dec 2005 18:43:16 -0000 1.2
@@ -41,12 +41,15 @@
uint32_t *uid_r);
int dbox_uidlist_sync_init(struct dbox_uidlist *uidlist,
- struct dbox_uidlist_sync_ctx **ctx_r);
+ struct dbox_uidlist_sync_ctx **ctx_r,
+ time_t *mtime_r);
int dbox_uidlist_sync_commit(struct dbox_uidlist_sync_ctx *ctx,
time_t *mtime_r);
void dbox_uidlist_sync_rollback(struct dbox_uidlist_sync_ctx *ctx);
+void dbox_uidlist_sync_from_scratch(struct dbox_uidlist_sync_ctx *ctx);
void dbox_uidlist_sync_set_modified(struct dbox_uidlist_sync_ctx *ctx);
+
void dbox_uidlist_sync_append(struct dbox_uidlist_sync_ctx *ctx,
const struct dbox_uidlist_entry *entry);
int dbox_uidlist_sync_unlink(struct dbox_uidlist_sync_ctx *ctx,
More information about the dovecot-cvs
mailing list