dovecot-2.0: mdbox: Avoid rebuilding storage if another process ...
dovecot at dovecot.org
dovecot at dovecot.org
Thu Apr 29 18:43:07 EEST 2010
details: http://hg.dovecot.org/dovecot-2.0/rev/de0ed7bea201
changeset: 11216:de0ed7bea201
user: Timo Sirainen <tss at iki.fi>
date: Thu Apr 29 18:43:02 2010 +0300
description:
mdbox: Avoid rebuilding storage if another process already did it.
diffstat:
src/lib-storage/index/dbox-common/dbox-file.c | 4 +-
src/lib-storage/index/dbox-common/dbox-storage.h | 5 +-
src/lib-storage/index/dbox-multi/mdbox-mail.c | 7 +-
src/lib-storage/index/dbox-multi/mdbox-map.c | 57 ++++++++----------
src/lib-storage/index/dbox-multi/mdbox-map.h | 5 +
src/lib-storage/index/dbox-multi/mdbox-purge.c | 2 +-
src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 24 ++++++-
src/lib-storage/index/dbox-multi/mdbox-storage.c | 35 ++++++++++-
src/lib-storage/index/dbox-multi/mdbox-storage.h | 7 ++
src/lib-storage/index/dbox-multi/mdbox-sync.c | 10 +-
src/lib-storage/index/dbox-single/sdbox-storage.c | 8 ++-
11 files changed, 109 insertions(+), 55 deletions(-)
diffs (truncated from 438 to 300 lines):
diff -r 85d6fb746783 -r de0ed7bea201 src/lib-storage/index/dbox-common/dbox-file.c
--- a/src/lib-storage/index/dbox-common/dbox-file.c Thu Apr 29 17:52:40 2010 +0300
+++ b/src/lib-storage/index/dbox-common/dbox-file.c Thu Apr 29 18:43:02 2010 +0300
@@ -54,14 +54,14 @@
{
va_list args;
- file->storage->files_corrupted = TRUE;
-
va_start(args, reason);
mail_storage_set_critical(&file->storage->storage,
"Corrupted dbox file %s (around offset=%"PRIuUOFF_T"): %s",
file->cur_path, file->input == NULL ? 0 : file->input->v_offset,
t_strdup_vprintf(reason, args));
va_end(args);
+
+ file->storage->v.set_file_corrupted(file);
}
void dbox_file_init(struct dbox_file *file)
diff -r 85d6fb746783 -r de0ed7bea201 src/lib-storage/index/dbox-common/dbox-storage.h
--- a/src/lib-storage/index/dbox-common/dbox-storage.h Thu Apr 29 17:52:40 2010 +0300
+++ b/src/lib-storage/index/dbox-common/dbox-storage.h Thu Apr 29 18:43:02 2010 +0300
@@ -5,6 +5,7 @@
struct dbox_file;
struct dbox_mail;
+struct dbox_storage;
#define DBOX_SUBSCRIPTION_FILE_NAME "subscriptions"
#define DBOX_UIDVALIDITY_FILE_NAME "dovecot-uidvalidity"
@@ -37,13 +38,13 @@
/* create/update mailbox indexes */
int (*mailbox_create_indexes)(struct mailbox *box,
const struct mailbox_update *update);
+ /* mark the file corrupted */
+ void (*set_file_corrupted)(struct dbox_file *file);
};
struct dbox_storage {
struct mail_storage storage;
struct dbox_storage_vfuncs v;
-
- unsigned int files_corrupted:1;
};
void dbox_storage_get_list_settings(const struct mail_namespace *ns,
diff -r 85d6fb746783 -r de0ed7bea201 src/lib-storage/index/dbox-multi/mdbox-mail.c
--- a/src/lib-storage/index/dbox-multi/mdbox-mail.c Thu Apr 29 17:52:40 2010 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-mail.c Thu Apr 29 18:43:02 2010 +0300
@@ -30,13 +30,13 @@
mail_storage_set_critical(&mbox->storage->storage.storage,
"dbox %s: map uid lost for uid %u",
mbox->box.path, uid);
- mbox->storage->storage.files_corrupted = TRUE;
+ mdbox_storage_set_corrupted(mbox->storage);
return -1;
}
if (mbox->map_uid_validity == 0) {
if (mdbox_read_header(mbox, &hdr) < 0) {
- mbox->storage->storage.files_corrupted = TRUE;
+ mdbox_storage_set_corrupted(mbox->storage);
return -1;
}
mbox->map_uid_validity = hdr.map_uid_validity;
@@ -50,7 +50,7 @@
"dbox %s: map uidvalidity mismatch (%u vs %u)",
mbox->box.path, mbox->map_uid_validity,
cur_map_uid_validity);
- mbox->storage->storage.files_corrupted = TRUE;
+ mdbox_storage_set_corrupted(mbox->storage);
return -1;
}
*map_uid_r = dbox_rec->map_uid;
@@ -72,7 +72,6 @@
"Unexpectedly lost %s uid=%u map_uid=%u",
mailbox_get_vname(_mail->box),
_mail->uid, map_uid);
- mbox->storage->storage.files_corrupted = TRUE;
}
static int dbox_mail_open_init(struct dbox_mail *mail, uint32_t map_uid)
diff -r 85d6fb746783 -r de0ed7bea201 src/lib-storage/index/dbox-multi/mdbox-map.c
--- a/src/lib-storage/index/dbox-multi/mdbox-map.c Thu Apr 29 17:52:40 2010 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-map.c Thu Apr 29 18:43:02 2010 +0300
@@ -33,14 +33,14 @@
{
va_list args;
- map->storage->storage.files_corrupted = TRUE;
-
va_start(args, format);
mail_storage_set_critical(MAP_STORAGE(map),
"dbox map %s corrupted: %s",
map->index->filepath,
t_strdup_vprintf(format, args));
va_end(args);
+
+ mdbox_storage_set_corrupted(map->storage);
}
struct dbox_map *
@@ -188,6 +188,26 @@
return 0;
}
+static void
+mdbox_map_get_ext_hdr(struct dbox_map *map, struct mail_index_view *view,
+ struct dbox_map_mail_index_header *hdr_r)
+{
+ const void *data;
+ size_t data_size;
+
+ mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size);
+ memset(hdr_r, 0, sizeof(*hdr_r));
+ memcpy(hdr_r, data, I_MIN(data_size, sizeof(*hdr_r)));
+}
+
+uint32_t mdbox_map_get_rebuild_count(struct dbox_map *map)
+{
+ struct dbox_map_mail_index_header hdr;
+
+ mdbox_map_get_ext_hdr(map, map->view, &hdr);
+ return hdr.rebuild_count;
+}
+
static int dbox_map_lookup_seq(struct dbox_map *map, uint32_t seq,
const struct dbox_map_mail_index_record **rec_r)
{
@@ -397,7 +417,7 @@
i_warning("dbox %s: Inconsistency in map index "
"(%u,%"PRIuUOFF_T" != %u,%"PRIuUOFF_T")",
map->path, seq1, offset1, seq2, offset2);
- map->storage->storage.files_corrupted = TRUE;
+ mdbox_storage_set_corrupted(map->storage);
} else {
while (mail_index_sync_next(sync_ctx, &sync_rec)) ;
}
@@ -962,35 +982,12 @@
appends[count-1].size = cur_offset - appends[count-1].offset;
}
-static int
-dbox_map_get_next_file_id(struct dbox_map *map, struct mail_index_view *view,
- uint32_t *file_id_r)
-{
- const struct dbox_map_mail_index_header *hdr;
- const void *data;
- size_t data_size;
-
- mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size);
- if (data_size != sizeof(*hdr)) {
- if (data_size != 0) {
- dbox_map_set_corrupted(map, "hdr size=%"PRIuSIZE_T,
- data_size);
- return -1;
- }
- /* first file */
- *file_id_r = 1;
- } else {
- hdr = data;
- *file_id_r = hdr->highest_file_id + 1;
- }
- return 0;
-}
-
static int dbox_map_assign_file_ids(struct dbox_map_append_context *ctx,
bool separate_transaction)
{
struct dbox_file_append_context *const *file_appends;
unsigned int i, count;
+ struct dbox_map_mail_index_header hdr;
uint32_t first_file_id, file_id;
int ret;
@@ -1006,10 +1003,8 @@
}
dbox_map_sync_handle(ctx->map, ctx->sync_ctx);
- if (dbox_map_get_next_file_id(ctx->map, ctx->sync_view, &file_id) < 0) {
- mail_index_sync_rollback(&ctx->sync_ctx);
- return -1;
- }
+ mdbox_map_get_ext_hdr(ctx->map, ctx->sync_view, &hdr);
+ file_id = hdr.highest_file_id + 1;
/* assign file_ids for newly created files */
first_file_id = file_id;
diff -r 85d6fb746783 -r de0ed7bea201 src/lib-storage/index/dbox-multi/mdbox-map.h
--- a/src/lib-storage/index/dbox-multi/mdbox-map.h Thu Apr 29 17:52:40 2010 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-map.h Thu Apr 29 18:43:02 2010 +0300
@@ -13,6 +13,8 @@
struct dbox_map_mail_index_header {
uint32_t highest_file_id;
+ /* increased every time storage is rebuilt */
+ uint32_t rebuild_count;
};
struct dbox_map_mail_index_record {
@@ -41,6 +43,9 @@
/* Refresh the map. Returns 0 if ok, -1 if error. */
int dbox_map_refresh(struct dbox_map *map);
+/* Return the current rebuild counter */
+uint32_t mdbox_map_get_rebuild_count(struct dbox_map *map);
+
/* Look up file_id and offset for given map UID. Returns 1 if ok, 0 if UID
is already expunged, -1 if error. */
int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
diff -r 85d6fb746783 -r de0ed7bea201 src/lib-storage/index/dbox-multi/mdbox-purge.c
--- a/src/lib-storage/index/dbox-multi/mdbox-purge.c Thu Apr 29 17:52:40 2010 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-purge.c Thu Apr 29 18:43:02 2010 +0300
@@ -533,7 +533,7 @@
} T_END;
mdbox_purge_free(&ctx);
- if (storage->storage.files_corrupted) {
+ if (storage->corrupted) {
/* purging found corrupted files */
(void)mdbox_storage_rebuild(storage);
ret = -1;
diff -r 85d6fb746783 -r de0ed7bea201 src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c
--- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Thu Apr 29 17:52:40 2010 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Thu Apr 29 18:43:02 2010 +0300
@@ -43,6 +43,7 @@
struct hash_table *guid_hash;
ARRAY_DEFINE(msgs, struct mdbox_rebuild_msg *);
+ uint32_t rebuild_count;
uint32_t prev_file_id;
uint32_t highest_seen_map_uid;
@@ -743,6 +744,11 @@
if (rebuild_handle_zero_refs(ctx) < 0)
return -1;
rebuild_update_refcounts(ctx);
+
+ ctx->rebuild_count++;
+ mail_index_update_header_ext(ctx->trans, ctx->storage->map->map_ext_id,
+ offsetof(struct dbox_map_mail_index_header, rebuild_count),
+ &ctx->rebuild_count, sizeof(ctx->rebuild_count));
return 0;
}
@@ -817,6 +823,16 @@
return -1;
}
+ /* get storage rebuild counter after locking */
+ ctx->rebuild_count = mdbox_map_get_rebuild_count(ctx->storage->map);
+ if (ctx->rebuild_count != ctx->storage->corrupted_rebuild_count &&
+ ctx->storage->corrupted) {
+ /* storage was already rebuilt by someone else */
+ return 0;
+ }
+
+ i_warning("dbox %s: rebuilding indexes", ctx->storage->storage_dir);
+
uid_validity = dbox_map_get_uid_validity(ctx->storage->map);
hdr = mail_index_get_header(ctx->sync_view);
if (hdr->uid_validity != uid_validity) {
@@ -859,13 +875,13 @@
return -1;
}
- i_warning("dbox %s: rebuilding indexes", storage->storage_dir);
-
ctx = mdbox_storage_rebuild_init(storage);
ret = mdbox_storage_rebuild_scan(ctx);
mdbox_storage_rebuild_deinit(ctx);
- if (ret == 0)
- storage->storage.files_corrupted = FALSE;
+ if (ret == 0) {
+ storage->corrupted = FALSE;
+ storage->corrupted_rebuild_count = 0;
+ }
return ret;
}
diff -r 85d6fb746783 -r de0ed7bea201 src/lib-storage/index/dbox-multi/mdbox-storage.c
--- a/src/lib-storage/index/dbox-multi/mdbox-storage.c Thu Apr 29 17:52:40 2010 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c Thu Apr 29 18:43:02 2010 +0300
@@ -127,8 +127,7 @@
{
struct mdbox_storage *mstorage = (struct mdbox_storage *)box->storage;
- if (mstorage->storage.files_corrupted &&
- !mstorage->rebuilding_storage)
+ if (mstorage->corrupted && !mstorage->rebuilding_storage)
(void)mdbox_storage_rebuild(mstorage);
index_storage_mailbox_close(box);
@@ -145,8 +144,9 @@
if (data_size < MDBOX_INDEX_HEADER_MIN_SIZE &&
(!mbox->creating || data_size != 0)) {
mail_storage_set_critical(&mbox->storage->storage.storage,
- "dbox %s: Invalid dbox header size",
- mbox->box.path);
More information about the dovecot-cvs
mailing list