dovecot-2.0: mdbox: Optimization: Don't update map UIDs' refcoun...
dovecot at dovecot.org
dovecot at dovecot.org
Mon Apr 19 16:16:14 EEST 2010
details: http://hg.dovecot.org/dovecot-2.0/rev/c2c2c1dd252e
changeset: 11170:c2c2c1dd252e
user: Timo Sirainen <tss at iki.fi>
date: Mon Apr 19 16:15:57 2010 +0300
description:
mdbox: Optimization: Don't update map UIDs' refcounts via array when it's not necessary.
diffstat:
src/lib-storage/index/dbox-multi/mdbox-map.c | 73 ++++++++++++++----------
src/lib-storage/index/dbox-multi/mdbox-map.h | 2 +
src/lib-storage/index/dbox-multi/mdbox-storage.c | 30 +++------
src/lib-storage/index/dbox-multi/mdbox-sync.c | 53 +++++++----------
src/lib-storage/index/dbox-multi/mdbox-sync.h | 4 +-
5 files changed, 77 insertions(+), 85 deletions(-)
diffs (265 lines):
diff -r 0eb423848dc1 -r c2c2c1dd252e src/lib-storage/index/dbox-multi/mdbox-map.c
--- a/src/lib-storage/index/dbox-multi/mdbox-map.c Mon Apr 19 16:14:37 2010 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-map.c Mon Apr 19 16:15:57 2010 +0300
@@ -409,50 +409,61 @@
i_free(ctx);
}
-int dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx,
- const ARRAY_TYPE(uint32_t) *map_uids, int diff)
+int dbox_map_update_refcount(struct dbox_map_transaction_context *ctx,
+ uint32_t map_uid, int diff)
{
struct dbox_map *map = ctx->map;
- const uint32_t *uidp;
- unsigned int i, count;
const void *data;
uint32_t seq;
bool expunged;
int cur_diff;
- if (ctx->trans == NULL)
+ if (unlikely(ctx->trans == NULL))
+ return -1;
+
+ if (!mail_index_lookup_seq(map->view, map_uid, &seq)) {
+ /* we can't refresh map here since view has a
+ transaction open. */
+ dbox_map_set_corrupted(map, "refcount update lost map_uid=%u",
+ map_uid);
+ return -1;
+ }
+ mail_index_lookup_ext(map->view, seq, map->ref_ext_id,
+ &data, &expunged);
+ cur_diff = data == NULL ? 0 : *((const uint16_t *)data);
+ ctx->changed = TRUE;
+ cur_diff += mail_index_atomic_inc_ext(ctx->trans, seq,
+ map->ref_ext_id, diff);
+ if (cur_diff < 0) {
+ dbox_map_set_corrupted(map, "map_uid=%u refcount too low",
+ map_uid);
+ return -1;
+ }
+ if (cur_diff >= 32768) {
+ /* we're getting close to the 64k limit. fail early
+ to make it less likely that two processes increase
+ the refcount enough times to cross the limit */
+ mail_storage_set_error(MAP_STORAGE(map), MAIL_ERROR_NOTPOSSIBLE,
+ "Message has been copied too many times");
+ return -1;
+ }
+ return 0;
+}
+
+int dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx,
+ const ARRAY_TYPE(uint32_t) *map_uids, int diff)
+{
+ const uint32_t *uidp;
+ unsigned int i, count;
+
+ if (unlikely(ctx->trans == NULL))
return -1;
count = array_count(map_uids);
for (i = 0; i < count; i++) {
uidp = array_idx(map_uids, i);
- if (!mail_index_lookup_seq(map->view, *uidp, &seq)) {
- /* we can't refresh map here since view has a
- transaction open. */
- dbox_map_set_corrupted(map,
- "refcount update lost map_uid=%u", *uidp);
+ if (dbox_map_update_refcount(ctx, *uidp, diff) < 0)
return -1;
- }
- mail_index_lookup_ext(map->view, seq, map->ref_ext_id,
- &data, &expunged);
- cur_diff = data == NULL ? 0 : *((const uint16_t *)data);
- ctx->changed = TRUE;
- cur_diff += mail_index_atomic_inc_ext(ctx->trans, seq,
- map->ref_ext_id, diff);
- if (cur_diff < 0) {
- dbox_map_set_corrupted(map,
- "map_uid=%u refcount too low", *uidp);
- return -1;
- }
- if (cur_diff >= 32768) {
- /* we're getting close to the 64k limit. fail early
- to make it less likely that two processes increase
- the refcount enough times to cross the limit */
- mail_storage_set_error(MAP_STORAGE(map),
- MAIL_ERROR_NOTPOSSIBLE,
- "Message has been copied too many times");
- return -1;
- }
}
return 0;
}
diff -r 0eb423848dc1 -r c2c2c1dd252e src/lib-storage/index/dbox-multi/mdbox-map.h
--- a/src/lib-storage/index/dbox-multi/mdbox-map.h Mon Apr 19 16:14:37 2010 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-map.h Mon Apr 19 16:15:57 2010 +0300
@@ -58,6 +58,8 @@
void dbox_map_transaction_set_failed(struct dbox_map_transaction_context *ctx);
void dbox_map_transaction_free(struct dbox_map_transaction_context **ctx);
+int dbox_map_update_refcount(struct dbox_map_transaction_context *ctx,
+ uint32_t map_uid, int diff);
int dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx,
const ARRAY_TYPE(uint32_t) *map_uids, int diff);
int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id);
diff -r 0eb423848dc1 -r c2c2c1dd252e src/lib-storage/index/dbox-multi/mdbox-storage.c
--- a/src/lib-storage/index/dbox-multi/mdbox-storage.c Mon Apr 19 16:14:37 2010 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c Mon Apr 19 16:15:57 2010 +0300
@@ -282,35 +282,27 @@
{
struct dbox_map_transaction_context *map_trans;
const struct mail_index_header *hdr;
- const struct mdbox_mail_index_record *dbox_rec;
- ARRAY_TYPE(uint32_t) map_uids;
- const void *data;
- bool expunged;
- uint32_t seq;
- int ret;
+ uint32_t seq, map_uid;
+ int ret = 0;
- /* get a list of all map_uids in this mailbox */
- i_array_init(&map_uids, 128);
+ map_trans = dbox_map_transaction_begin(mbox->storage->map, FALSE);
hdr = mail_index_get_header(mbox->box.view);
for (seq = 1; seq <= hdr->messages_count; seq++) {
- mail_index_lookup_ext(mbox->box.view, seq, mbox->ext_id,
- &data, &expunged);
- dbox_rec = data;
- if (dbox_rec == NULL) {
- /* no multi-mails */
+ if (mdbox_mail_lookup(mbox, mbox->box.view, seq,
+ &map_uid) < 0) {
+ ret = -1;
break;
}
- if (dbox_rec->map_uid != 0)
- array_append(&map_uids, &dbox_rec->map_uid, 1);
+
+ if (dbox_map_update_refcount(map_trans, map_uid, -1) < 0) {
+ ret = -1;
+ break;
+ }
}
- /* unreference the map_uids */
- map_trans = dbox_map_transaction_begin(mbox->storage->map, FALSE);
- ret = dbox_map_update_refcounts(map_trans, &map_uids, -1);
if (ret == 0)
ret = dbox_map_transaction_commit(map_trans);
dbox_map_transaction_free(&map_trans);
- array_free(&map_uids);
return ret;
}
diff -r 0eb423848dc1 -r c2c2c1dd252e src/lib-storage/index/dbox-multi/mdbox-sync.c
--- a/src/lib-storage/index/dbox-multi/mdbox-sync.c Mon Apr 19 16:14:37 2010 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c Mon Apr 19 16:15:57 2010 +0300
@@ -55,12 +55,13 @@
if (mdbox_mail_lookup(ctx->mbox, ctx->sync_view, seq, &map_uid) < 0)
return -1;
+ if (dbox_map_update_refcount(ctx->map_trans, map_uid, -1) < 0)
+ return -1;
seq_range_array_add(&ctx->expunged_seqs, 0, seq);
- array_append(&ctx->expunged_map_uids, &map_uid, 1);
return 0;
}
-static int mdbox_sync_add(struct mdbox_sync_context *ctx,
+static int mdbox_sync_rec(struct mdbox_sync_context *ctx,
const struct mail_index_sync_rec *sync_rec)
{
uint32_t seq, seq1, seq2;
@@ -119,30 +120,6 @@
return 0;
}
-static int mdbox_sync_index_finish_expunges(struct mdbox_sync_context *ctx)
-{
- struct dbox_map_transaction_context *map_trans;
- int ret;
-
- map_trans = dbox_map_transaction_begin(ctx->mbox->storage->map, FALSE);
- ret = dbox_map_update_refcounts(map_trans, &ctx->expunged_map_uids, -1);
- if (ret == 0) {
- /* write refcount changes to map index */
- ret = dbox_map_transaction_commit(map_trans);
- if (ret == 0) {
- /* write changes to mailbox index */
- ret = dbox_sync_mark_expunges(ctx);
- }
- }
-
- /* this finally finishes the map sync and makes it clear that the
- map transaction was successfully finished. */
- if (ret < 0)
- dbox_map_transaction_set_failed(map_trans);
- dbox_map_transaction_free(&map_trans);
- return ret;
-}
-
static int mdbox_sync_index(struct mdbox_sync_context *ctx)
{
struct mailbox *box = &ctx->mbox->box;
@@ -164,21 +141,33 @@
seq1, seq2);
}
- /* read all changes and group changes to same file_id together */
+ ctx->map_trans =
+ dbox_map_transaction_begin(ctx->mbox->storage->map, FALSE);
i_array_init(&ctx->expunged_seqs, 64);
- i_array_init(&ctx->expunged_map_uids, 64);
while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) {
- if ((ret = mdbox_sync_add(ctx, &sync_rec)) < 0)
+ if ((ret = mdbox_sync_rec(ctx, &sync_rec)) < 0)
break;
}
- if (ret == 0 && array_count(&ctx->expunged_seqs) > 0)
- ret = mdbox_sync_index_finish_expunges(ctx);
+
+ if (ret == 0) {
+ /* write refcount changes to map index */
+ ret = dbox_map_transaction_commit(ctx->map_trans);
+ }
+ if (ret == 0) {
+ /* write changes to mailbox index */
+ ret = dbox_sync_mark_expunges(ctx);
+ }
+
+ /* this finally finishes the map sync and marks the map transaction
+ to be successfully finished. */
+ if (ret < 0)
+ dbox_map_transaction_set_failed(ctx->map_trans);
+ dbox_map_transaction_free(&ctx->map_trans);
if (box->v.sync_notify != NULL)
box->v.sync_notify(box, 0, 0);
array_free(&ctx->expunged_seqs);
- array_free(&ctx->expunged_map_uids);
return ret == 0 ? 1 :
(ctx->mbox->storage->storage.files_corrupted ? 0 : -1);
}
diff -r 0eb423848dc1 -r c2c2c1dd252e src/lib-storage/index/dbox-multi/mdbox-sync.h
--- a/src/lib-storage/index/dbox-multi/mdbox-sync.h Mon Apr 19 16:14:37 2010 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-sync.h Mon Apr 19 16:15:57 2010 +0300
@@ -16,12 +16,10 @@
struct mail_index_sync_ctx *index_sync_ctx;
struct mail_index_view *sync_view;
struct mail_index_transaction *trans;
+ struct dbox_map_transaction_context *map_trans;
enum mdbox_sync_flags flags;
ARRAY_TYPE(seq_range) expunged_seqs;
- /* list of expunged map_uids. the same map_uid may be listed more than
- once in case message has been copied multiple times to mailbox. */
- ARRAY_TYPE(uint32_t) expunged_map_uids;
};
int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags,
More information about the dovecot-cvs
mailing list