dovecot-2.0: Keep track of expunged messages' GUIDs and expose t...
dovecot at dovecot.org
dovecot at dovecot.org
Tue Jul 14 05:24:33 EEST 2009
details: http://hg.dovecot.org/dovecot-2.0/rev/2558ba736207
changeset: 9624:2558ba736207
user: Timo Sirainen <tss at iki.fi>
date: Mon Jul 13 22:24:27 2009 -0400
description:
Keep track of expunged messages' GUIDs and expose them via mailbox_get_expunges().
The message GUIDs are stored in expunge records to transaction log. Before
doing the final expunge, Maildir and dbox verify that the GUID in expunge
request matches the current actual GUID. The GUID is stored in 128 bit
field. If the real GUID isn't 128 bit, the bits are taken from SHA1 of the
GUID.
diffstat:
40 files changed, 605 insertions(+), 228 deletions(-)
src/imap/imap-fetch.c | 29 +++++-
src/lib-index/mail-index-sync-update.c | 76 +++++++++--------
src/lib-index/mail-index-sync.c | 26 +++++
src/lib-index/mail-index-transaction-export.c | 2
src/lib-index/mail-index-transaction-finish.c | 48 ++++++++++
src/lib-index/mail-index-transaction-private.h | 8 +
src/lib-index/mail-index-transaction-update.c | 27 +++++-
src/lib-index/mail-index-transaction-view.c | 3
src/lib-index/mail-index-transaction.c | 13 ++
src/lib-index/mail-index-view-sync.c | 88 ++++++++++++++------
src/lib-index/mail-index.h | 6 +
src/lib-index/mail-transaction-log-append.c | 3
src/lib-index/mail-transaction-log-file.c | 1
src/lib-index/mail-transaction-log-view.c | 20 ++++
src/lib-index/mail-transaction-log.h | 10 +-
src/lib-mail/mail-types.h | 2
src/lib-storage/index/dbox/dbox-file-fix.c | 2
src/lib-storage/index/dbox/dbox-file.c | 21 ----
src/lib-storage/index/dbox/dbox-file.h | 1
src/lib-storage/index/dbox/dbox-map.c | 9 +-
src/lib-storage/index/dbox/dbox-save.c | 15 ---
src/lib-storage/index/dbox/dbox-storage-rebuild.c | 15 +--
src/lib-storage/index/dbox/dbox-storage.c | 2
src/lib-storage/index/dbox/dbox-storage.h | 1
src/lib-storage/index/dbox/dbox-sync-file.c | 74 +++++++++++++---
src/lib-storage/index/dbox/dbox-sync.c | 17 ++-
src/lib-storage/index/dbox/dbox-sync.h | 11 ++
src/lib-storage/index/index-fetch.c | 80 ++++++++++++------
src/lib-storage/index/index-mail.c | 12 ++
src/lib-storage/index/index-storage.h | 6 -
src/lib-storage/index/index-sync-changes.c | 19 +++-
src/lib-storage/index/index-sync-changes.h | 3
src/lib-storage/index/maildir/maildir-sync-index.c | 63 +++++++++++++-
src/lib-storage/index/mbox/mbox-sync.c | 5 -
src/lib-storage/mail-storage-private.h | 11 +-
src/lib-storage/mail-storage.c | 32 ++++---
src/lib-storage/mail-storage.h | 24 +++--
src/lib-storage/mail.c | 27 +++++-
src/lib-storage/test-mailbox.c | 6 -
src/util/logview.c | 15 +++
diffs (truncated from 1713 to 300 lines):
diff -r 3da42dafa798 -r 2558ba736207 src/imap/imap-fetch.c
--- a/src/imap/imap-fetch.c Mon Jul 13 21:11:05 2009 -0400
+++ b/src/imap/imap-fetch.c Mon Jul 13 22:24:27 2009 -0400
@@ -255,6 +255,18 @@ static int get_expunges_fallback(struct
return ret;
}
+static void
+mailbox_expunge_to_range(const ARRAY_TYPE(mailbox_expunge_rec) *input,
+ ARRAY_TYPE(seq_range) *output)
+{
+ const struct mailbox_expunge_rec *expunges;
+ unsigned int i, count;
+
+ expunges = array_get(input, &count);
+ for (i = 0; i < count; i++)
+ seq_range_array_add(output, 0, expunges[i].uid);
+}
+
static int
imap_fetch_send_vanished(struct imap_fetch_context *ctx)
{
@@ -262,27 +274,32 @@ imap_fetch_send_vanished(struct imap_fet
const struct mail_search_arg *modseqarg = uidarg->next;
const ARRAY_TYPE(seq_range) *uids = &uidarg->value.seqset;
uint64_t modseq = modseqarg->value.modseq->modseq;
- ARRAY_TYPE(seq_range) expunges;
+ ARRAY_TYPE(mailbox_expunge_rec) expunges;
+ ARRAY_TYPE(seq_range) expunges_range;
string_t *str;
int ret = 0;
i_array_init(&expunges, array_count(uids));
- if (!mailbox_get_expunged_uids(ctx->box, modseq, uids, &expunges)) {
+ i_array_init(&expunges_range, array_count(uids));
+ if (mailbox_get_expunges(ctx->box, modseq, uids, &expunges))
+ mailbox_expunge_to_range(&expunges, &expunges_range);
+ else {
/* return all expunged UIDs */
- if (get_expunges_fallback(ctx, uids, &expunges) < 0) {
- array_clear(&expunges);
+ if (get_expunges_fallback(ctx, uids, &expunges_range) < 0) {
+ array_clear(&expunges_range);
ret = -1;
}
}
- if (array_count(&expunges) > 0) {
+ if (array_count(&expunges_range) > 0) {
str = str_new(default_pool, 128);
str_append(str, "* VANISHED (EARLIER) ");
- imap_write_seq_range(str, &expunges);
+ imap_write_seq_range(str, &expunges_range);
str_append(str, "\r\n");
o_stream_send(ctx->client->output, str_data(str), str_len(str));
str_free(&str);
}
array_free(&expunges);
+ array_free(&expunges_range);
return ret;
}
diff -r 3da42dafa798 -r 2558ba736207 src/lib-index/mail-index-sync-update.c
--- a/src/lib-index/mail-index-sync-update.c Mon Jul 13 21:11:05 2009 -0400
+++ b/src/lib-index/mail-index-sync-update.c Mon Jul 13 22:24:27 2009 -0400
@@ -236,44 +236,36 @@ sync_expunge_call_handlers(struct mail_i
}
}
-static int
-sync_expunge(const struct mail_transaction_expunge *e, unsigned int count,
- struct mail_index_sync_map_ctx *ctx)
+static void
+sync_expunge(struct mail_index_sync_map_ctx *ctx, uint32_t uid1, uint32_t uid2)
{
struct mail_index_map *map = ctx->view->map;
struct mail_index_record *rec;
uint32_t seq_count, seq, seq1, seq2;
- unsigned int i;
-
- for (i = 0; i < count; i++, e++) {
- if (!mail_index_lookup_seq_range(ctx->view, e->uid1, e->uid2,
- &seq1, &seq2)) {
- /* everything expunged already */
- continue;
- }
-
- sync_expunge_call_handlers(ctx, seq1, seq2);
-
- map = mail_index_sync_get_atomic_map(ctx);
- for (seq = seq1; seq <= seq2; seq++) {
- rec = MAIL_INDEX_MAP_IDX(map, seq-1);
- mail_index_sync_header_update_counts(ctx, rec->uid,
- rec->flags, 0,
- FALSE);
- }
-
- /* @UNSAFE */
- memmove(MAIL_INDEX_MAP_IDX(map, seq1-1),
- MAIL_INDEX_MAP_IDX(map, seq2),
- (map->rec_map->records_count - seq2) *
- map->hdr.record_size);
-
- seq_count = seq2 - seq1 + 1;
- map->rec_map->records_count -= seq_count;
- map->hdr.messages_count -= seq_count;
- mail_index_modseq_expunge(ctx->modseq_ctx, seq1, seq2);
- }
- return 1;
+
+ if (!mail_index_lookup_seq_range(ctx->view, uid1, uid2, &seq1, &seq2)) {
+ /* everything expunged already */
+ return;
+ }
+
+ sync_expunge_call_handlers(ctx, seq1, seq2);
+
+ map = mail_index_sync_get_atomic_map(ctx);
+ for (seq = seq1; seq <= seq2; seq++) {
+ rec = MAIL_INDEX_MAP_IDX(map, seq-1);
+ mail_index_sync_header_update_counts(ctx, rec->uid, rec->flags,
+ 0, FALSE);
+ }
+
+ /* @UNSAFE */
+ memmove(MAIL_INDEX_MAP_IDX(map, seq1-1),
+ MAIL_INDEX_MAP_IDX(map, seq2),
+ (map->rec_map->records_count - seq2) * map->hdr.record_size);
+
+ seq_count = seq2 - seq1 + 1;
+ map->rec_map->records_count -= seq_count;
+ map->hdr.messages_count -= seq_count;
+ mail_index_modseq_expunge(ctx->modseq_ctx, seq1, seq2);
}
void mail_index_sync_write_seq_update(struct mail_index_sync_map_ctx *ctx,
@@ -484,7 +476,21 @@ int mail_index_sync_record(struct mail_i
break;
}
end = CONST_PTR_OFFSET(data, hdr->size);
- ret = sync_expunge(rec, end - rec, ctx);
+ for (; rec != end; rec++)
+ sync_expunge(ctx, rec->uid1, rec->uid2);
+ break;
+ }
+ case MAIL_TRANSACTION_EXPUNGE_GUID:
+ case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT: {
+ const struct mail_transaction_expunge_guid *rec = data, *end;
+
+ if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
+ /* this is simply a request for expunge */
+ break;
+ }
+ end = CONST_PTR_OFFSET(data, hdr->size);
+ for (; rec != end; rec++)
+ sync_expunge(ctx, rec->uid, rec->uid);
break;
}
case MAIL_TRANSACTION_FLAG_UPDATE: {
diff -r 3da42dafa798 -r 2558ba736207 src/lib-index/mail-index-sync.c
--- a/src/lib-index/mail-index-sync.c Mon Jul 13 21:11:05 2009 -0400
+++ b/src/lib-index/mail-index-sync.c Mon Jul 13 22:24:27 2009 -0400
@@ -41,6 +41,17 @@ static void mail_index_sync_add_expunge(
}
}
+static void mail_index_sync_add_expunge_guid(struct mail_index_sync_ctx *ctx)
+{
+ const struct mail_transaction_expunge_guid *e = ctx->data;
+ size_t i, size = ctx->hdr->size / sizeof(*e);
+
+ for (i = 0; i < size; i++) {
+ mail_index_expunge_guid(ctx->sync_trans, e[i].uid,
+ e[i].guid_128);
+ }
+}
+
static void mail_index_sync_add_flag_update(struct mail_index_sync_ctx *ctx)
{
const struct mail_transaction_flag_update *u = ctx->data;
@@ -128,6 +139,9 @@ static bool mail_index_sync_add_transact
switch (ctx->hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_EXPUNGE:
mail_index_sync_add_expunge(ctx);
+ break;
+ case MAIL_TRANSACTION_EXPUNGE_GUID:
+ mail_index_sync_add_expunge_guid(ctx);
break;
case MAIL_TRANSACTION_FLAG_UPDATE:
mail_index_sync_add_flag_update(ctx);
@@ -512,6 +526,7 @@ static bool mail_index_sync_view_have_an
to be synced, but cache syncing relies on tail
offsets being updated. */
case MAIL_TRANSACTION_EXPUNGE:
+ case MAIL_TRANSACTION_EXPUNGE_GUID:
case MAIL_TRANSACTION_FLAG_UPDATE:
case MAIL_TRANSACTION_KEYWORD_UPDATE:
case MAIL_TRANSACTION_KEYWORD_RESET:
@@ -550,11 +565,12 @@ void mail_index_sync_get_offsets(struct
static void
mail_index_sync_get_expunge(struct mail_index_sync_rec *rec,
- const struct mail_transaction_expunge *exp)
+ const struct mail_transaction_expunge_guid *exp)
{
rec->type = MAIL_INDEX_SYNC_TYPE_EXPUNGE;
- rec->uid1 = exp->uid1;
- rec->uid2 = exp->uid2;
+ rec->uid1 = exp->uid;
+ rec->uid2 = exp->uid;
+ memcpy(rec->guid_128, exp->guid_128, sizeof(rec->guid_128));
}
static void
@@ -605,6 +621,8 @@ bool mail_index_sync_next(struct mail_in
/* FIXME: replace with a priority queue so we don't have to go
through the whole list constantly. and remember to make sure that
keyword resets are sent before adds! */
+ /* FIXME: pretty ugly to do this for expunges, which isn't even a
+ seq_range. */
sync_list = array_get_modifiable(&ctx->sync_list, &count);
for (i = 0; i < count; i++) {
if (!array_is_created(sync_list[i].array) ||
@@ -641,7 +659,7 @@ bool mail_index_sync_next(struct mail_in
if (sync_list[i].array == (void *)&sync_trans->expunges) {
mail_index_sync_get_expunge(sync_rec,
- (const struct mail_transaction_expunge *)uid_range);
+ (const struct mail_transaction_expunge_guid *)uid_range);
} else if (sync_list[i].array == (void *)&sync_trans->updates) {
mail_index_sync_get_update(sync_rec,
(const struct mail_transaction_flag_update *)uid_range);
diff -r 3da42dafa798 -r 2558ba736207 src/lib-index/mail-index-transaction-export.c
--- a/src/lib-index/mail-index-transaction-export.c Mon Jul 13 21:11:05 2009 -0400
+++ b/src/lib-index/mail-index-transaction-export.c Mon Jul 13 22:24:27 2009 -0400
@@ -380,7 +380,7 @@ void mail_index_transaction_export(struc
if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
change_mask |= MAIL_INDEX_SYNC_TYPE_EXPUNGE;
log_append_buffer(&ctx, t->expunges.arr.buffer,
- MAIL_TRANSACTION_EXPUNGE);
+ MAIL_TRANSACTION_EXPUNGE_GUID);
}
if (t->post_hdr_changed) {
diff -r 3da42dafa798 -r 2558ba736207 src/lib-index/mail-index-transaction-finish.c
--- a/src/lib-index/mail-index-transaction-finish.c Mon Jul 13 21:11:05 2009 -0400
+++ b/src/lib-index/mail-index-transaction-finish.c Mon Jul 13 22:24:27 2009 -0400
@@ -5,6 +5,26 @@
#include "mail-index-private.h"
#include "mail-index-modseq.h"
#include "mail-index-transaction-private.h"
+
+int mail_transaction_expunge_guid_cmp(const struct mail_transaction_expunge_guid *e1,
+ const struct mail_transaction_expunge_guid *e2)
+{
+ if (e1->uid < e2->uid)
+ return -1;
+ else if (e1->uid > e2->uid)
+ return 1;
+ else
+ return 0;
+}
+
+void mail_index_transaction_sort_expunges(struct mail_index_transaction *t)
+{
+ if (!t->expunges_nonsorted)
+ return;
+
+ array_sort(&t->expunges, mail_transaction_expunge_guid_cmp);
+ t->expunges_nonsorted = FALSE;
+}
static void
ext_reset_update_atomic(struct mail_index_transaction *t,
@@ -312,6 +332,31 @@ static void keyword_updates_convert_to_u
}
}
+static void expunges_convert_to_uids(struct mail_index_transaction *t)
+{
+ struct mail_transaction_expunge_guid *expunges;
+ unsigned int src, dest, count;
+
+ if (!array_is_created(&t->expunges))
+ return;
+
+ mail_index_transaction_sort_expunges(t);
+
+ expunges = array_get_modifiable(&t->expunges, &count);
+ if (count == 0)
+ return;
+
+ /* convert uids and drop duplicates */
+ expunges[0].uid = mail_index_transaction_get_uid(t, expunges[0].uid);
+ for (src = dest = 1; src < count; src++) {
+ expunges[dest].uid =
+ mail_index_transaction_get_uid(t, expunges[src].uid);
+ if (expunges[dest-1].uid != expunges[dest].uid)
+ dest++;
+ }
+ array_delete(&t->expunges, dest, count-dest);
More information about the dovecot-cvs
mailing list