dovecot-1.2: Mailbox view sync: Figure out the changes by compar...
dovecot at dovecot.org
dovecot at dovecot.org
Tue Jun 17 15:31:11 EEST 2008
details: http://hg.dovecot.org/dovecot-1.2/rev/bc5eca410ed3
changeset: 7883:bc5eca410ed3
user: Timo Sirainen <tss at iki.fi>
date: Tue Jun 17 15:30:54 2008 +0300
description:
Mailbox view sync: Figure out the changes by comparing old and new maps if
any of the required transaction logs are missing.
diffstat:
3 files changed, 322 insertions(+), 72 deletions(-)
TODO | 1
src/lib-index/mail-index-sync-update.c | 4
src/lib-index/mail-index-view-sync.c | 389 ++++++++++++++++++++++++++------
diffs (truncated from 571 to 300 lines):
diff -r ab08272053c5 -r bc5eca410ed3 TODO
--- a/TODO Tue Jun 17 14:00:28 2008 +0300
+++ b/TODO Tue Jun 17 15:30:54 2008 +0300
@@ -120,7 +120,6 @@
selected in read-only state
- index
- - if log file is lost, generate it from old and new index
- read-only support for mailboxes where we don't have write-access
- index file format changes:
- pack UIDs to beginning of file with UID ranges
diff -r ab08272053c5 -r bc5eca410ed3 src/lib-index/mail-index-sync-update.c
--- a/src/lib-index/mail-index-sync-update.c Tue Jun 17 14:00:28 2008 +0300
+++ b/src/lib-index/mail-index-sync-update.c Tue Jun 17 15:30:54 2008 +0300
@@ -23,6 +23,10 @@ mail_index_sync_update_log_offset(struct
mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
&prev_seq, &prev_offset);
+ if (prev_seq == 0) {
+ /* handling lost changes in view syncing */
+ return;
+ }
if (!eol) {
if (prev_offset == ctx->ext_intro_end_offset &&
diff -r ab08272053c5 -r bc5eca410ed3 src/lib-index/mail-index-view-sync.c
--- a/src/lib-index/mail-index-view-sync.c Tue Jun 17 14:00:28 2008 +0300
+++ b/src/lib-index/mail-index-view-sync.c Tue Jun 17 15:30:54 2008 +0300
@@ -8,6 +8,8 @@
#include "mail-index-modseq.h"
#include "mail-transaction-log.h"
+#include <stdlib.h>
+
struct mail_index_view_sync_ctx {
struct mail_index_view *view;
enum mail_index_view_sync_flags flags;
@@ -21,12 +23,18 @@ struct mail_index_view_sync_ctx {
const struct mail_transaction_header *hdr;
const void *data;
+
+ ARRAY_TYPE(keyword_indexes) lost_old_kw, lost_new_kw;
+ buffer_t *lost_kw_buf;
+ ARRAY_TYPE(seq_range) lost_flags;
+ unsigned int lost_flag_idx;
size_t data_offset;
unsigned int failed:1;
unsigned int sync_map_update:1;
unsigned int skipped_expunges:1;
unsigned int last_read:1;
+ unsigned int log_was_lost:1;
unsigned int hidden:1;
};
@@ -104,10 +112,10 @@ view_sync_set_log_view_range(struct mail
uoff_t start_offset, end_offset;
int ret;
+ start_seq = view->log_file_expunge_seq;
+ start_offset = view->log_file_expunge_offset;
end_seq = hdr->log_file_seq;
end_offset = hdr->log_file_head_offset;
- start_seq = view->log_file_expunge_seq;
- start_offset = view->log_file_expunge_offset;
for (;;) {
/* the view begins from the first non-synced transaction */
@@ -115,23 +123,14 @@ view_sync_set_log_view_range(struct mail
start_seq, start_offset,
end_seq, end_offset,
reset_r);
- if (ret <= 0) {
- if (ret < 0)
- return -1;
-
- /* FIXME: use the new index to get needed
- changes */
- mail_index_set_error(view->index,
- "Transaction log got desynced for index %s",
- view->index->filepath);
- view->inconsistent = TRUE;
- return -1;
- }
+ if (ret <= 0)
+ return ret;
if (!*reset_r || sync_expunges)
break;
- /* we can't do this. sync only up to reset. */
+ /* log was reset, but we don't want to sync expunges.
+ we can't do this, so sync only up to the reset. */
mail_transaction_log_view_get_prev_pos(view->log_view,
&end_seq, &end_offset);
end_seq--; end_offset = (uoff_t)-1;
@@ -142,26 +141,49 @@ view_sync_set_log_view_range(struct mail
break;
}
}
-
- return 0;
-}
-
-static int
-view_sync_get_expunges(struct mail_index_view *view,
- ARRAY_TYPE(seq_range) *expunges_r,
- unsigned int *expunge_count_r)
-{
- const struct mail_transaction_header *hdr;
+ return 1;
+}
+
+static unsigned int
+view_sync_expunges2seqs(struct mail_index_view_sync_ctx *ctx)
+{
+ struct mail_index_view *view = ctx->view;
struct seq_range *src, *src_end, *dest;
- const void *data;
unsigned int count, expunge_count = 0;
uint32_t prev_seq = 0;
+
+ /* convert UIDs to sequences */
+ src = dest = array_get_modifiable(&ctx->expunges, &count);
+ src_end = src + count;
+ for (; src != src_end; src++) {
+ if (!mail_index_lookup_seq_range(view, src->seq1, src->seq2,
+ &dest->seq1, &dest->seq2))
+ count--;
+ else {
+ i_assert(dest->seq1 > prev_seq);
+ prev_seq = dest->seq2;
+
+ expunge_count += dest->seq2 - dest->seq1 + 1;
+ dest++;
+ }
+ }
+ array_delete(&ctx->expunges, count,
+ array_count(&ctx->expunges) - count);
+ return expunge_count;
+}
+
+static int
+view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
+ unsigned int *expunge_count_r)
+{
+ struct mail_index_view *view = ctx->view;
+ const struct mail_transaction_header *hdr;
+ const void *data;
int ret;
/* get a list of expunge transactions. there may be some that we have
already synced, but it doesn't matter because they'll get dropped
out when converting to sequences */
- i_array_init(expunges_r, 64);
mail_transaction_log_view_mark(view->log_view);
while ((ret = mail_transaction_log_view_next(view->log_view,
&hdr, &data)) > 0) {
@@ -172,38 +194,16 @@ view_sync_get_expunges(struct mail_index
continue;
}
- if (mail_transaction_log_sort_expunges(expunges_r, data,
+ if (mail_transaction_log_sort_expunges(&ctx->expunges, data,
hdr->size) < 0) {
mail_transaction_log_view_set_corrupted(view->log_view,
"Corrupted expunge record");
- ret = -1;
- break;
+ return -1;
}
}
mail_transaction_log_view_rewind(view->log_view);
- if (ret < 0) {
- array_free(expunges_r);
- return -1;
- }
-
- /* convert UIDs to sequences */
- src = dest = array_get_modifiable(expunges_r, &count);
- src_end = src + count;
- for (; src != src_end; src++) {
- if (!mail_index_lookup_seq_range(view, src->seq1, src->seq2,
- &dest->seq1, &dest->seq2))
- count--;
- else {
- i_assert(dest->seq1 > prev_seq);
- prev_seq = dest->seq2;
-
- expunge_count += dest->seq2 - dest->seq1 + 1;
- dest++;
- }
- }
- array_delete(expunges_r, count, array_count(expunges_r) - count);
- *expunge_count_r = expunge_count;
+ *expunge_count_r = view_sync_expunges2seqs(ctx);
return 0;
}
@@ -254,14 +254,222 @@ static bool view_sync_have_expunges(stru
return ret < 0 || have_expunges;
}
+static int uint_cmp(const void *p1, const void *p2)
+{
+ const unsigned int *u1 = p1, *u2 = p2;
+
+ if (*u1 < *u2)
+ return -1;
+ if (*u1 > *u2)
+ return 1;
+ return 0;
+}
+
+static bool view_sync_lost_keywords_equal(struct mail_index_view_sync_ctx *ctx)
+{
+ unsigned int *old_idx, *new_idx;
+ unsigned int old_count, new_count;
+
+ old_idx = array_get_modifiable(&ctx->lost_old_kw, &old_count);
+ new_idx = array_get_modifiable(&ctx->lost_new_kw, &new_count);
+ if (old_count != new_count)
+ return FALSE;
+
+ qsort(old_idx, old_count, sizeof(*old_idx), uint_cmp);
+ qsort(new_idx, new_count, sizeof(*new_idx), uint_cmp);
+ return memcmp(old_idx, new_idx, old_count * sizeof(old_idx)) == 0;
+}
+
+static int view_sync_update_keywords(struct mail_index_view_sync_ctx *ctx,
+ uint32_t uid)
+{
+ struct mail_transaction_header thdr;
+ struct mail_transaction_keyword_update kw_up;
+ const unsigned int *kw_idx;
+ const char *const *kw_names;
+ unsigned int i, count;
+
+ kw_idx = array_get(&ctx->lost_new_kw, &count);
+ if (count == 0)
+ return 0;
+ kw_names = array_idx(&ctx->view->index->keywords, 0);
+
+ memset(&thdr, 0, sizeof(thdr));
+ thdr.type = MAIL_TRANSACTION_KEYWORD_UPDATE | MAIL_TRANSACTION_EXTERNAL;
+ memset(&kw_up, 0, sizeof(kw_up));
+ kw_up.modify_type = MODIFY_ADD;
+ /* add new flags one by one */
+ for (i = 0; i < count; i++) {
+ kw_up.name_size = strlen(kw_names[kw_idx[i]]);
+ buffer_set_used_size(ctx->lost_kw_buf, 0);
+ buffer_append(ctx->lost_kw_buf, &kw_up, sizeof(kw_up));
+ buffer_append(ctx->lost_kw_buf, kw_names[kw_idx[i]],
+ kw_up.name_size);
+ if (ctx->lost_kw_buf->used % 4 != 0) {
+ buffer_append_zero(ctx->lost_kw_buf,
+ 4 - ctx->lost_kw_buf->used % 4);
+ }
+ buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid));
+ buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid));
+
+ thdr.size = ctx->lost_kw_buf->used;
+ if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
+ ctx->lost_kw_buf->data) < 0)
+ return -1;
+ }
+ return 0;
+}
+
+static int view_sync_apply_lost_changes(struct mail_index_view_sync_ctx *ctx,
+ uint32_t old_seq, uint32_t new_seq)
+{
+ struct mail_index_map *old_map = ctx->view->map;
+ struct mail_index_map *new_map = ctx->view->index->map;
+ const struct mail_index_record *old_rec, *new_rec;
+ struct mail_transaction_header thdr;
+ bool changed = FALSE;
+
+ old_rec = MAIL_INDEX_MAP_IDX(old_map, old_seq - 1);
+ new_rec = MAIL_INDEX_MAP_IDX(new_map, new_seq - 1);
+ memset(&thdr, 0, sizeof(thdr));
+ if (old_rec->flags != new_rec->flags) {
+ struct mail_transaction_flag_update flag_update;
+
+ /* check this before syncing the record, since it updates
+ old_rec. */
+ if ((old_rec->flags & MAIL_INDEX_FLAGS_MASK) !=
+ (new_rec->flags & MAIL_INDEX_FLAGS_MASK))
+ changed = TRUE;
+
+ thdr.type = MAIL_TRANSACTION_FLAG_UPDATE |
+ MAIL_TRANSACTION_EXTERNAL;
+ thdr.size = sizeof(flag_update);
+
+ memset(&flag_update, 0, sizeof(flag_update));
+ flag_update.uid1 = flag_update.uid2 = new_rec->uid;
+ flag_update.add_flags = new_rec->flags;
+ flag_update.remove_flags = ~new_rec->flags & 0xff;
More information about the dovecot-cvs
mailing list