diff -r 5bbdfee113a5 src/doveadm/doveadm-dump-log.c --- a/src/doveadm/doveadm-dump-log.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/doveadm/doveadm-dump-log.c Mon Jul 16 17:00:56 2012 +0300 @@ -282,8 +282,8 @@ const struct mail_transaction_flag_update *u = data; for (; size > 0; size -= sizeof(*u), u++) { - printf(" - uids=%u-%u (flags +%x-%x)\n", - u->uid1, u->uid2, u->add_flags, u->remove_flags); + printf(" - uids=%u-%u (flags +%x-%x, modseq_inc_flag=%d)\n", + u->uid1, u->uid2, u->add_flags, u->remove_flags, u->modseq_inc_flag); } break; } diff -r 5bbdfee113a5 src/lib-index/mail-index-private.h --- a/src/lib-index/mail-index-private.h Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-index/mail-index-private.h Mon Jul 16 17:00:56 2012 +0300 @@ -39,8 +39,8 @@ PTR_OFFSET((map)->rec_map->records, (idx) * (map)->hdr.record_size)) #define MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u) \ - ((((u)->add_flags | (u)->remove_flags) & \ - MAIL_INDEX_FLAGS_MASK) == 0) + ((((u)->add_flags | (u)->remove_flags) & MAIL_INDEX_FLAGS_MASK) == 0 && \ + (u)->modseq_inc_flag == 0) #define MAIL_INDEX_EXT_KEYWORDS "keywords" diff -r 5bbdfee113a5 src/lib-index/mail-index-transaction-export.c --- a/src/lib-index/mail-index-transaction-export.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-index/mail-index-transaction-export.c Mon Jul 16 17:00:56 2012 +0300 @@ -20,6 +20,34 @@ buf->data, buf->used); } +static void log_append_flag_updates(struct mail_index_export_context *ctx, + struct mail_index_transaction *t) +{ + ARRAY_DEFINE(log_updates, struct mail_transaction_flag_update); + const struct mail_index_flag_update *updates; + struct mail_transaction_flag_update *log_update; + unsigned int i, count; + + updates = array_get(&t->updates, &count); + if (count == 0) + return; + + i_array_init(&log_updates, count); + + for (i = 0; i < count; i++) { + log_update = array_append_space(&log_updates); + log_update->uid1 = updates[i].uid1; + log_update->uid2 = updates[i].uid2; + log_update->add_flags = updates[i].add_flags & 0xff; + log_update->remove_flags = updates[i].remove_flags & 0xff; + if ((updates[i].add_flags & MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ) != 0) + log_update->modseq_inc_flag = 1; + } + log_append_buffer(ctx, log_updates.arr.buffer, + MAIL_TRANSACTION_FLAG_UPDATE); + array_free(&log_updates); +} + static const buffer_t * log_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend) { @@ -373,8 +401,7 @@ if (array_is_created(&t->updates)) { change_mask |= MAIL_INDEX_SYNC_TYPE_FLAGS; - log_append_buffer(&ctx, t->updates.arr.buffer, - MAIL_TRANSACTION_FLAG_UPDATE); + log_append_flag_updates(&ctx, t); } if (array_is_created(&t->ext_rec_updates)) { diff -r 5bbdfee113a5 src/lib-index/mail-index-transaction-finish.c --- a/src/lib-index/mail-index-transaction-finish.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-index/mail-index-transaction-finish.c Mon Jul 16 17:00:56 2012 +0300 @@ -80,7 +80,7 @@ static unsigned int mail_transaction_drop_range(struct mail_index_transaction *t, - struct mail_transaction_flag_update update, + struct mail_index_flag_update update, unsigned int update_idx, ARRAY_TYPE(seq_range) *keeps) { @@ -109,7 +109,7 @@ static void mail_index_transaction_finish_flag_updates(struct mail_index_transaction *t) { - const struct mail_transaction_flag_update *updates, *u; + const struct mail_index_flag_update *updates, *u; const struct mail_index_record *rec; unsigned int i, count; ARRAY_TYPE(seq_range) keeps; diff -r 5bbdfee113a5 src/lib-index/mail-index-transaction-private.h --- a/src/lib-index/mail-index-transaction-private.h Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-index/mail-index-transaction-private.h Mon Jul 16 17:00:56 2012 +0300 @@ -29,6 +29,12 @@ struct mail_index_module_register *reg; }; +struct mail_index_flag_update { + uint32_t uid1, uid2; + uint16_t add_flags; + uint16_t remove_flags; +}; + struct mail_index_transaction { int refcount; @@ -47,7 +53,7 @@ ARRAY_DEFINE(modseq_updates, struct mail_transaction_modseq_update); ARRAY_DEFINE(expunges, struct mail_transaction_expunge_guid); - ARRAY_DEFINE(updates, struct mail_transaction_flag_update); + ARRAY_DEFINE(updates, struct mail_index_flag_update); size_t last_update_idx; unsigned char pre_hdr_change[sizeof(struct mail_index_header)]; diff -r 5bbdfee113a5 src/lib-index/mail-index-transaction-update.c --- a/src/lib-index/mail-index-transaction-update.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-index/mail-index-transaction-update.c Mon Jul 16 17:00:56 2012 +0300 @@ -393,7 +393,7 @@ unsigned int right_idx, uint32_t seq) { - const struct mail_transaction_flag_update *updates; + const struct mail_index_flag_update *updates; unsigned int idx, count; updates = array_get(&t->updates, &count); @@ -419,10 +419,10 @@ static void mail_index_insert_flag_update(struct mail_index_transaction *t, - struct mail_transaction_flag_update u, + struct mail_index_flag_update u, unsigned int idx) { - struct mail_transaction_flag_update *updates, tmp_update; + struct mail_index_flag_update *updates, tmp_update; unsigned int count, first_idx, max; updates = array_get_modifiable(&t->updates, &count); @@ -548,7 +548,7 @@ enum mail_flags flags) { struct mail_index_record *rec; - struct mail_transaction_flag_update u, *last_update; + struct mail_index_flag_update u, *last_update; unsigned int idx, first_idx, count; update_minmax_flagupdate_seq(t, seq1, seq2); @@ -1092,7 +1092,7 @@ bool mail_index_cancel_flag_updates(struct mail_index_transaction *t, uint32_t seq) { - struct mail_transaction_flag_update *updates, tmp_update; + struct mail_index_flag_update *updates, tmp_update; unsigned int i, count; if (!array_is_created(&t->updates)) diff -r 5bbdfee113a5 src/lib-index/mail-index-transaction-view.c --- a/src/lib-index/mail-index-transaction-view.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-index/mail-index-transaction-view.c Mon Jul 16 17:00:56 2012 +0300 @@ -85,7 +85,7 @@ const struct mail_index_record *rec, uint32_t seq) { struct mail_index_transaction *t = tview->t; - const struct mail_transaction_flag_update *updates; + const struct mail_index_flag_update *updates; struct mail_index_record *trec; unsigned int idx, count; diff -r 5bbdfee113a5 src/lib-index/mail-index.h --- a/src/lib-index/mail-index.h Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-index/mail-index.h Mon Jul 16 17:00:56 2012 +0300 @@ -42,9 +42,11 @@ enum mail_index_mail_flags { /* For private use by backend. Replacing flags doesn't change this. */ - MAIL_INDEX_MAIL_FLAG_BACKEND = 0x40, + MAIL_INDEX_MAIL_FLAG_BACKEND = 0x40, /* Message flags haven't been written to backend */ - MAIL_INDEX_MAIL_FLAG_DIRTY = 0x80 + MAIL_INDEX_MAIL_FLAG_DIRTY = 0x80, + /* Force updating this message's modseq via a flag update record */ + MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ = 0x100 }; #define MAIL_INDEX_FLAGS_MASK \ diff -r 5bbdfee113a5 src/lib-index/mail-transaction-log.h --- a/src/lib-index/mail-transaction-log.h Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-index/mail-transaction-log.h Mon Jul 16 17:00:56 2012 +0300 @@ -87,7 +87,8 @@ uint32_t uid1, uid2; uint8_t add_flags; uint8_t remove_flags; - uint16_t padding; + uint8_t modseq_inc_flag; + uint8_t padding; }; struct mail_transaction_keyword_update { diff -r 5bbdfee113a5 src/lib-index/test-mail-index-transaction-finish.c --- a/src/lib-index/test-mail-index-transaction-finish.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-index/test-mail-index-transaction-finish.c Mon Jul 16 17:00:56 2012 +0300 @@ -60,8 +60,8 @@ static void test_mail_index_transaction_finish_flag_updates(void) { struct mail_index_transaction *t; - const struct mail_transaction_flag_update *updates; - struct mail_transaction_flag_update u; + const struct mail_index_flag_update *updates; + struct mail_index_flag_update u; unsigned int count; t = t_new(struct mail_index_transaction, 1); diff -r 5bbdfee113a5 src/lib-index/test-mail-index-transaction-update.c --- a/src/lib-index/test-mail-index-transaction-update.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-index/test-mail-index-transaction-update.c Mon Jul 16 17:00:56 2012 +0300 @@ -131,7 +131,7 @@ static void test_mail_index_flag_update_fastpath(void) { struct mail_index_transaction *t; - const struct mail_transaction_flag_update *updates; + const struct mail_index_flag_update *updates; unsigned int count; hdr.messages_count = 20; @@ -160,12 +160,10 @@ test_assert(updates[0].add_flags == MAIL_DELETED); test_assert(updates[0].remove_flags == (MAIL_ANSWERED | MAIL_FLAGGED | MAIL_SEEN | MAIL_DRAFT)); - test_assert(updates[0].padding == 0); test_assert(updates[1].uid1 == 16); test_assert(updates[1].uid2 == 16); test_assert(updates[1].add_flags == MAIL_DELETED); test_assert(updates[1].remove_flags == 0); - test_assert(updates[1].padding == 0); test_assert(!t->log_updates); test_end(); } @@ -173,7 +171,7 @@ static void test_mail_index_flag_update_simple_merges(void) { struct mail_index_transaction *t; - const struct mail_transaction_flag_update *updates; + const struct mail_index_flag_update *updates; unsigned int count; hdr.messages_count = 20; @@ -216,7 +214,7 @@ static void test_mail_index_flag_update_complex_merges(void) { struct mail_index_transaction *t; - const struct mail_transaction_flag_update *updates; + const struct mail_index_flag_update *updates; unsigned int count; hdr.messages_count = 20; @@ -275,7 +273,7 @@ flags_array_check(struct mail_index_transaction *t, const enum mail_flags *flags, unsigned int msg_count) { - const struct mail_transaction_flag_update *updates; + const struct mail_index_flag_update *updates; unsigned int i, count, seq; if (array_is_created(&t->updates)) @@ -349,7 +347,7 @@ static void test_mail_index_cancel_flag_updates(void) { struct mail_index_transaction *t; - const struct mail_transaction_flag_update *updates; + const struct mail_index_flag_update *updates; unsigned int count; hdr.messages_count = 20; @@ -382,7 +380,7 @@ { struct mail_index_transaction *t; const struct mail_index_record *appends; - const struct mail_transaction_flag_update *updates; + const struct mail_index_flag_update *updates; unsigned int count; uint32_t seq; diff -r 5bbdfee113a5 src/lib-storage/index/Makefile.am --- a/src/lib-storage/index/Makefile.am Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/index/Makefile.am Mon Jul 16 17:00:56 2012 +0300 @@ -26,6 +26,7 @@ index-storage.c \ index-sync.c \ index-sync-changes.c \ + index-sync-pvt.c \ index-sync-search.c \ index-thread.c \ index-thread-finish.c \ diff -r 5bbdfee113a5 src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/index/index-mail.c Mon Jul 16 17:00:56 2012 +0300 @@ -138,18 +138,45 @@ size_r, sizeof(*size_r)); } -enum mail_flags index_mail_get_flags(struct mail *mail) +static bool index_mail_get_pvt(struct mail *_mail) { + struct mail_private *mail = (struct mail_private *)_mail; + + if (mail->seq_pvt != 0) + return TRUE; + if (_mail->box->view_pvt == NULL) { + /* no private view (set by view syncing) -> no private flags */ + return FALSE; + } + + index_transaction_init_pvt(_mail->transaction); + if (!mail_index_lookup_seq(_mail->transaction->view_pvt, _mail->uid, + &mail->seq_pvt)) + mail->seq_pvt = 0; + return mail->seq_pvt != 0; +} + +enum mail_flags index_mail_get_flags(struct mail *_mail) +{ + struct mail_private *mail = (struct mail_private *)_mail; const struct mail_index_record *rec; - enum mail_flags flags; + enum mail_flags flags, pvt_flags_mask; - rec = mail_index_lookup(mail->transaction->view, mail->seq); + rec = mail_index_lookup(_mail->transaction->view, _mail->seq); flags = rec->flags & (MAIL_FLAGS_NONRECENT | MAIL_INDEX_MAIL_FLAG_BACKEND); - if (index_mailbox_is_recent(mail->box, mail->uid)) + if (index_mailbox_is_recent(_mail->box, _mail->uid)) flags |= MAIL_RECENT; + if (index_mail_get_pvt(_mail)) { + /* mailbox has private flags */ + pvt_flags_mask = mailbox_get_private_flags_mask(_mail->box); + flags &= ~pvt_flags_mask; + rec = mail_index_lookup(_mail->transaction->view_pvt, + mail->seq_pvt); + flags |= rec->flags & pvt_flags_mask; + } return flags; } @@ -1223,6 +1250,7 @@ mail->mail.mail.seq = 0; mail->mail.mail.uid = 0; + mail->mail.seq_pvt = 0; mail->mail.mail.expunged = FALSE; mail->mail.mail.has_nuls = FALSE; mail->mail.mail.has_no_nuls = FALSE; @@ -1599,15 +1627,43 @@ } } -void index_mail_update_flags(struct mail *mail, enum modify_type modify_type, +void index_mail_update_flags(struct mail *_mail, enum modify_type modify_type, enum mail_flags flags) { + struct mail_private *mail = (struct mail_private *)_mail; + enum mail_flags pvt_flags_mask, pvt_flags = 0; + bool update_modseq = FALSE; + if ((flags & MAIL_RECENT) == 0 && - index_mailbox_is_recent(mail->box, mail->uid)) - index_mail_drop_recent_flag(mail); + index_mailbox_is_recent(_mail->box, _mail->uid)) + index_mail_drop_recent_flag(_mail); + flags &= MAIL_FLAGS_NONRECENT | MAIL_INDEX_MAIL_FLAG_BACKEND; - flags &= MAIL_FLAGS_NONRECENT | MAIL_INDEX_MAIL_FLAG_BACKEND; - mail_index_update_flags(mail->transaction->itrans, mail->seq, + if (_mail->box->view_pvt != NULL) { + /* mailbox has private flags */ + pvt_flags_mask = mailbox_get_private_flags_mask(_mail->box); + pvt_flags = flags & pvt_flags_mask; + flags &= ~pvt_flags_mask; + if (index_mail_get_pvt(_mail) && + (pvt_flags != 0 || modify_type == MODIFY_REPLACE)) { + mail_index_update_flags(_mail->transaction->itrans_pvt, + mail->seq_pvt, + modify_type, pvt_flags); + update_modseq = TRUE; + } + } + + if (!update_modseq) { + /* no forced modseq update */ + } else if (modify_type == MODIFY_REMOVE) { + /* add the modseq update separately */ + mail_index_update_flags(_mail->transaction->itrans, _mail->seq, + MODIFY_ADD, (enum mail_flags )MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ); + } else { + /* add as part of the flag updates */ + flags |= MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ; + } + mail_index_update_flags(_mail->transaction->itrans, _mail->seq, modify_type, flags); } diff -r 5bbdfee113a5 src/lib-storage/index/index-search-private.h --- a/src/lib-storage/index/index-search-private.h Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/index/index-search-private.h Mon Jul 16 17:00:56 2012 +0300 @@ -10,6 +10,8 @@ struct mail_index_view *view; struct mailbox *box; + uint32_t pvt_uid, pvt_seq; + enum mail_fetch_field extra_wanted_fields; struct mailbox_header_lookup_ctx *extra_wanted_headers; diff -r 5bbdfee113a5 src/lib-storage/index/index-search.c --- a/src/lib-storage/index/index-search.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/index/index-search.c Mon Jul 16 17:00:56 2012 +0300 @@ -147,12 +147,28 @@ return 1; } +static bool +index_search_get_pvt(struct index_search_context *ctx, uint32_t uid) +{ + if (ctx->box->view_pvt == NULL) { + /* no private view (set by view syncing) -> no private flags */ + return FALSE; + } + + if (ctx->pvt_uid == uid) + return ctx->pvt_seq != 0; + ctx->pvt_uid = uid; + ctx->pvt_seq = uid; + return mail_index_lookup_seq(ctx->mail_ctx.transaction->view_pvt, + uid, &ctx->pvt_seq); +} + /* Returns >0 = matched, 0 = not matched, -1 = unknown */ static int search_arg_match_index(struct index_search_context *ctx, struct mail_search_arg *arg, const struct mail_index_record *rec) { - enum mail_flags flags; + enum mail_flags flags, pvt_flags_mask; uint64_t modseq; int ret; @@ -167,6 +183,13 @@ if ((arg->value.flags & MAIL_RECENT) != 0 && index_mailbox_is_recent(ctx->box, rec->uid)) flags |= MAIL_RECENT; + if (index_search_get_pvt(ctx, rec->uid)) { + pvt_flags_mask = mailbox_get_private_flags_mask(ctx->box); + flags &= ~pvt_flags_mask; + rec = mail_index_lookup(ctx->mail_ctx.transaction->view_pvt, + ctx->pvt_seq); + flags |= rec->flags & pvt_flags_mask; + } return (flags & arg->value.flags) == arg->value.flags; case SEARCH_KEYWORDS: T_BEGIN { @@ -868,9 +891,21 @@ struct mail_search_arg *args, uint32_t *seq1, uint32_t *seq2) { - const struct mail_index_header *hdr; + const struct mail_index_header *hdr_seen, *hdr_del, *hdr_pvt; + enum mail_flags pvt_flags_mask; - hdr = mail_index_get_header(ctx->view); + hdr_seen = hdr_del = mail_index_get_header(ctx->view); + + if (ctx->box->view_pvt != NULL) { + pvt_flags_mask = mailbox_get_private_flags_mask(ctx->box); + index_transaction_init_pvt(ctx->mail_ctx.transaction); + hdr_pvt = mail_index_get_header(ctx->mail_ctx.transaction->view_pvt); + if ((pvt_flags_mask & MAIL_SEEN) != 0) + hdr_seen = hdr_pvt; + if ((pvt_flags_mask & MAIL_DELETED) != 0) + hdr_del = hdr_pvt; + } + for (; args != NULL; args = args->next) { if (args->type != SEARCH_FLAGS) { if (args->type == SEARCH_ALL) { @@ -881,10 +916,11 @@ } if ((args->value.flags & MAIL_SEEN) != 0) { /* SEEN with 0 seen? */ - if (!args->match_not && hdr->seen_messages_count == 0) + if (!args->match_not && hdr_seen->seen_messages_count == 0) return FALSE; - if (hdr->seen_messages_count == hdr->messages_count) { + if (hdr_seen->seen_messages_count == + hdr_seen->messages_count) { /* UNSEEN with all seen? */ if (args->match_not) return FALSE; @@ -894,17 +930,17 @@ } else if (args->match_not) { /* UNSEEN with lowwater limiting */ search_limit_lowwater(ctx, - hdr->first_unseen_uid_lowwater, seq1); + hdr_seen->first_unseen_uid_lowwater, seq1); } } if ((args->value.flags & MAIL_DELETED) != 0) { /* DELETED with 0 deleted? */ if (!args->match_not && - hdr->deleted_messages_count == 0) + hdr_del->deleted_messages_count == 0) return FALSE; - if (hdr->deleted_messages_count == - hdr->messages_count) { + if (hdr_del->deleted_messages_count == + hdr_del->messages_count) { /* UNDELETED with all deleted? */ if (args->match_not) return FALSE; @@ -914,7 +950,7 @@ } else if (!args->match_not) { /* DELETED with lowwater limiting */ search_limit_lowwater(ctx, - hdr->first_deleted_uid_lowwater, seq1); + hdr_del->first_deleted_uid_lowwater, seq1); } } } diff -r 5bbdfee113a5 src/lib-storage/index/index-status.c --- a/src/lib-storage/index/index-status.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/index/index-status.c Mon Jul 16 17:00:56 2012 +0300 @@ -33,7 +33,7 @@ enum mailbox_status_items items, struct mailbox_status *status_r) { - const struct mail_index_header *hdr; + const struct mail_index_header *hdr, *hdr_pvt; memset(status_r, 0, sizeof(struct mailbox_status)); @@ -46,6 +46,8 @@ /* we can get most of the status items without any trouble */ hdr = mail_index_get_header(box->view); + hdr_pvt = box->view_pvt == NULL ? NULL : + mail_index_get_header(box->view_pvt); status_r->messages = hdr->messages_count; if ((items & STATUS_RECENT) != 0) { /* make sure recent count is set, in case syncing hasn't @@ -54,7 +56,14 @@ status_r->recent = index_mailbox_get_recent_count(box); i_assert(status_r->recent <= status_r->messages); } - status_r->unseen = hdr->messages_count - hdr->seen_messages_count; + if (hdr_pvt == NULL || + (mailbox_get_private_flags_mask(box) & MAIL_SEEN) == 0) { + status_r->unseen = hdr->messages_count - + hdr->seen_messages_count; + } else { + status_r->unseen = hdr_pvt->messages_count - + hdr_pvt->seen_messages_count; + } status_r->uidvalidity = hdr->uid_validity; status_r->uidnext = hdr->next_uid; status_r->first_recent_uid = hdr->first_recent_uid; @@ -71,8 +80,14 @@ } if ((items & STATUS_FIRST_UNSEEN_SEQ) != 0) { - mail_index_lookup_first(box->view, 0, MAIL_SEEN, - &status_r->first_unseen_seq); + if (hdr_pvt == NULL || + (mailbox_get_private_flags_mask(box) & MAIL_SEEN) == 0) { + mail_index_lookup_first(box->view, 0, MAIL_SEEN, + &status_r->first_unseen_seq); + } else { + mail_index_lookup_first(box->view_pvt, 0, MAIL_SEEN, + &status_r->first_unseen_seq); + } } if ((items & STATUS_LAST_CACHED_SEQ) != 0) get_last_cached_seq(box, &status_r->last_cached_seq); diff -r 5bbdfee113a5 src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/index/index-storage.c Mon Jul 16 17:00:56 2012 +0300 @@ -332,6 +332,10 @@ if (box->input != NULL) i_stream_unref(&box->input); + if (box->view_pvt != NULL) + mail_index_view_close(&box->view_pvt); + if (box->index_pvt != NULL) + mail_index_close(box->index_pvt); mail_index_view_close(&box->view); mail_index_close(box->index); box->cache = NULL; @@ -349,6 +353,8 @@ void index_storage_mailbox_free(struct mailbox *box) { + if (box->index_pvt != NULL) + mail_index_alloc_cache_unref(&box->index_pvt); if (box->index != NULL) mail_index_alloc_cache_unref(&box->index); } diff -r 5bbdfee113a5 src/lib-storage/index/index-storage.h --- a/src/lib-storage/index/index-storage.h Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/index/index-storage.h Mon Jul 16 17:00:56 2012 +0300 @@ -129,6 +129,7 @@ void index_transaction_init(struct mailbox_transaction_context *t, struct mailbox *box, enum mailbox_transaction_flags flags); +void index_transaction_init_pvt(struct mailbox_transaction_context *t); int index_transaction_commit(struct mailbox_transaction_context *t, struct mail_transaction_commit_changes *changes_r); void index_transaction_rollback(struct mailbox_transaction_context *t); diff -r 5bbdfee113a5 src/lib-storage/index/index-sync-private.h --- a/src/lib-storage/index/index-sync-private.h Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/index/index-sync-private.h Mon Jul 16 17:00:56 2012 +0300 @@ -22,4 +22,6 @@ void index_sync_search_results_update(struct index_mailbox_sync_context *ctx); void index_sync_search_results_expunge(struct index_mailbox_sync_context *ctx); +int index_storage_mailbox_sync_pvt(struct mailbox *box); + #endif diff -r 5bbdfee113a5 src/lib-storage/index/index-sync-pvt.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/index-sync-pvt.c Mon Jul 16 17:00:56 2012 +0300 @@ -0,0 +1,222 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-index-alloc-cache.h" +#include "mailbox-list-private.h" +#include "index-sync-private.h" + +static int index_storage_mailbox_alloc_index_pvt(struct mailbox *box) +{ + const char *index_dir; + + if (box->index_pvt != NULL) + return 1; + + index_dir = mailbox_list_get_path(box->list, box->name, + MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE); + if (index_dir == NULL) { + /* no private indexes */ + return 0; + } + + if (mailbox_list_create_missing_index_pvt_dir(box->list, box->name) < 0) { + mail_storage_set_internal_error(box->storage); + return -1; + } + + box->index_pvt = mail_index_alloc_cache_get(NULL, index_dir, + t_strconcat(box->index_prefix, ".pvt", NULL)); + mail_index_set_fsync_mode(box->index_pvt, + box->storage->set->parsed_fsync_mode, 0); + mail_index_set_lock_method(box->index_pvt, + box->storage->set->parsed_lock_method, + mail_storage_get_lock_timeout(box->storage, -1U)); + return 1; +} + +static int index_storage_mailbox_open_index_pvt(struct mailbox *box) +{ + int ret; + + if (box->view_pvt != NULL) + return 1; + + if ((ret = index_storage_mailbox_alloc_index_pvt(box)) <= 0) + return ret; + if (mail_index_open(box->index_pvt, MAIL_INDEX_OPEN_FLAG_CREATE) < 0) + return -1; + box->view_pvt = mail_index_view_open(box->index_pvt); + return 1; +} + +static int sync_pvt_expunges(struct mailbox *box, + struct mail_index_view *view_pvt, + struct mail_index_transaction *trans_pvt, + struct mail_index_view *view_shared) +{ + uint32_t seq_shared, seq_pvt, count_shared, count_pvt; + uint32_t uid_shared, uid_pvt; + + count_shared = mail_index_view_get_messages_count(view_shared); + count_pvt = mail_index_view_get_messages_count(view_pvt); + seq_shared = seq_pvt = 1; + while (seq_pvt <= count_pvt && seq_shared <= count_shared) { + mail_index_lookup_uid(view_pvt, seq_pvt, &uid_pvt); + mail_index_lookup_uid(view_shared, seq_shared, &uid_shared); + if (uid_pvt == uid_shared) { + seq_pvt++; + seq_shared++; + } else if (uid_pvt < uid_shared) { + /* message expunged */ + mail_index_expunge(trans_pvt, seq_pvt); + seq_pvt++; + } else { + mail_storage_set_critical(box->storage, + "%s: Message UID=%u unexpectedly inserted to mailbox", + box->index_pvt->filepath, uid_shared); + return -1; + } + } + return 0; +} + +static void +sync_pvt_copy_self_flags(struct mailbox *box, + struct mail_index_view *view, + struct mail_index_transaction *trans, + ARRAY_TYPE(keyword_indexes) *keywords, + uint32_t seq_old, uint32_t seq_new) +{ + const struct mail_index_record *old_rec; + + old_rec = mail_index_lookup(view, seq_old); + mail_index_lookup_keywords(view, seq_old, keywords); + if (old_rec->flags != 0) { + mail_index_update_flags(trans, seq_new, + MODIFY_ADD, old_rec->flags); + } + if (array_count(keywords) > 0) { + struct mail_keywords *kw; + + kw = mail_index_keywords_create_from_indexes(box->index_pvt, + keywords); + mail_index_update_keywords(trans, seq_new, MODIFY_ADD, kw); + mail_index_keywords_unref(&kw); + } +} + +static int +index_storage_mailbox_sync_pvt_index(struct mailbox *box) +{ + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view_pvt; + struct mail_index_transaction *trans_pvt; + const struct mail_index_header *hdr_shared, *hdr_pvt; + struct mail_index_view *view_shared; + ARRAY_TYPE(keyword_indexes) keywords; + uint32_t seq_shared, seq_pvt, seq_old_pvt, seq2, count_shared, uid; + bool reset = FALSE, preserve_old_flags = FALSE; + int ret; + + /* open a view for the latest version of the index */ + if (mail_index_refresh(box->index) < 0) { + mail_storage_set_index_error(box); + return -1; + } + view_shared = mail_index_view_open(box->index); + hdr_shared = mail_index_get_header(view_shared); + if (hdr_shared->uid_validity == 0) { + /* the mailbox hasn't been fully created yet, + no need for a private index yet */ + mail_index_view_close(&view_shared); + return 0; + } + hdr_pvt = mail_index_get_header(box->view_pvt); + if (hdr_pvt->next_uid == hdr_shared->next_uid && + hdr_pvt->messages_count == hdr_shared->messages_count) { + /* no new or expunged mails, don't bother syncing */ + mail_index_view_close(&view_shared); + return 0; + } + + if (mail_index_sync_begin(box->index_pvt, &sync_ctx, + &view_pvt, &trans_pvt, 0) < 0) { + mail_storage_set_index_error(box); + mail_index_view_close(&view_shared); + return -1; + } + /* get an updated private header */ + hdr_pvt = mail_index_get_header(view_pvt); + + if (hdr_shared->uid_validity == hdr_pvt->uid_validity) { + /* same mailbox. expunge messages from private index that + no longer exist. */ + if (sync_pvt_expunges(box, view_pvt, trans_pvt, view_shared) < 0) { + reset = TRUE; + preserve_old_flags = TRUE; + t_array_init(&keywords, 32); + } + } else if (hdr_pvt->uid_validity == 0 || hdr_pvt->uid_validity != 0) { + /* mailbox created/recreated */ + reset = TRUE; + } + + count_shared = mail_index_view_get_messages_count(view_shared); + if (!reset) { + if (!mail_index_lookup_seq_range(view_shared, hdr_pvt->next_uid, + hdr_shared->next_uid, + &seq_shared, &seq2)) { + /* no new messages */ + seq_shared = count_shared+1; + } + } else { + mail_index_reset(trans_pvt); + mail_index_update_header(trans_pvt, + offsetof(struct mail_index_header, uid_validity), + &hdr_shared->uid_validity, + sizeof(hdr_shared->uid_validity), TRUE); + seq_shared = 1; + } + + for (; seq_shared <= count_shared; seq_shared++) { + mail_index_lookup_uid(view_shared, seq_shared, &uid); + mail_index_append(trans_pvt, uid, &seq_pvt); + if (preserve_old_flags && + mail_index_lookup_seq(view_pvt, uid, &seq_old_pvt)) { + /* copy flags from the original index */ + sync_pvt_copy_self_flags(box, view_pvt, trans_pvt, + &keywords, + seq_old_pvt, seq_pvt); + } + } + if ((ret = mail_index_sync_commit(&sync_ctx)) < 0) + mail_storage_set_index_error(box); + mail_index_view_close(&view_shared); + return ret; +} + +int index_storage_mailbox_sync_pvt(struct mailbox *box) +{ + struct mail_index_view_sync_ctx *view_sync_ctx; + enum mail_flags pvt_flags; + bool delayed_expunges; + int ret; + + pvt_flags = mailbox_get_private_flags_mask(box); + if (pvt_flags == 0) + return 0; + + if ((ret = index_storage_mailbox_open_index_pvt(box)) <= 0) + return ret; + + /* sync private index against shared index by adding/removing mails */ + if (index_storage_mailbox_sync_pvt_index(box) < 0) + return -1; + + /* sync the private view */ + view_sync_ctx = mail_index_view_sync_begin(box->view_pvt, + MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); + if (mail_index_view_sync_commit(&view_sync_ctx, &delayed_expunges) < 0) + return -1; + return 0; +} diff -r 5bbdfee113a5 src/lib-storage/index/index-sync.c --- a/src/lib-storage/index/index-sync.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/index/index-sync.c Mon Jul 16 17:00:56 2012 +0300 @@ -189,6 +189,9 @@ return &ctx->ctx; } + /* sync private index if needed */ + (void)index_storage_mailbox_sync_pvt(box); + if ((flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES) != 0) sync_flags |= MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES; diff -r 5bbdfee113a5 src/lib-storage/index/index-transaction.c --- a/src/lib-storage/index/index-transaction.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/index/index-transaction.c Mon Jul 16 17:00:56 2012 +0300 @@ -7,6 +7,8 @@ static void index_transaction_free(struct mailbox_transaction_context *t) { + if (t->view_pvt != NULL) + mail_index_view_close(&t->view_pvt); mail_cache_view_close(&t->cache_view); mail_index_view_close(&t->view); array_free(&t->module_contexts); @@ -67,24 +69,46 @@ index_transaction_free(t); } +static enum mail_index_transaction_flags +index_transaction_flags_get(enum mailbox_transaction_flags flags) +{ + enum mail_index_transaction_flags itrans_flags; + + itrans_flags = MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_UPDATES; + if ((flags & MAILBOX_TRANSACTION_FLAG_HIDE) != 0) + itrans_flags |= MAIL_INDEX_TRANSACTION_FLAG_HIDE; + if ((flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0) + itrans_flags |= MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL; + return itrans_flags; +} + +void index_transaction_init_pvt(struct mailbox_transaction_context *t) +{ + enum mail_index_transaction_flags itrans_flags; + + if (t->box->view_pvt == NULL || t->itrans_pvt != NULL) + return; + + itrans_flags = index_transaction_flags_get(t->flags); + t->itrans_pvt = mail_index_transaction_begin(t->box->view_pvt, + itrans_flags); + t->view_pvt = mail_index_transaction_open_updated_view(t->itrans_pvt); +} + void index_transaction_init(struct mailbox_transaction_context *t, struct mailbox *box, enum mailbox_transaction_flags flags) { - enum mail_index_transaction_flags trans_flags; + enum mail_index_transaction_flags itrans_flags; i_assert(box->opened); - trans_flags = MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_UPDATES; - if ((flags & MAILBOX_TRANSACTION_FLAG_HIDE) != 0) - trans_flags |= MAIL_INDEX_TRANSACTION_FLAG_HIDE; - if ((flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0) - trans_flags |= MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL; + itrans_flags = index_transaction_flags_get(flags); if ((flags & MAILBOX_TRANSACTION_FLAG_REFRESH) != 0) (void)mail_index_refresh(box->index); t->box = box; - t->itrans = mail_index_transaction_begin(box->view, trans_flags); + t->itrans = mail_index_transaction_begin(box->view, itrans_flags); t->view = mail_index_transaction_open_updated_view(t->itrans); array_create(&t->module_contexts, default_pool, @@ -121,7 +145,7 @@ struct mailbox *box = t->box; struct mail_index_transaction *itrans = t->itrans; struct mail_index_transaction_commit_result result; - int ret; + int ret = 0; memset(changes_r, 0, sizeof(*changes_r)); changes_r->pool = pool_alloconly_create(MEMPOOL_GROWING @@ -129,7 +153,10 @@ p_array_init(&changes_r->saved_uids, changes_r->pool, 32); t->changes = changes_r; - ret = mail_index_transaction_commit_full(&itrans, &result); + if (t->itrans_pvt != NULL) + ret = mail_index_transaction_commit(&t->itrans_pvt); + if (mail_index_transaction_commit_full(&itrans, &result) < 0) + ret = -1; t = NULL; if (ret < 0 && mail_index_is_deleted(box->index)) @@ -143,5 +170,7 @@ { struct mail_index_transaction *itrans = t->itrans; + if (t->itrans_pvt != NULL) + mail_index_transaction_rollback(&t->itrans_pvt); mail_index_transaction_rollback(&itrans); } diff -r 5bbdfee113a5 src/lib-storage/index/shared/shared-list.c --- a/src/lib-storage/index/shared/shared-list.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/index/shared/shared-list.c Mon Jul 16 17:00:56 2012 +0300 @@ -103,6 +103,7 @@ case MAILBOX_LIST_PATH_TYPE_MAILBOX: case MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX: case MAILBOX_LIST_PATH_TYPE_CONTROL: + case MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE: break; case MAILBOX_LIST_PATH_TYPE_INDEX: /* we can safely say we don't use indexes */ diff -r 5bbdfee113a5 src/lib-storage/list/mailbox-list-fs.c --- a/src/lib-storage/list/mailbox-list-fs.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-fs.c Mon Jul 16 17:00:56 2012 +0300 @@ -193,6 +193,11 @@ set->mailbox_dir_name, name); } break; + case MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE: + if (set->index_pvt_dir == NULL) + return NULL; + return t_strdup_printf("%s/%s%s", set->index_pvt_dir, + set->mailbox_dir_name, name); } if (type == MAILBOX_LIST_PATH_TYPE_ALT_DIR || diff -r 5bbdfee113a5 src/lib-storage/list/mailbox-list-maildir.c --- a/src/lib-storage/list/mailbox-list-maildir.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-maildir.c Mon Jul 16 17:00:56 2012 +0300 @@ -225,6 +225,11 @@ _list->set.index_dir, name); } break; + case MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE: + if (_list->set.index_pvt_dir == NULL) + return NULL; + return maildir_list_get_dirname_path(_list, + _list->set.index_pvt_dir, name); } if (type == MAILBOX_LIST_PATH_TYPE_ALT_DIR || diff -r 5bbdfee113a5 src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/mail-storage-private.h Mon Jul 16 17:00:56 2012 +0300 @@ -220,6 +220,11 @@ struct mail_index *index; struct mail_index_view *view; struct mail_cache *cache; + /* Private per-user index/view for shared mailboxes. These are synced + against the primary index and used to store per-user flags. + These are non-NULL only when mailbox has per-user flags. */ + struct mail_index *index_pvt; + struct mail_index_view *view_pvt; /* Filled lazily by mailbox_get_permissions() */ struct mailbox_permissions _perm; /* Filled lazily by mailbox_get_path() */ @@ -346,6 +351,8 @@ struct mail mail; struct mail_vfuncs v, *vlast; + uint32_t seq_pvt; + /* initial wanted fields/headers, set by mail_alloc(): */ enum mail_fetch_field wanted_fields; struct mailbox_header_lookup_ctx *wanted_headers; @@ -388,6 +395,10 @@ /* view contains all changes done within this transaction */ struct mail_index_view *view; + /* for private index updates: */ + struct mail_index_transaction *itrans_pvt; + struct mail_index_view *view_pvt; + struct mail_cache_view *cache_view; struct mail_cache_transaction_ctx *cache_trans; diff -r 5bbdfee113a5 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/mail-storage.c Mon Jul 16 17:00:56 2012 +0300 @@ -1305,10 +1305,12 @@ enum mail_flags mailbox_get_private_flags_mask(struct mailbox *box) { - if (box->v.get_private_flags_mask == NULL) + if (box->v.get_private_flags_mask != NULL) + return box->v.get_private_flags_mask(box); + else if (box->list->set.index_pvt_dir != NULL) + return MAIL_SEEN; // FIXME + else return 0; - else - return box->v.get_private_flags_mask(box); } struct mailbox_sync_context * diff -r 5bbdfee113a5 src/lib-storage/mailbox-list-private.h --- a/src/lib-storage/mailbox-list-private.h Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/mailbox-list-private.h Mon Jul 16 17:00:56 2012 +0300 @@ -196,6 +196,8 @@ const char **name); int mailbox_list_create_missing_index_dir(struct mailbox_list *list, const char *name); +int mailbox_list_create_missing_index_pvt_dir(struct mailbox_list *list, + const char *name); void mailbox_list_add_change(struct mailbox_list *list, enum mailbox_log_record_type type, diff -r 5bbdfee113a5 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/mailbox-list.c Mon Jul 16 17:00:56 2012 +0300 @@ -149,6 +149,9 @@ list->set.index_dir = set->index_dir == NULL || strcmp(set->index_dir, set->root_dir) == 0 ? NULL : p_strdup(list->pool, set->index_dir); + list->set.index_pvt_dir = set->index_pvt_dir == NULL || + strcmp(set->index_pvt_dir, set->root_dir) == 0 ? NULL : + p_strdup(list->pool, set->index_pvt_dir); list->set.control_dir = set->control_dir == NULL || strcmp(set->control_dir, set->root_dir) == 0 ? NULL : p_strdup(list->pool, set->control_dir); @@ -175,10 +178,11 @@ list->set.utf8 = set->utf8; if (ns->mail_set->mail_debug) { - i_debug("%s: root=%s, index=%s, control=%s, inbox=%s, alt=%s", + i_debug("%s: root=%s, index=%s, indexpvt=%s, control=%s, inbox=%s, alt=%s", list->name, list->set.root_dir == NULL ? "" : list->set.root_dir, list->set.index_dir == NULL ? "" : list->set.index_dir, + list->set.index_pvt_dir == NULL ? "" : list->set.index_pvt_dir, list->set.control_dir == NULL ? "" : list->set.control_dir, list->set.inbox_path == NULL ? @@ -289,6 +293,8 @@ dest = &set_r->inbox_path; else if (strcmp(key, "INDEX") == 0) dest = &set_r->index_dir; + else if (strcmp(key, "INDEXPVT") == 0) + dest = &set_r->index_pvt_dir; else if (strcmp(key, "CONTROL") == 0) dest = &set_r->control_dir; else if (strcmp(key, "ALT") == 0) @@ -967,6 +973,8 @@ case MAILBOX_LIST_PATH_TYPE_INDEX: return set->index_dir != NULL ? set->index_dir : set->root_dir; + case MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE: + return set->index_pvt_dir; } i_unreached(); } @@ -1363,19 +1371,18 @@ return mailbox_list_mkdir(list, mailbox, t_strdup_until(path, p)); } -int mailbox_list_create_missing_index_dir(struct mailbox_list *list, - const char *name) +static int mailbox_list_create_missing_dir(struct mailbox_list *list, + const char *name, + enum mailbox_list_path_type type) { const char *root_dir, *index_dir, *parent_dir, *p, *error; struct mailbox_permissions perm; unsigned int n = 0; - list->index_root_dir_created = TRUE; root_dir = mailbox_list_get_path(list, name, MAILBOX_LIST_PATH_TYPE_MAILBOX); - index_dir = mailbox_list_get_path(list, name, - MAILBOX_LIST_PATH_TYPE_INDEX); - if (*index_dir == '\0') + index_dir = mailbox_list_get_path(list, name, type); + if (index_dir == NULL || *index_dir == '\0') return 0; if (strcmp(index_dir, root_dir) == 0) { if ((list->props & MAILBOX_LIST_PROP_AUTOCREATE_DIRS) == 0) @@ -1384,8 +1391,7 @@ } if (name == NULL) { - if (mailbox_list_mkdir_root(list, index_dir, - MAILBOX_LIST_PATH_TYPE_INDEX, + if (mailbox_list_mkdir_root(list, index_dir, type, &error) < 0) { mailbox_list_set_critical(list, "Couldn't create index root dir %s: %s", @@ -1410,8 +1416,7 @@ } /* create the parent directory first */ parent_dir = t_strdup_until(index_dir, p); - if (mailbox_list_mkdir_root(list, parent_dir, - MAILBOX_LIST_PATH_TYPE_INDEX, + if (mailbox_list_mkdir_root(list, parent_dir, type, &error) < 0) { mailbox_list_set_critical(list, "Couldn't create index dir %s: %s", @@ -1422,6 +1427,21 @@ return 0; } +int mailbox_list_create_missing_index_dir(struct mailbox_list *list, + const char *name) +{ + list->index_root_dir_created = TRUE; + return mailbox_list_create_missing_dir(list, name, + MAILBOX_LIST_PATH_TYPE_INDEX); +} + +int mailbox_list_create_missing_index_pvt_dir(struct mailbox_list *list, + const char *name) +{ + return mailbox_list_create_missing_dir(list, name, + MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE); +} + const char *mailbox_list_get_last_error(struct mailbox_list *list, enum mail_error *error_r) { diff -r 5bbdfee113a5 src/lib-storage/mailbox-list.h --- a/src/lib-storage/mailbox-list.h Mon Jul 16 16:49:16 2012 +0300 +++ b/src/lib-storage/mailbox-list.h Mon Jul 16 17:00:56 2012 +0300 @@ -101,8 +101,10 @@ MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX, /* Return control directory */ MAILBOX_LIST_PATH_TYPE_CONTROL, - /* Return index file directory */ - MAILBOX_LIST_PATH_TYPE_INDEX + /* Return index directory ("" for in-memory) */ + MAILBOX_LIST_PATH_TYPE_INDEX, + /* Return the private index directory (NULL if none) */ + MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE }; enum mailbox_list_file_type { @@ -117,6 +119,7 @@ const char *layout; /* FIXME: shouldn't be here */ const char *root_dir; const char *index_dir; + const char *index_pvt_dir; const char *control_dir; const char *alt_dir; /* FIXME: dbox-specific.. */