dovecot-2.2: dsync: Various importer fixes.
dovecot at dovecot.org
dovecot at dovecot.org
Sat Feb 16 13:31:59 EET 2013
details: http://hg.dovecot.org/dovecot-2.2/rev/360661d99c42
changeset: 15772:360661d99c42
user: Timo Sirainen <tss at iki.fi>
date: Sat Feb 16 13:31:37 2013 +0200
description:
dsync: Various importer fixes.
diffstat:
src/doveadm/dsync/dsync-brain-mails.c | 3 +-
src/doveadm/dsync/dsync-mailbox-import.c | 276 +++++++++++++++++++-----------
src/doveadm/dsync/dsync-mailbox-import.h | 4 +-
3 files changed, 175 insertions(+), 108 deletions(-)
diffs (truncated from 499 to 300 lines):
diff -r e247f3a74947 -r 360661d99c42 src/doveadm/dsync/dsync-brain-mails.c
--- a/src/doveadm/dsync/dsync-brain-mails.c Sat Feb 16 08:09:41 2013 +0200
+++ b/src/doveadm/dsync/dsync-brain-mails.c Sat Feb 16 13:31:37 2013 +0200
@@ -74,7 +74,8 @@
brain->box_recv_state = DSYNC_BOX_STATE_MAILS;
return TRUE;
}
- dsync_mailbox_import_change(brain->box_importer, change);
+ if (dsync_mailbox_import_change(brain->box_importer, change) < 0)
+ brain->failed = TRUE;
return TRUE;
}
diff -r e247f3a74947 -r 360661d99c42 src/doveadm/dsync/dsync-mailbox-import.c
--- a/src/doveadm/dsync/dsync-mailbox-import.c Sat Feb 16 08:09:41 2013 +0200
+++ b/src/doveadm/dsync/dsync-mailbox-import.c Sat Feb 16 13:31:37 2013 +0200
@@ -35,6 +35,7 @@
unsigned int uid_in_local:1;
unsigned int uid_is_usable:1;
unsigned int skip:1;
+ unsigned int expunged:1;
unsigned int copy_failed:1;
};
@@ -432,10 +433,9 @@
if (newmail->uid_in_local) {
importer->cur_mail_skip = TRUE;
importer->next_local_seq++;
- } else {
- /* NOTE: assumes save_change is allocated from importer pool */
- newmail->change = save_change;
}
+ /* NOTE: assumes save_change is allocated from importer pool */
+ newmail->change = save_change;
array_append(&importer->newmails, &newmail, 1);
newmail_link(importer, newmail,
@@ -502,7 +502,6 @@
dsync_import_unexpected_state(importer, t_strdup_printf(
"Unexpected GUID mismatch for UID=%u: %s != %s",
change->uid, guid, change->guid));
- importer->last_common_uid = 1;
importer->failed = TRUE;
return FALSE;
}
@@ -518,7 +517,6 @@
dsync_import_unexpected_state(importer, t_strdup_printf(
"Unexpected GUID mismatch (2) for UID=%u: %s != %s",
change->uid, importer->cur_guid, change->guid));
- importer->last_common_uid = 1;
importer->failed = TRUE;
return FALSE;
}
@@ -935,7 +933,8 @@
/* the local mail is expunged. we'll decide later if we want
to save this mail locally or expunge it form remote. */
i_assert(change->uid > importer->last_common_uid);
- i_assert(change->uid < importer->cur_mail->uid);
+ i_assert(importer->cur_mail == NULL ||
+ change->uid < importer->cur_mail->uid);
array_append(&importer->maybe_saves, &save, 1);
}
}
@@ -1128,14 +1127,17 @@
dsync_mailbox_find_common_expunged_uid(importer, change);
}
-void dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,
- const struct dsync_mail_change *change)
+int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail_change *change)
{
i_assert(!importer->new_uids_assigned);
i_assert(importer->prev_uid < change->uid);
importer->prev_uid = change->uid;
+ if (importer->failed)
+ return -1;
+
if (!importer->last_common_uid_found)
dsync_mailbox_find_common_uid(importer, change);
@@ -1155,7 +1157,7 @@
i_assert(change->type != DSYNC_MAIL_CHANGE_TYPE_SAVE);
} else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
/* ignore */
- return;
+ return 0;
} else {
i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_SAVE);
}
@@ -1171,6 +1173,7 @@
find its GUID */
if (change->uid > importer->last_common_uid) {
i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE ||
+ importer->cur_mail == NULL ||
change->uid < importer->cur_mail->uid);
}
}
@@ -1187,24 +1190,7 @@
dsync_mailbox_import_flag_change(importer, change);
break;
}
-}
-
-static void
-dsync_msg_update_uid(struct dsync_mailbox_importer *importer,
- uint32_t old_uid, uint32_t new_uid)
-{
- struct mail_save_context *save_ctx;
-
- IMPORTER_DEBUG_CHANGE(importer);
-
- if (!mail_set_uid(importer->mail, old_uid))
- return;
-
- save_ctx = mailbox_save_alloc(importer->ext_trans);
- mailbox_save_copy_flags(save_ctx, importer->mail);
- mailbox_save_set_uid(save_ctx, new_uid);
- if (mailbox_move(&save_ctx, importer->mail) == 0)
- array_append(&importer->wanted_uids, &new_uid, 1);
+ return importer->failed ? -1 : 0;
}
static void
@@ -1212,7 +1198,6 @@
{
struct importer_new_mail *newmail, *const *newmailp;
uint32_t common_uid_next, new_uid;
- bool linked_uid;
common_uid_next = I_MAX(importer->local_uid_next,
importer->remote_uid_next);
@@ -1220,11 +1205,6 @@
newmail = *newmailp;
if (newmail->skip) {
/* already assigned */
- if (newmail->uid_in_local) {
- IMPORTER_DEBUG_CHANGE(importer);
- if (mail_set_uid(importer->mail, newmail->local_uid))
- mail_expunge(importer->mail);
- }
continue;
}
@@ -1232,30 +1212,18 @@
if (newmail->uid_is_usable) {
/* keep the UID */
new_uid = newmail->final_uid;
- linked_uid = FALSE;
} else if (newmail->link != NULL &&
newmail->link->uid_is_usable) {
+ /* we can use the linked message's UID and expunge
+ this mail */
new_uid = newmail->link->final_uid;
- linked_uid = TRUE;
} else {
new_uid = common_uid_next++;
- linked_uid = FALSE;
}
- if (newmail->uid_in_local && newmail->final_uid != new_uid) {
- /* local UID changed, reassign it by copying */
- dsync_msg_update_uid(importer, newmail->final_uid,
- new_uid);
- } else if (linked_uid && newmail->link->uid_in_local) {
- /* the linked message already exists. we'll just need
- to forget about this message. */
- i_assert(!newmail->uid_in_local);
- newmail->skip = TRUE;
- }
newmail->final_uid = new_uid;
-
- if (newmail->link != NULL && !newmail->skip) {
- /* skip the linked mail */
+ if (newmail->link != NULL) {
+ /* skip processing the linked mail */
newmail->link->skip = TRUE;
}
}
@@ -1265,13 +1233,13 @@
static int
dsync_mailbox_import_local_uid(struct dsync_mailbox_importer *importer,
- struct importer_new_mail *mail,
+ uint32_t uid, const char *guid,
struct dsync_mail *dmail_r)
{
const char *error_field, *errstr;
enum mail_error error;
- if (!mail_set_uid(importer->mail, mail->local_uid))
+ if (!mail_set_uid(importer->mail, uid))
return 0;
if (dsync_mail_fill(importer->mail, dmail_r, &error_field) < 0) {
@@ -1281,25 +1249,136 @@
i_error("Mailbox %s: Can't lookup %s for UID=%u: %s",
mailbox_get_vname(importer->box),
- error_field, mail->local_uid, errstr);
+ error_field, uid, errstr);
return -1;
}
- if (*mail->guid != '\0' && strcmp(mail->guid, dmail_r->guid) != 0) {
+ if (*guid != '\0' && strcmp(guid, dmail_r->guid) != 0) {
dsync_import_unexpected_state(importer, t_strdup_printf(
"Unexpected GUID mismatch (3) for UID=%u: %s != %s",
- mail->local_uid, dmail_r->guid, mail->guid));
+ uid, dmail_r->guid, guid));
return -1;
}
return 1;
}
-static bool newmails_need_save(struct importer_new_mail *all_newmails)
+static bool
+dsync_msg_change_uid(struct dsync_mailbox_importer *importer,
+ uint32_t old_uid, uint32_t new_uid)
{
+ struct mail_save_context *save_ctx;
+
+ IMPORTER_DEBUG_CHANGE(importer);
+
+ if (!mail_set_uid(importer->mail, old_uid))
+ return FALSE;
+
+ save_ctx = mailbox_save_alloc(importer->ext_trans);
+ mailbox_save_copy_flags(save_ctx, importer->mail);
+ mailbox_save_set_uid(save_ctx, new_uid);
+ if (mailbox_move(&save_ctx, importer->mail) < 0)
+ return FALSE;
+ array_append(&importer->wanted_uids, &new_uid, 1);
+ return TRUE;
+}
+
+static bool
+dsync_mailbox_import_change_uid(struct dsync_mailbox_importer *importer,
+ ARRAY_TYPE(seq_range) *unwanted_uids,
+ uint32_t wanted_uid)
+{
+ const struct seq_range *range;
+ unsigned int count;
+
+ while ((count = array_count(unwanted_uids)) > 0) {
+ range = array_idx(unwanted_uids, count-1);
+ if (dsync_msg_change_uid(importer, range->seq2, wanted_uid)) {
+ seq_range_array_remove(unwanted_uids, range->seq2);
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static bool
+dsync_mailbox_import_try_local(struct dsync_mailbox_importer *importer,
+ struct importer_new_mail *all_newmails,
+ ARRAY_TYPE(seq_range) *local_uids,
+ ARRAY_TYPE(seq_range) *wanted_uids)
+{
+ ARRAY_TYPE(seq_range) assigned_uids, unwanted_uids;
+ struct seq_range_iter local_iter, wanted_iter;
+ unsigned int local_n, wanted_n;
+ uint32_t local_uid, wanted_uid;
struct importer_new_mail *mail;
+ struct dsync_mail dmail;
+ if (array_count(local_uids) == 0)
+ return FALSE;
+
+ local_n = wanted_n = 0;
+ seq_range_array_iter_init(&local_iter, local_uids);
+ seq_range_array_iter_init(&wanted_iter, wanted_uids);
+
+ /* wanted_uids contains UIDs that need to exist at the end. those that
+ don't already exist in local_uids have a higher UID than any
+ existing local UID */
+ t_array_init(&assigned_uids, array_count(wanted_uids));
+ t_array_init(&unwanted_uids, 8);
+ while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
+ if (seq_range_array_iter_nth(&wanted_iter, wanted_n,
+ &wanted_uid)) {
+ if (local_uid == wanted_uid) {
+ /* we have exactly the UID we want. keep it. */
+ seq_range_array_add(&assigned_uids, wanted_uid);
+ wanted_n++;
+ continue;
+ }
+ i_assert(local_uid < wanted_uid);
+ }
+ /* we no longer want this local UID. */
+ seq_range_array_add(&unwanted_uids, local_uid);
+ }
+
+ /* reuse as many existing messages as possible by changing their UIDs */
+ while (seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) {
+ if (!dsync_mailbox_import_change_uid(importer, &unwanted_uids,
+ wanted_uid))
+ break;
More information about the dovecot-cvs
mailing list