dovecot-2.2: dsync: Various importer fixes. Handle unexpectedly ...
dovecot at dovecot.org
dovecot at dovecot.org
Sat Feb 16 07:16:58 EET 2013
details: http://hg.dovecot.org/dovecot-2.2/rev/4260244e57e2
changeset: 15766:4260244e57e2
user: Timo Sirainen <tss at iki.fi>
date: Sat Feb 16 07:15:55 2013 +0200
description:
dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better.
diffstat:
src/doveadm/dsync/dsync-brain-mailbox.c | 1 +
src/doveadm/dsync/dsync-brain-mails.c | 2 +-
src/doveadm/dsync/dsync-mail.c | 40 +++
src/doveadm/dsync/dsync-mail.h | 2 +
src/doveadm/dsync/dsync-mailbox-export.c | 43 +---
src/doveadm/dsync/dsync-mailbox-import.c | 319 +++++++++++++++++++++++-------
src/doveadm/dsync/dsync-mailbox-import.h | 1 +
7 files changed, 299 insertions(+), 109 deletions(-)
diffs (truncated from 785 to 300 lines):
diff -r 44eda990f757 -r 4260244e57e2 src/doveadm/dsync/dsync-brain-mailbox.c
--- a/src/doveadm/dsync/dsync-brain-mailbox.c Sat Feb 16 07:12:18 2013 +0200
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c Sat Feb 16 07:15:55 2013 +0200
@@ -247,6 +247,7 @@
i_assert(brain->failed);
(void)dsync_mailbox_import_deinit(&brain->box_importer,
+ FALSE,
&last_common_uid,
&last_common_modseq,
&last_common_pvt_modseq,
diff -r 44eda990f757 -r 4260244e57e2 src/doveadm/dsync/dsync-brain-mails.c
--- a/src/doveadm/dsync/dsync-brain-mails.c Sat Feb 16 07:12:18 2013 +0200
+++ b/src/doveadm/dsync/dsync-brain-mails.c Sat Feb 16 07:15:55 2013 +0200
@@ -169,7 +169,7 @@
state.last_common_pvt_modseq =
brain->local_dsync_box.highest_pvt_modseq;
} else {
- if (dsync_mailbox_import_deinit(&brain->box_importer,
+ if (dsync_mailbox_import_deinit(&brain->box_importer, TRUE,
&state.last_common_uid,
&state.last_common_modseq,
&state.last_common_pvt_modseq,
diff -r 44eda990f757 -r 4260244e57e2 src/doveadm/dsync/dsync-mail.c
--- a/src/doveadm/dsync/dsync-mail.c Sat Feb 16 07:12:18 2013 +0200
+++ b/src/doveadm/dsync/dsync-mail.c Sat Feb 16 07:15:55 2013 +0200
@@ -54,6 +54,46 @@
return ret;
}
+int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r,
+ const char **error_field_r)
+{
+ const char *guid, *str;
+
+ memset(dmail_r, 0, sizeof(*dmail_r));
+
+ if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) {
+ *error_field_r = "GUID";
+ return -1;
+ }
+ dmail_r->guid = guid;
+ dmail_r->uid = mail->uid;
+
+ dmail_r->input_mail = mail;
+ dmail_r->input_mail_uid = mail->uid;
+ if (mail_get_stream(mail, NULL, NULL, &dmail_r->input) < 0) {
+ *error_field_r = "body";
+ return -1;
+ }
+
+ if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &dmail_r->pop3_uidl) < 0) {
+ *error_field_r = "pop3-uidl";
+ return -1;
+ }
+ if (mail_get_special(mail, MAIL_FETCH_POP3_ORDER, &str) < 0) {
+ *error_field_r = "pop3-order";
+ return -1;
+ }
+ if (*str != '\0') {
+ if (str_to_uint(str, &dmail_r->pop3_order) < 0)
+ i_unreached();
+ }
+ if (mail_get_received_date(mail, &dmail_r->received_date) < 0) {
+ *error_field_r = "received-date";
+ return -1;
+ }
+ return 0;
+}
+
static void
const_string_array_dup(pool_t pool, const ARRAY_TYPE(const_string) *src,
ARRAY_TYPE(const_string) *dest)
diff -r 44eda990f757 -r 4260244e57e2 src/doveadm/dsync/dsync-mail.h
--- a/src/doveadm/dsync/dsync-mail.h Sat Feb 16 07:12:18 2013 +0200
+++ b/src/doveadm/dsync/dsync-mail.h Sat Feb 16 07:15:55 2013 +0200
@@ -77,6 +77,8 @@
};
int dsync_mail_get_hdr_hash(struct mail *mail, const char **hdr_hash_r);
+int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r,
+ const char **error_field_r);
void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src,
struct dsync_mail_change *dest_r);
diff -r 44eda990f757 -r 4260244e57e2 src/doveadm/dsync/dsync-mailbox-export.c
--- a/src/doveadm/dsync/dsync-mailbox-export.c Sat Feb 16 07:12:18 2013 +0200
+++ b/src/doveadm/dsync/dsync-mailbox-export.c Sat Feb 16 07:15:55 2013 +0200
@@ -566,49 +566,30 @@
static int dsync_mailbox_export_mail(struct dsync_mailbox_exporter *exporter,
struct mail *mail)
{
- struct dsync_mail *dmail = &exporter->dsync_mail;
struct dsync_mail_guid_instances *instances;
- const char *guid, *str;
+ const char *error_field;
- if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0)
- return dsync_mail_error(exporter, mail, "GUID");
+ if (dsync_mail_fill(mail, &exporter->dsync_mail, &error_field) < 0)
+ return dsync_mail_error(exporter, mail, error_field);
- memset(dmail, 0, sizeof(*dmail));
- if (!seq_range_exists(&exporter->requested_uids, mail->uid))
- dmail->guid = guid;
- else {
- dmail->uid = mail->uid;
- dmail->guid = "";
- }
-
- instances = *guid == '\0' ? NULL :
- hash_table_lookup(exporter->export_guids, guid);
+ instances = *exporter->dsync_mail.guid == '\0' ? NULL :
+ hash_table_lookup(exporter->export_guids,
+ exporter->dsync_mail.guid);
if (instances != NULL) {
/* GUID found */
- } else if (dmail->uid != 0) {
+ } else if (exporter->dsync_mail.uid != 0) {
/* mail requested by UID */
} else {
exporter->error = p_strdup_printf(exporter->pool,
"GUID unexpectedly changed for UID=%u GUID=%s",
- mail->uid, guid);
+ mail->uid, exporter->dsync_mail.guid);
return -1;
}
- dmail->input_mail = mail;
- dmail->input_mail_uid = mail->uid;
- if (mail_get_stream(mail, NULL, NULL, &dmail->input) < 0)
- return dsync_mail_error(exporter, mail, "body");
-
- if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &dmail->pop3_uidl) < 0)
- return dsync_mail_error(exporter, mail, "pop3-uidl");
- if (mail_get_special(mail, MAIL_FETCH_POP3_ORDER, &str) < 0)
- return dsync_mail_error(exporter, mail, "pop3-order");
- if (*str != '\0') {
- if (str_to_uint(str, &dmail->pop3_order) < 0)
- i_unreached();
- }
- if (mail_get_received_date(mail, &dmail->received_date) < 0)
- return dsync_mail_error(exporter, mail, "received-date");
+ if (!seq_range_exists(&exporter->requested_uids, mail->uid))
+ exporter->dsync_mail.uid = 0;
+ else
+ exporter->dsync_mail.guid = "";
/* this message was successfully returned, don't try retrying it */
if (instances != NULL)
diff -r 44eda990f757 -r 4260244e57e2 src/doveadm/dsync/dsync-mailbox-import.c
--- a/src/doveadm/dsync/dsync-mailbox-import.c Sat Feb 16 07:12:18 2013 +0200
+++ b/src/doveadm/dsync/dsync-mailbox-import.c Sat Feb 16 07:15:55 2013 +0200
@@ -26,7 +26,12 @@
const char *guid;
struct dsync_mail_change *change;
- uint32_t uid;
+ /* the final UID for the message */
+ uint32_t final_uid;
+ /* the original local UID, or 0 if exists only remotely */
+ uint32_t local_uid;
+ /* the original remote UID, or 0 if exists only remotely */
+ uint32_t remote_uid;
unsigned int uid_in_local:1;
unsigned int uid_is_usable:1;
unsigned int skip:1;
@@ -79,6 +84,7 @@
unsigned int failed:1;
unsigned int debug:1;
+ unsigned int stateful_import:1;
unsigned int last_common_uid_found:1;
unsigned int cur_uid_has_change:1;
unsigned int cur_mail_skip:1;
@@ -90,6 +96,10 @@
unsigned int mails_have_guids:1;
};
+static void dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail *mail,
+ struct importer_new_mail *all_newmails);
+
static void
dsync_mailbox_import_search_init(struct dsync_mailbox_importer *importer)
{
@@ -147,6 +157,7 @@
importer->remote_first_recent_uid = remote_first_recent_uid;
importer->remote_highest_modseq = remote_highest_modseq;
importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq;
+ importer->stateful_import = importer->last_common_uid_found;
hash_table_create(&importer->import_guids, pool, 0, str_hash, strcmp);
hash_table_create_direct(&importer->import_uids, pool, 0);
@@ -195,7 +206,8 @@
if (error == MAIL_ERROR_EXPUNGED)
return;
- i_error("Can't lookup %s for UID=%u: %s", field, mail->uid, errstr);
+ i_error("Mailbox %s: Can't lookup %s for UID=%u: %s",
+ mailbox_get_vname(mail->box), field, mail->uid, errstr);
importer->failed = TRUE;
}
@@ -298,19 +310,6 @@
return 0;
}
-static void importer_mail_request(struct dsync_mailbox_importer *importer,
- struct importer_new_mail *newmail)
-{
- struct dsync_mail_request *request;
-
- if (importer->want_mail_requests && !newmail->uid_in_local) {
- IMPORTER_DEBUG_CHANGE(importer);
- request = array_append_space(&importer->mail_requests);
- request->guid = newmail->guid;
- request->uid = newmail->uid;
- }
-}
-
static void newmail_link(struct dsync_mailbox_importer *importer,
struct importer_new_mail *newmail, uint32_t remote_uid)
{
@@ -323,17 +322,21 @@
/* first mail for this GUID */
hash_table_insert(importer->import_guids,
newmail->guid, newmail);
- importer_mail_request(importer, newmail);
return;
}
} else {
+ if (remote_uid == 0) {
+ /* mail exists only locally. we don't want to request
+ it, and we'll assume it has no duplicate
+ instances. */
+ return;
+ }
first_mail = hash_table_lookup(importer->import_uids,
POINTER_CAST(remote_uid));
if (first_mail == NULL) {
/* first mail for this UID */
hash_table_insert(importer->import_uids,
POINTER_CAST(remote_uid), newmail);
- importer_mail_request(importer, newmail);
return;
}
}
@@ -341,7 +344,7 @@
2) find our link */
last = &first_mail->next;
for (mail = first_mail; mail != NULL; mail = mail->next) {
- if (mail->uid == newmail->uid)
+ if (mail->final_uid == newmail->final_uid)
mail->uid_is_usable = TRUE;
if (link == NULL && mail->link == NULL &&
mail->uid_in_local != newmail->uid_in_local)
@@ -389,19 +392,21 @@
}
newmail = p_new(importer->pool, struct importer_new_mail, 1);
newmail->guid = p_strdup(importer->pool, importer->cur_guid);
- newmail->uid = importer->cur_mail->uid;
+ newmail->final_uid = importer->cur_mail->uid;
+ newmail->local_uid = importer->cur_mail->uid;
newmail->uid_in_local = TRUE;
newmail->uid_is_usable =
- newmail->uid >= importer->remote_uid_next;
+ newmail->final_uid >= importer->remote_uid_next;
remote_saved = FALSE;
} else if (diff > 0) {
i_assert(save_change != NULL);
newmail = p_new(importer->pool, struct importer_new_mail, 1);
newmail->guid = save_change->guid;
- newmail->uid = save_change->uid;
+ newmail->final_uid = save_change->uid;
+ newmail->remote_uid = save_change->uid;
newmail->uid_in_local = FALSE;
newmail->uid_is_usable =
- newmail->uid >= importer->local_uid_next;
+ newmail->final_uid >= importer->local_uid_next;
remote_saved = TRUE;
} else {
/* identical */
@@ -409,7 +414,9 @@
i_assert(save_change != NULL);
newmail = p_new(importer->pool, struct importer_new_mail, 1);
newmail->guid = save_change->guid;
- newmail->uid = importer->cur_mail->uid;
+ newmail->final_uid = importer->cur_mail->uid;
+ newmail->local_uid = importer->cur_mail->uid;
+ newmail->remote_uid = save_change->uid;
newmail->uid_in_local = TRUE;
newmail->uid_is_usable = TRUE;
newmail->link = newmail;
@@ -425,7 +432,8 @@
}
array_append(&importer->newmails, &newmail, 1);
- newmail_link(importer, newmail, save_change->uid);
+ newmail_link(importer, newmail,
More information about the dovecot-cvs
mailing list