dovecot-2.2: dsync: Include messages_count in the mailbox states.
dovecot at dovecot.org
dovecot at dovecot.org
Mon Apr 28 17:11:22 UTC 2014
details: http://hg.dovecot.org/dovecot-2.2/rev/9669c9a8984f
changeset: 17282:9669c9a8984f
user: Timo Sirainen <tss at iki.fi>
date: Mon Apr 28 20:10:56 2014 +0300
description:
dsync: Include messages_count in the mailbox states.
This allows detecting that stateful dsync can't be done when message count
is suddenly wrong.
diffstat:
src/doveadm/dsync/dsync-brain-mailbox.c | 20 +++++++++----
src/doveadm/dsync/dsync-brain-mails.c | 3 ++
src/doveadm/dsync/dsync-brain.c | 3 +-
src/doveadm/dsync/dsync-ibc-stream.c | 9 +++++-
src/doveadm/dsync/dsync-mailbox-import.c | 4 ++
src/doveadm/dsync/dsync-mailbox-import.h | 1 +
src/doveadm/dsync/dsync-mailbox-state.c | 46 +++++++++++++++++++++++++++----
src/doveadm/dsync/dsync-mailbox-state.h | 1 +
8 files changed, 72 insertions(+), 15 deletions(-)
diffs (287 lines):
diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-brain-mailbox.c
--- a/src/doveadm/dsync/dsync-brain-mailbox.c Mon Apr 28 19:12:01 2014 +0300
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c Mon Apr 28 20:10:56 2014 +0300
@@ -248,7 +248,7 @@
(void)dsync_mailbox_export_deinit(&brain->box_exporter, &error);
}
if (brain->box_importer != NULL) {
- uint32_t last_common_uid;
+ uint32_t last_common_uid, last_messages_count;
uint64_t last_common_modseq, last_common_pvt_modseq;
bool changes_during_sync;
@@ -258,6 +258,7 @@
&last_common_uid,
&last_common_modseq,
&last_common_pvt_modseq,
+ &last_messages_count,
&changes_during_sync);
}
if (brain->log_scan != NULL)
@@ -329,7 +330,8 @@
state->last_uidvalidity != dsync_box->uid_validity ||
state->last_common_uid+1 != dsync_box->uid_next ||
state->last_common_modseq != dsync_box->highest_modseq ||
- state->last_common_pvt_modseq != dsync_box->highest_pvt_modseq;
+ state->last_common_pvt_modseq != dsync_box->highest_pvt_modseq ||
+ state->last_messages_count != dsync_box->messages_count;
}
static int
@@ -376,13 +378,14 @@
if (!dsync_brain_has_mailbox_state_changed(brain, &dsync_box)) {
if (brain->debug) {
i_debug("brain %c: Skipping mailbox %s with unchanged state "
- "uidvalidity=%u uidnext=%u highestmodseq=%llu highestpvtmodseq=%llu",
+ "uidvalidity=%u uidnext=%u highestmodseq=%llu highestpvtmodseq=%llu messages=%u",
brain->master_brain ? 'M' : 'S',
guid_128_to_string(dsync_box.mailbox_guid),
dsync_box.uid_validity,
dsync_box.uid_next,
(unsigned long long)dsync_box.highest_modseq,
- (unsigned long long)dsync_box.highest_pvt_modseq);
+ (unsigned long long)dsync_box.highest_pvt_modseq,
+ dsync_box.messages_count);
}
mailbox_free(&box);
return 0;
@@ -681,7 +684,8 @@
sizeof(dsync_box->mailbox_guid)) == 0);
dsync_ibc_send_mailbox(brain->ibc, &local_dsync_box);
- dsync_brain_mailbox_update_pre(brain, box, &local_dsync_box, dsync_box);
+ resync = !dsync_brain_mailbox_update_pre(brain, box, &local_dsync_box,
+ dsync_box);
if (!dsync_boxes_need_sync(brain, &local_dsync_box, dsync_box)) {
/* no fields appear to have changed, skip this mailbox */
@@ -696,8 +700,12 @@
/* start export/import */
dsync_brain_sync_mailbox_init(brain, box, &local_dsync_box, FALSE);
- if (dsync_brain_sync_mailbox_open(brain, dsync_box) < 0)
+ if ((ret = dsync_brain_sync_mailbox_open(brain, dsync_box)) < 0)
return TRUE;
+ if (ret == 0 || resync) {
+ brain->changes_during_sync = TRUE;
+ brain->require_full_resync = TRUE;
+ }
brain->state = DSYNC_STATE_SYNC_MAILS;
return TRUE;
diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-brain-mails.c
--- a/src/doveadm/dsync/dsync-brain-mails.c Mon Apr 28 19:12:01 2014 +0300
+++ b/src/doveadm/dsync/dsync-brain-mails.c Mon Apr 28 20:10:56 2014 +0300
@@ -200,11 +200,14 @@
brain->local_dsync_box.highest_modseq;
state.last_common_pvt_modseq =
brain->local_dsync_box.highest_pvt_modseq;
+ state.last_messages_count =
+ brain->local_dsync_box.messages_count;
} else {
if (dsync_mailbox_import_deinit(&brain->box_importer, TRUE,
&state.last_common_uid,
&state.last_common_modseq,
&state.last_common_pvt_modseq,
+ &state.last_messages_count,
&state.changes_during_sync) < 0) {
brain->failed = TRUE;
return;
diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-brain.c
--- a/src/doveadm/dsync/dsync-brain.c Mon Apr 28 19:12:01 2014 +0300
+++ b/src/doveadm/dsync/dsync-brain.c Mon Apr 28 20:10:56 2014 +0300
@@ -620,13 +620,14 @@
iter = hash_table_iterate_init(brain->mailbox_states);
while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) {
- i_debug("brain %c: Mailbox %s state: uidvalidity=%u uid=%u modseq=%llu pvt_modseq=%llu changes_during_sync=%d",
+ i_debug("brain %c: Mailbox %s state: uidvalidity=%u uid=%u modseq=%llu pvt_modseq=%llu messages=%u changes_during_sync=%d",
brain->master_brain ? 'M' : 'S',
guid_128_to_string(guid),
state->last_uidvalidity,
state->last_common_uid,
(unsigned long long)state->last_common_modseq,
(unsigned long long)state->last_common_pvt_modseq,
+ state->last_messages_count,
state->changes_during_sync);
}
hash_table_iterate_deinit(&iter);
diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-ibc-stream.c
--- a/src/doveadm/dsync/dsync-ibc-stream.c Mon Apr 28 19:12:01 2014 +0300
+++ b/src/doveadm/dsync/dsync-ibc-stream.c Mon Apr 28 20:10:56 2014 +0300
@@ -80,7 +80,7 @@
{ .name = "mailbox_state",
.chr = 'S',
.required_keys = "mailbox_guid last_uidvalidity last_common_uid "
- "last_common_modseq last_common_pvt_modseq",
+ "last_common_modseq last_common_pvt_modseq last_messages_count",
.optional_keys = "changes_during_sync"
},
{ .name = "mailbox_tree_node",
@@ -801,6 +801,8 @@
dec2str(state->last_common_modseq));
dsync_serializer_encode_add(encoder, "last_common_pvt_modseq",
dec2str(state->last_common_pvt_modseq));
+ dsync_serializer_encode_add(encoder, "last_messages_count",
+ dec2str(state->last_messages_count));
if (state->changes_during_sync)
dsync_serializer_encode_add(encoder, "changes_during_sync", "");
@@ -848,6 +850,11 @@
dsync_ibc_input_error(ibc, decoder, "Invalid last_common_pvt_modseq");
return DSYNC_IBC_RECV_RET_TRYAGAIN;
}
+ value = dsync_deserializer_decode_get(decoder, "last_messages_count");
+ if (str_to_uint32(value, &state_r->last_messages_count) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid last_messages_count");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
if (dsync_deserializer_decode_try(decoder, "changes_during_sync", &value))
state_r->changes_during_sync = TRUE;
return DSYNC_IBC_RECV_RET_OK;
diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-mailbox-import.c
--- a/src/doveadm/dsync/dsync-mailbox-import.c Mon Apr 28 19:12:01 2014 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Apr 28 20:10:56 2014 +0300
@@ -2340,9 +2340,11 @@
uint32_t *last_common_uid_r,
uint64_t *last_common_modseq_r,
uint64_t *last_common_pvt_modseq_r,
+ uint32_t *last_messages_count_r,
bool *changes_during_sync_r)
{
struct dsync_mailbox_importer *importer = *_importer;
+ struct mailbox_status status;
int ret;
*_importer = NULL;
@@ -2392,6 +2394,8 @@
*last_common_modseq_r = importer->local_initial_highestmodseq;
*last_common_pvt_modseq_r = importer->local_initial_highestpvtmodseq;
}
+ mailbox_get_open_status(importer->box, STATUS_MESSAGES, &status);
+ *last_messages_count_r = status.messages;
ret = importer->failed ? -1 : 0;
pool_unref(&importer->pool);
diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-mailbox-import.h
--- a/src/doveadm/dsync/dsync-mailbox-import.h Mon Apr 28 19:12:01 2014 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-import.h Mon Apr 28 20:10:56 2014 +0300
@@ -41,6 +41,7 @@
uint32_t *last_common_uid_r,
uint64_t *last_common_modseq_r,
uint64_t *last_common_pvt_modseq_r,
+ uint32_t *last_messages_count_r,
bool *changes_during_sync_r);
const char *dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer *importer);
diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-mailbox-state.c
--- a/src/doveadm/dsync/dsync-mailbox-state.c Mon Apr 28 19:12:01 2014 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-state.c Mon Apr 28 20:10:56 2014 +0300
@@ -7,7 +7,11 @@
#include "hash.h"
#include "dsync-mailbox-state.h"
-#define MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8)
+#define DSYNC_STATE_MAJOR_VERSION 1
+#define DSYNC_STATE_MINOR_VERSION 0
+
+#define V0_MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8)
+#define MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8 + 4)
static void put_uint32(buffer_t *output, uint32_t num)
{
@@ -32,6 +36,11 @@
buffer_t *buf = buffer_create_dynamic(pool_datastack_create(), 128);
uint32_t crc = 0;
+ buffer_append_c(buf, DSYNC_STATE_MAJOR_VERSION);
+ buffer_append_c(buf, DSYNC_STATE_MINOR_VERSION);
+ buffer_append_c(buf, '\0');
+ buffer_append_c(buf, '\0');
+
iter = hash_table_iterate_init(states);
while (hash_table_iterate(iter, states, &guid, &state)) {
buffer_append(buf, state->mailbox_guid,
@@ -42,6 +51,7 @@
put_uint32(buf, state->last_common_modseq >> 32);
put_uint32(buf, state->last_common_pvt_modseq & 0xffffffffU);
put_uint32(buf, state->last_common_pvt_modseq >> 32);
+ put_uint32(buf, state->last_messages_count); /* v1 */
if (buf->used % 3 == 0) {
crc = crc32_data_more(crc, buf->data, buf->used);
base64_encode(buf->data, buf->used, output);
@@ -55,6 +65,19 @@
base64_encode(buf->data, buf->used, output);
}
+static int dsync_mailbox_states_retry_import_v0(const buffer_t *buf)
+{
+ const unsigned char *data = buf->data;
+
+ /* v0 had no version header and no last_messages_count */
+
+ if ((buf->used-4) % V0_MAILBOX_SIZE != 0 ||
+ get_uint32(data + buf->used-4) != crc32_data(data, buf->used-4))
+ return -1;
+ /* looks like valid v0 format, silently treat it as empty state */
+ return 0;
+}
+
int dsync_mailbox_states_import(HASH_TABLE_TYPE(dsync_mailbox_state) states,
pool_t pool, const char *input,
const char **error_r)
@@ -71,21 +94,29 @@
*error_r = "Invalid base64 data";
return -1;
}
- if (buf->used < 4) {
+ /* v1: 4 byte header, mailboxes[], CRC32 */
+ data = buf->data;
+
+ if (buf->used == 4 && get_uint32(data) == 0) {
+ /* v0: Empty state */
+ return 0;
+ }
+ if (buf->used < 8) {
*error_r = "Input too small";
return -1;
}
- if ((buf->used-4) % MAILBOX_SIZE != 0) {
+
+ if ((buf->used-8) % MAILBOX_SIZE != 0) {
*error_r = "Invalid input size";
- return -1;
+ return dsync_mailbox_states_retry_import_v0(buf);
}
- data = buf->data;
- count = (buf->used-4) / MAILBOX_SIZE;
if (get_uint32(data + buf->used-4) != crc32_data(data, buf->used-4)) {
*error_r = "CRC32 mismatch";
- return -1;
+ return dsync_mailbox_states_retry_import_v0(buf);
}
+ data += 4;
+ count = (buf->used-8) / MAILBOX_SIZE;
for (i = 0; i < count; i++, data += MAILBOX_SIZE) {
state = p_new(pool, struct dsync_mailbox_state, 1);
@@ -98,6 +129,7 @@
state->last_common_pvt_modseq =
get_uint32(data + GUID_128_SIZE + 16) |
(uint64_t)get_uint32(data + GUID_128_SIZE + 20) << 32;
+ state->last_messages_count = get_uint32(data + GUID_128_SIZE + 24);
guid_p = state->mailbox_guid;
hash_table_insert(states, guid_p, state);
}
diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-mailbox-state.h
--- a/src/doveadm/dsync/dsync-mailbox-state.h Mon Apr 28 19:12:01 2014 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-state.h Mon Apr 28 20:10:56 2014 +0300
@@ -9,6 +9,7 @@
uint32_t last_common_uid;
uint64_t last_common_modseq;
uint64_t last_common_pvt_modseq;
+ uint32_t last_messages_count;
bool changes_during_sync;
};
ARRAY_DEFINE_TYPE(dsync_mailbox_state, struct dsync_mailbox_state);
More information about the dovecot-cvs
mailing list