dovecot-2.2: doveadm sync/backup: Added -s <state> parameter to ...
dovecot at dovecot.org
dovecot at dovecot.org
Sun Oct 21 12:20:47 EEST 2012
details: http://hg.dovecot.org/dovecot-2.2/rev/6c850258002f
changeset: 15232:6c850258002f
user: Timo Sirainen <tss at iki.fi>
date: Sun Oct 21 12:20:30 2012 +0300
description:
doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Initially use an empty string for the input state. The output state is
written to stdout.
diffstat:
src/doveadm/dsync/Makefile.am | 1 -
src/doveadm/dsync/doveadm-dsync.c | 19 +++++-
src/doveadm/dsync/dsync-brain-mailbox.c | 33 ++-------
src/doveadm/dsync/dsync-brain-private.h | 9 +-
src/doveadm/dsync/dsync-brain.c | 87 +++++++++++++++++++++----
src/doveadm/dsync/dsync-brain.h | 3 +
src/doveadm/dsync/dsync-mailbox-export.c | 20 ++++-
src/doveadm/dsync/dsync-mailbox-import.c | 4 +-
src/doveadm/dsync/dsync-mailbox-state-export.h | 14 ----
src/doveadm/dsync/dsync-mailbox-state.c | 35 +++++++--
src/doveadm/dsync/dsync-mailbox-state.h | 10 +-
src/doveadm/dsync/dsync-mailbox-tree.c | 9 ++
src/doveadm/dsync/dsync-mailbox-tree.h | 5 +
13 files changed, 166 insertions(+), 83 deletions(-)
diffs (truncated from 524 to 300 lines):
diff -r 454d0563927d -r 6c850258002f src/doveadm/dsync/Makefile.am
--- a/src/doveadm/dsync/Makefile.am Sun Oct 21 10:01:54 2012 +0300
+++ b/src/doveadm/dsync/Makefile.am Sun Oct 21 12:20:30 2012 +0300
@@ -41,7 +41,6 @@
dsync-mailbox-import.h \
dsync-mailbox-export.h \
dsync-mailbox-state.h \
- dsync-mailbox-state-export.h \
dsync-mailbox-tree.h \
dsync-mailbox-tree-private.h \
dsync-serializer.h \
diff -r 454d0563927d -r 6c850258002f src/doveadm/dsync/doveadm-dsync.c
--- a/src/doveadm/dsync/doveadm-dsync.c Sun Oct 21 10:01:54 2012 +0300
+++ b/src/doveadm/dsync/doveadm-dsync.c Sun Oct 21 12:20:30 2012 +0300
@@ -23,12 +23,13 @@
#include <unistd.h>
#include <ctype.h>
-#define DSYNC_COMMON_GETOPT_ARGS "+dEfl:m:n:"
+#define DSYNC_COMMON_GETOPT_ARGS "+dEfl:m:n:s:"
struct dsync_cmd_context {
struct doveadm_mail_cmd_context ctx;
enum dsync_brain_sync_type sync_type;
const char *mailbox, *namespace_prefix;
+ const char *state_input;
const char *remote_name;
const char *local_location;
@@ -355,7 +356,9 @@
brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND;
brain = dsync_brain_master_init(user, ibc, sync_ns,
- ctx->sync_type, brain_flags, "");
+ ctx->sync_type, brain_flags,
+ ctx->state_input == NULL ? "" :
+ ctx->state_input);
if (!ctx->remote) {
if (cmd_dsync_run_local(ctx, user, brain, ibc2) < 0)
@@ -364,6 +367,12 @@
cmd_dsync_run_remote(user);
}
+ if (ctx->state_input != NULL) {
+ string_t *str = t_str_new(128);
+ dsync_brain_get_state(brain, str);
+ printf("%s\n", str_c(str));
+ }
+
if (dsync_brain_deinit(&brain) < 0)
_ctx->exit_code = EX_TEMPFAIL;
dsync_ibc_deinit(&ibc);
@@ -482,6 +491,12 @@
return FALSE;
ctx->reverse_backup = TRUE;
break;
+ case 's':
+ if (ctx->sync_type != DSYNC_BRAIN_SYNC_TYPE_FULL &&
+ *optarg != '\0')
+ ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_STATE;
+ ctx->state_input = optarg;
+ break;
default:
return FALSE;
}
diff -r 454d0563927d -r 6c850258002f src/doveadm/dsync/dsync-brain-mailbox.c
--- a/src/doveadm/dsync/dsync-brain-mailbox.c Sun Oct 21 10:01:54 2012 +0300
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c Sun Oct 21 12:20:30 2012 +0300
@@ -63,31 +63,21 @@
dsync_mailbox_state_find(struct dsync_brain *brain,
const guid_128_t mailbox_guid)
{
- const struct dsync_mailbox_state *state;
+ const uint8_t *guid_p;
- array_foreach(&brain->mailbox_states, state) {
- if (memcmp(state->mailbox_guid, mailbox_guid,
- sizeof(state->mailbox_guid)) == 0)
- return state;
- }
- return NULL;
+ guid_p = mailbox_guid;
+ return hash_table_lookup(brain->mailbox_states, guid_p);
}
static void
dsync_mailbox_state_remove(struct dsync_brain *brain,
const guid_128_t mailbox_guid)
{
- const struct dsync_mailbox_state *states;
- unsigned int i, count;
+ const uint8_t *guid_p;
- states = array_get(&brain->mailbox_states, &count);
- for (i = 0; i < count; i++) {
- if (memcmp(states[i].mailbox_guid, mailbox_guid,
- sizeof(states[i].mailbox_guid)) == 0) {
- array_delete(&brain->mailbox_states, i, 1);
- break;
- }
- }
+ guid_p = mailbox_guid;
+ if (hash_table_lookup(brain->mailbox_states, guid_p) != NULL)
+ hash_table_remove(brain->mailbox_states, guid_p);
}
void dsync_brain_sync_init_box_states(struct dsync_brain *brain)
@@ -217,16 +207,9 @@
void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain)
{
- struct dsync_mailbox_state *state;
- uint8_t *guid_p;
-
i_assert(brain->box != NULL);
- state = p_new(brain->pool, struct dsync_mailbox_state, 1);
- *state = brain->mailbox_state;
- guid_p = state->mailbox_guid;
- hash_table_insert(brain->remote_mailbox_states, guid_p, state);
-
+ array_append(&brain->remote_mailbox_states, &brain->mailbox_state, 1);
if (brain->box_exporter != NULL) {
const char *error;
diff -r 454d0563927d -r 6c850258002f src/doveadm/dsync/dsync-brain-private.h
--- a/src/doveadm/dsync/dsync-brain-private.h Sun Oct 21 10:01:54 2012 +0300
+++ b/src/doveadm/dsync/dsync-brain-private.h Sun Oct 21 12:20:30 2012 +0300
@@ -66,15 +66,14 @@
/* list of mailbox states
for master brain: given to brain at init and
for slave brain: received from DSYNC_STATE_SLAVE_RECV_LAST_COMMON */
- ARRAY_TYPE(dsync_mailbox_state) mailbox_states;
+ HASH_TABLE_TYPE(dsync_mailbox_state) mailbox_states;
/* DSYNC_STATE_MASTER_SEND_LAST_COMMON: current send position */
- unsigned int mailbox_state_idx;
+ struct hash_iterate_context *mailbox_states_iter;
/* state of the mailbox we're currently syncing, changed at
init and deinit */
struct dsync_mailbox_state mailbox_state;
- /* GUID -> dsync_mailbox_state for mailboxes that have already
- been synced */
- HASH_TABLE(uint8_t *, struct dsync_mailbox_state *) remote_mailbox_states;
+ /* new states for synced mailboxes */
+ ARRAY_TYPE(dsync_mailbox_state) remote_mailbox_states;
unsigned int master_brain:1;
unsigned int guid_requests:1;
diff -r 454d0563927d -r 6c850258002f src/doveadm/dsync/dsync-brain.c
--- a/src/doveadm/dsync/dsync-brain.c Sun Oct 21 10:01:54 2012 +0300
+++ b/src/doveadm/dsync/dsync-brain.c Sun Oct 21 12:20:30 2012 +0300
@@ -47,9 +47,9 @@
brain->user = user;
brain->ibc = ibc;
brain->sync_type = DSYNC_BRAIN_SYNC_TYPE_UNKNOWN;
- hash_table_create(&brain->remote_mailbox_states,
- brain->pool, 0, guid_128_hash, guid_128_cmp);
- p_array_init(&brain->mailbox_states, pool, 64);
+ hash_table_create(&brain->mailbox_states, pool, 0,
+ guid_128_hash, guid_128_cmp);
+ p_array_init(&brain->remote_mailbox_states, pool, 64);
return brain;
}
@@ -87,12 +87,14 @@
brain->state = DSYNC_STATE_SEND_MAILBOX_TREE;
if (sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE) {
- if (dsync_mailbox_states_import(&brain->mailbox_states, state,
+ if (dsync_mailbox_states_import(brain->mailbox_states,
+ brain->pool, state,
&error) < 0) {
- array_clear(&brain->mailbox_states);
+ hash_table_clear(brain->mailbox_states, FALSE);
i_error("Saved sync state is invalid, "
"falling back to full sync: %s", error);
- brain->sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL;
+ brain->sync_type = sync_type =
+ DSYNC_BRAIN_SYNC_TYPE_FULL;
} else {
brain->state = DSYNC_STATE_MASTER_SEND_LAST_COMMON;
}
@@ -142,8 +144,9 @@
dsync_brain_sync_mailbox_deinit(brain);
if (brain->local_tree_iter != NULL)
dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter);
-
- hash_table_destroy(&brain->remote_mailbox_states);
+ if (brain->mailbox_states_iter != NULL)
+ hash_table_iterate_deinit(&brain->mailbox_states_iter);
+ hash_table_destroy(&brain->mailbox_states);
ret = brain->failed ? -1 : 0;
pool_unref(&brain->pool);
@@ -184,22 +187,41 @@
static void dsync_brain_master_send_last_common(struct dsync_brain *brain)
{
- const struct dsync_mailbox_state *states;
- unsigned int count;
+ struct dsync_mailbox_state *state;
+ uint8_t *guid;
enum dsync_ibc_send_ret ret = DSYNC_IBC_SEND_RET_OK;
i_assert(brain->master_brain);
- states = array_get(&brain->mailbox_states, &count);
- while (brain->mailbox_state_idx < count) {
+ if (brain->mailbox_states_iter == NULL) {
+ brain->mailbox_states_iter =
+ hash_table_iterate_init(brain->mailbox_states);
+ }
+
+ for (;;) {
if (ret == DSYNC_IBC_SEND_RET_FULL)
return;
- ret = dsync_ibc_send_mailbox_state(brain->ibc,
- &states[brain->mailbox_state_idx++]);
+ if (!hash_table_iterate(brain->mailbox_states_iter,
+ brain->mailbox_states, &guid, &state))
+ break;
+ ret = dsync_ibc_send_mailbox_state(brain->ibc, state);
}
+ hash_table_iterate_deinit(&brain->mailbox_states_iter);
+
dsync_ibc_send_end_of_list(brain->ibc);
brain->state = DSYNC_STATE_SEND_MAILBOX_TREE;
- brain->mailbox_state_idx = 0;
+}
+
+static void dsync_mailbox_state_add(struct dsync_brain *brain,
+ const struct dsync_mailbox_state *state)
+{
+ struct dsync_mailbox_state *dupstate;
+ uint8_t *guid_p;
+
+ dupstate = p_new(brain->pool, struct dsync_mailbox_state, 1);
+ *dupstate = *state;
+ guid_p = dupstate->mailbox_guid;
+ hash_table_insert(brain->mailbox_states, guid_p, dupstate);
}
static bool dsync_brain_slave_recv_last_common(struct dsync_brain *brain)
@@ -211,7 +233,7 @@
i_assert(!brain->master_brain);
while ((ret = dsync_ibc_recv_mailbox_state(brain->ibc, &state)) > 0) {
- array_append(&brain->mailbox_states, &state, 1);
+ dsync_mailbox_state_add(brain, &state);
changed = TRUE;
}
if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
@@ -292,6 +314,39 @@
return ret;
}
+void dsync_brain_get_state(struct dsync_brain *brain, string_t *output)
+{
+ struct hash_iterate_context *iter;
+ struct dsync_mailbox_node *node;
+ const struct dsync_mailbox_state *new_state;
+ struct dsync_mailbox_state *state;
+ const uint8_t *guid_p;
+ uint8_t *guid;
+
+ /* update mailbox states */
+ array_foreach(&brain->remote_mailbox_states, new_state) {
+ guid_p = new_state->mailbox_guid;
+ state = hash_table_lookup(brain->mailbox_states, guid_p);
+ if (state != NULL)
+ *state = *new_state;
+ else
+ dsync_mailbox_state_add(brain, new_state);
+ }
+
+ /* remove nonexistent mailboxes */
+ iter = hash_table_iterate_init(brain->mailbox_states);
+ while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) {
+ node = dsync_mailbox_tree_lookup_guid(brain->local_mailbox_tree,
+ guid);
+ if (node == NULL ||
+ node->existence != DSYNC_MAILBOX_NODE_EXISTS)
+ hash_table_remove(brain->mailbox_states, guid);
+ }
+ hash_table_iterate_deinit(&iter);
+
+ dsync_mailbox_states_export(brain->mailbox_states, output);
+}
+
bool dsync_brain_has_failed(struct dsync_brain *brain)
{
return brain->failed;
diff -r 454d0563927d -r 6c850258002f src/doveadm/dsync/dsync-brain.h
--- a/src/doveadm/dsync/dsync-brain.h Sun Oct 21 10:01:54 2012 +0300
+++ b/src/doveadm/dsync/dsync-brain.h Sun Oct 21 12:20:30 2012 +0300
@@ -40,5 +40,8 @@
bool dsync_brain_run(struct dsync_brain *brain, bool *changed_r);
/* Returns TRUE if brain has failed, and there's no point in continuing. */
bool dsync_brain_has_failed(struct dsync_brain *brain);
More information about the dovecot-cvs
mailing list