dovecot-2.0: dsync: Initial implementation.
dovecot at dovecot.org
dovecot at dovecot.org
Tue Jun 30 05:04:32 EEST 2009
details: http://hg.dovecot.org/dovecot-2.0/rev/34eef8a2716b
changeset: 9559:34eef8a2716b
user: Timo Sirainen <tss at iki.fi>
date: Mon Jun 29 22:04:23 2009 -0400
description:
dsync: Initial implementation.
diffstat:
28 files changed, 4981 insertions(+), 1 deletion(-)
.hgignore | 3
TODO | 8
configure.in | 1
src/Makefile.am | 1
src/dsync/Makefile.am | 68 ++
src/dsync/dsync-brain-private.h | 93 +++
src/dsync/dsync-brain.c | 706 ++++++++++++++++++++++++++++
src/dsync/dsync-brain.h | 12
src/dsync/dsync-data.c | 79 +++
src/dsync/dsync-data.h | 41 +
src/dsync/dsync-proxy-client.c | 571 +++++++++++++++++++++++
src/dsync/dsync-proxy-server-cmd.c | 309 ++++++++++++
src/dsync/dsync-proxy-server.c | 179 +++++++
src/dsync/dsync-proxy-server.h | 35 +
src/dsync/dsync-proxy.c | 191 +++++++
src/dsync/dsync-proxy.h | 27 +
src/dsync/dsync-worker-local.c | 757 +++++++++++++++++++++++++++++++
src/dsync/dsync-worker-private.h | 71 ++
src/dsync/dsync-worker.c | 152 ++++++
src/dsync/dsync-worker.h | 97 +++
src/dsync/dsync.c | 155 ++++++
src/dsync/test-dsync-brain.c | 376 +++++++++++++++
src/dsync/test-dsync-common.c | 51 ++
src/dsync/test-dsync-common.h | 17
src/dsync/test-dsync-proxy-server-cmd.c | 365 ++++++++++++++
src/dsync/test-dsync-proxy.c | 149 ++++++
src/dsync/test-dsync-worker.c | 385 +++++++++++++++
src/dsync/test-dsync-worker.h | 83 +++
diffs (truncated from 5129 to 300 lines):
diff -r 320d2164bc17 -r 34eef8a2716b .hgignore
--- a/.hgignore Mon Jun 29 21:57:54 2009 -0400
+++ b/.hgignore Mon Jun 29 22:04:23 2009 -0400
@@ -59,6 +59,7 @@ src/config/doveconf
src/config/doveconf
src/lda/dovecot-lda
src/dict/dict
+src/dsync/dsync
src/imap-login/imap-login
src/imap/imap
src/lib/unicodemap.c
@@ -80,7 +81,7 @@ src/util/dovecotpw
src/util/dovecotpw
src/util/gdbhelper
src/util/idxview
-src/util/imap_utf7
+src/util/imap-utf7
src/util/listview
src/util/logview
src/util/maildirlock
diff -r 320d2164bc17 -r 34eef8a2716b TODO
--- a/TODO Mon Jun 29 21:57:54 2009 -0400
+++ b/TODO Mon Jun 29 22:04:23 2009 -0400
@@ -1,3 +1,10 @@
+ - dsync:
+ - do copying before expunges
+ - subscriptions
+ - cache
+ - Add mailbox_list_create_dir() and remove bool directory from mailbox_create()
+doveconf: Fatal: Error in configuration file /usr/local/etc/dovecot2.conf: Namespace '0RealMails/': alias_for points to unknown namespace: 0lol
+ - protocol lda { recipient_delimiter=+ }
- search: use mail_get_parts() only when it's already cached. if it's not,
add it to cache afterwards.
- mail_storage_service_multi_*() doesn't support giving module/service to
@@ -16,6 +23,7 @@
.libs/*.a changes not
- o_stream_set_file_path()
- dict pooling
+ - master: if many service processes (login) are idling, kill some of them
/* currently non-external transactions can be applied multiple times,
causing multiple increments. */
diff -r 320d2164bc17 -r 34eef8a2716b configure.in
--- a/configure.in Mon Jun 29 21:57:54 2009 -0400
+++ b/configure.in Mon Jun 29 22:04:23 2009 -0400
@@ -2471,6 +2471,7 @@ src/anvil/Makefile
src/anvil/Makefile
src/auth/Makefile
src/config/Makefile
+src/dsync/Makefile
src/lda/Makefile
src/log/Makefile
src/lmtp/Makefile
diff -r 320d2164bc17 -r 34eef8a2716b src/Makefile.am
--- a/src/Makefile.am Mon Jun 29 21:57:54 2009 -0400
+++ b/src/Makefile.am Mon Jun 29 22:04:23 2009 -0400
@@ -32,6 +32,7 @@ SUBDIRS = \
log \
config \
util \
+ dsync \
plugins
test:
diff -r 320d2164bc17 -r 34eef8a2716b src/dsync/Makefile.am
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/dsync/Makefile.am Mon Jun 29 22:04:23 2009 -0400
@@ -0,0 +1,68 @@
+pkglibexecdir = $(libexecdir)/dovecot
+
+pkglibexec_PROGRAMS = dsync
+
+AM_CPPFLAGS = \
+ -I$(top_srcdir)/src/lib \
+ -I$(top_srcdir)/src/lib-test \
+ -I$(top_srcdir)/src/lib-master \
+ -I$(top_srcdir)/src/lib-mail \
+ -I$(top_srcdir)/src/lib-imap \
+ -I$(top_srcdir)/src/lib-index \
+ -I$(top_srcdir)/src/lib-storage
+
+dsync_LDADD = $(LIBDOVECOT_STORAGE) $(LIBDOVECOT) $(MODULE_LIBS)
+dsync_DEPENDENCIES = $(LIBDOVECOT_STORAGE) $(LIBDOVECOT)
+dsync_SOURCES = \
+ dsync.c \
+ dsync-brain.c \
+ dsync-data.c \
+ dsync-proxy.c \
+ dsync-proxy-client.c \
+ dsync-proxy-server.c \
+ dsync-proxy-server-cmd.c \
+ dsync-worker.c \
+ dsync-worker-local.c
+
+noinst_HEADERS = \
+ dsync-brain.h \
+ dsync-brain-private.h \
+ dsync-data.h \
+ dsync-proxy.h \
+ dsync-proxy-server.h \
+ dsync-worker.h \
+ dsync-worker-private.h \
+ test-dsync-common.h \
+ test-dsync-worker.h
+
+test_programs = \
+ test-dsync-brain \
+ test-dsync-proxy \
+ test-dsync-proxy-server-cmd
+
+noinst_PROGRAMS = $(test_programs)
+
+test_libs = \
+ test-dsync-common.o \
+ ../lib-test/libtest.la \
+ ../lib-mail/libmail.la \
+ ../lib-imap/libimap.la \
+ ../lib/liblib.la
+
+test_dsync_brain_SOURCES = test-dsync-brain.c
+test_dsync_brain_LDADD = test-dsync-worker.o dsync-data.o dsync-brain.o dsync-worker.o $(test_libs)
+test_dsync_brain_DEPENDENCIES = test-dsync-worker.o dsync-data.o dsync-brain.o dsync-worker.o $(test_libs)
+
+test_dsync_proxy_SOURCES = test-dsync-proxy.c
+test_dsync_proxy_LDADD = dsync-proxy.o $(test_libs)
+test_dsync_proxy_DEPENDENCIES = dsync-proxy.o $(test_libs)
+
+test_dsync_proxy_server_cmd_SOURCES = test-dsync-proxy-server-cmd.c
+test_dsync_proxy_server_cmd_LDADD = test-dsync-worker.o dsync-worker.o dsync-proxy.o dsync-proxy-server-cmd.o $(test_libs)
+test_dsync_proxy_server_cmd_DEPENDENCIES = test-dsync-worker.o dsync-worker.o dsync-proxy.o dsync-proxy-server-cmd.o $(test_libs)
+
+check: check-am check-test
+check-test: $(test_programs)
+ for bin in $(test_programs); do \
+ if ! ./$$bin; then exit 1; fi; \
+ done
diff -r 320d2164bc17 -r 34eef8a2716b src/dsync/dsync-brain-private.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/dsync/dsync-brain-private.h Mon Jun 29 22:04:23 2009 -0400
@@ -0,0 +1,93 @@
+#ifndef DSYNC_BRAIN_PRIVATE_H
+#define DSYNC_BRAIN_PRIVATE_H
+
+#include "dsync-data.h"
+#include "dsync-brain.h"
+
+enum dsync_state {
+ DSYNC_STATE_GET_MAILBOXES = 0,
+ DSYNC_STATE_CREATE_MAILBOXES,
+ DSYNC_STATE_SYNC_EXISTING_MSGS,
+ DSYNC_STATE_SYNC_NEW_MSGS,
+ DSYNC_STATE_SYNC_RETRY_COPIES,
+ DSYNC_STATE_SYNC_UPDATE_MAILBOX,
+ DSYNC_STATE_SYNC_RESOLVE_UID_CONFLICTS,
+ DSYNC_STATE_SYNC_FLUSH,
+ DSYNC_STATE_SYNC_END
+};
+
+struct dsync_brain_mailbox_list {
+ pool_t pool;
+ struct dsync_brain *brain;
+ struct dsync_worker *worker;
+ struct dsync_worker_mailbox_iter *iter;
+ ARRAY_DEFINE(mailboxes, struct dsync_mailbox *);
+};
+
+struct dsync_brain_guid_instance {
+ struct dsync_brain_guid_instance *next;
+ uint32_t uid;
+ /* mailbox index in dsync_brain_mailbox_list.mailboxes */
+ unsigned int mailbox_idx:31;
+ unsigned int failed:1;
+};
+
+struct dsync_brain_msg_iter {
+ struct dsync_brain_mailbox_sync *sync;
+ struct dsync_worker *worker;
+
+ unsigned int wanted_mailbox_idx;
+
+ struct dsync_worker_msg_iter *iter;
+ struct dsync_message msg;
+ unsigned int mailbox_idx;
+
+ unsigned int save_guids:1;
+};
+
+struct dsync_brain_uid_conflict {
+ uint32_t mailbox_idx;
+ uint32_t uid;
+};
+
+struct dsync_brain_new_msg {
+ uint32_t mailbox_idx;
+ struct dsync_message *msg;
+};
+
+struct dsync_brain_mailbox_sync {
+ struct dsync_brain *brain;
+ pool_t pool;
+
+ /* char *guid -> struct dsync_brain_guid_instance* */
+ struct hash_table *guid_hash;
+
+ struct dsync_brain_msg_iter *src_msg_iter;
+ struct dsync_brain_msg_iter *dest_msg_iter;
+
+ ARRAY_DEFINE(uid_conflicts, struct dsync_brain_uid_conflict);
+ ARRAY_DEFINE(new_msgs, struct dsync_brain_new_msg);
+ unsigned int next_new_msg;
+
+ /* copy operations that failed. indexes point to new_msgs array */
+ ARRAY_TYPE(uint32_t) copy_retry_indexes;
+ unsigned int copy_results_left;
+
+ unsigned int uid_conflict:1;
+};
+
+struct dsync_brain {
+ struct dsync_worker *src_worker;
+ struct dsync_worker *dest_worker;
+
+ enum dsync_state state;
+
+ struct dsync_brain_mailbox_list *src_mailbox_list;
+ struct dsync_brain_mailbox_list *dest_mailbox_list;
+
+ struct dsync_brain_mailbox_sync *mailbox_sync;
+
+ unsigned int failed:1;
+};
+
+#endif
diff -r 320d2164bc17 -r 34eef8a2716b src/dsync/dsync-brain.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/dsync/dsync-brain.c Mon Jun 29 22:04:23 2009 -0400
@@ -0,0 +1,706 @@
+/* Copyright (c) 2009 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "master-service.h"
+#include "dsync-worker.h"
+#include "dsync-brain-private.h"
+
+static void
+dsync_brain_mailbox_list_deinit(struct dsync_brain_mailbox_list **list);
+static void dsync_brain_msg_sync_deinit(struct dsync_brain_mailbox_sync **sync);
+
+struct dsync_brain *dsync_brain_init(struct dsync_worker *src_worker,
+ struct dsync_worker *dest_worker)
+{
+ struct dsync_brain *brain;
+
+ brain = i_new(struct dsync_brain, 1);
+ brain->src_worker = src_worker;
+ brain->dest_worker = dest_worker;
+ return brain;
+}
+
+static void dsync_brain_fail(struct dsync_brain *brain)
+{
+ brain->failed = TRUE;
+ master_service_stop(master_service);
+}
+
+int dsync_brain_deinit(struct dsync_brain **_brain)
+{
+ struct dsync_brain *brain = *_brain;
+ int ret = brain->failed ? -1 : 0;
+
+ if (brain->mailbox_sync != NULL)
+ dsync_brain_msg_sync_deinit(&brain->mailbox_sync);
+ if (brain->src_mailbox_list != NULL)
+ dsync_brain_mailbox_list_deinit(&brain->src_mailbox_list);
+ if (brain->dest_mailbox_list != NULL)
+ dsync_brain_mailbox_list_deinit(&brain->dest_mailbox_list);
+
+ *_brain = NULL;
+ i_free(brain);
+ return ret;
+}
+
+static void dsync_brain_mailbox_list_finished(struct dsync_brain *brain)
+{
+ if (brain->src_mailbox_list->iter != NULL ||
+ brain->dest_mailbox_list->iter != NULL)
+ return;
+
+ /* both lists are finished */
+ brain->state++;
+ dsync_brain_sync(brain);
+}
+
+static void dsync_worker_mailbox_input(void *context)
+{
+ struct dsync_brain_mailbox_list *list = context;
+ struct dsync_mailbox dsync_box, *dup_box;
+ int ret;
More information about the dovecot-cvs
mailing list