dovecot-2.2: Added pop3-migration plugin for getting POP3 UIDLs ...
dovecot at dovecot.org
dovecot at dovecot.org
Sun May 20 03:26:30 EEST 2012
details: http://hg.dovecot.org/dovecot-2.2/rev/78317179b4af
changeset: 14398:78317179b4af
user: Timo Sirainen <tss at iki.fi>
date: Fri Mar 30 03:46:37 2012 +0300
description:
Added pop3-migration plugin for getting POP3 UIDLs from POP3 server.
The idea is to use this with dsync to migrate mails via imapc, but for
getting POP3 UIDLs via pop3c.
diffstat:
configure.in | 1 +
src/plugins/Makefile.am | 1 +
src/plugins/pop3-migration/Makefile.am | 17 +
src/plugins/pop3-migration/pop3-migration-plugin.c | 595 +++++++++++++++++++++
src/plugins/pop3-migration/pop3-migration-plugin.h | 7 +
5 files changed, 621 insertions(+), 0 deletions(-)
diffs (truncated from 653 to 300 lines):
diff -r 8a94981d8040 -r 78317179b4af configure.in
--- a/configure.in Fri Mar 30 03:43:08 2012 +0300
+++ b/configure.in Fri Mar 30 03:46:37 2012 +0300
@@ -2809,6 +2809,7 @@
src/plugins/listescape/Makefile
src/plugins/mail-log/Makefile
src/plugins/notify/Makefile
+src/plugins/pop3-migration/Makefile
src/plugins/quota/Makefile
src/plugins/imap-quota/Makefile
src/plugins/replication/Makefile
diff -r 8a94981d8040 -r 78317179b4af src/plugins/Makefile.am
--- a/src/plugins/Makefile.am Fri Mar 30 03:43:08 2012 +0300
+++ b/src/plugins/Makefile.am Fri Mar 30 03:46:37 2012 +0300
@@ -23,6 +23,7 @@
mail-log \
quota \
imap-quota \
+ pop3-migration \
replication \
snarf \
stats \
diff -r 8a94981d8040 -r 78317179b4af src/plugins/pop3-migration/Makefile.am
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/plugins/pop3-migration/Makefile.am Fri Mar 30 03:46:37 2012 +0300
@@ -0,0 +1,17 @@
+AM_CPPFLAGS = \
+ -I$(top_srcdir)/src/lib \
+ -I$(top_srcdir)/src/lib-mail \
+ -I$(top_srcdir)/src/lib-index \
+ -I$(top_srcdir)/src/lib-storage
+
+NOPLUGIN_LDFLAGS =
+lib05_pop3_migration_plugin_la_LDFLAGS = -module -avoid-version
+
+module_LTLIBRARIES = \
+ lib05_pop3_migration_plugin.la
+
+lib05_pop3_migration_plugin_la_SOURCES = \
+ pop3-migration-plugin.c
+
+noinst_HEADERS = \
+ pop3-migration-plugin.h
diff -r 8a94981d8040 -r 78317179b4af src/plugins/pop3-migration/pop3-migration-plugin.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/plugins/pop3-migration/pop3-migration-plugin.c Fri Mar 30 03:46:37 2012 +0300
@@ -0,0 +1,595 @@
+/* Copyright (c) 2007-2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "istream.h"
+#include "istream-header-filter.h"
+#include "sha1.h"
+#include "mail-namespace.h"
+#include "mail-search-build.h"
+#include "mail-storage-private.h"
+#include "pop3-migration-plugin.h"
+
+#define POP3_MIGRATION_CONTEXT(obj) \
+ MODULE_CONTEXT(obj, pop3_migration_storage_module)
+#define POP3_MIGRATION_MAIL_CONTEXT(obj) \
+ MODULE_CONTEXT(obj, pop3_migration_mail_module)
+
+struct pop3_uidl_map {
+ uint32_t pop3_seq;
+
+ /* UIDL */
+ const char *pop3_uidl;
+ /* LIST size */
+ uoff_t size;
+ /* sha1(TOP 0) - set only when needed */
+ unsigned char hdr_sha1[SHA1_RESULTLEN];
+ unsigned int hdr_sha1_set:1;
+ unsigned int imap_uid_found:1;
+};
+
+struct imap_msg_map {
+ uint32_t uid;
+ uoff_t psize;
+ const char *pop3_uidl;
+
+ /* sha1(header) - set only when needed */
+ unsigned char hdr_sha1[SHA1_RESULTLEN];
+ unsigned int hdr_sha1_set:1;
+};
+
+struct pop3_migration_mail_storage {
+ union mail_storage_module_context module_ctx;
+
+ const char *pop3_box_vname;
+ struct mailbox *pop3_box;
+ ARRAY_DEFINE(pop3_uidl_map, struct pop3_uidl_map);
+
+ unsigned int all_mailboxes:1;
+ unsigned int pop3_all_hdr_sha1_set:1;
+};
+
+struct pop3_migration_mailbox {
+ union mailbox_module_context module_ctx;
+
+ ARRAY_DEFINE(imap_msg_map, struct imap_msg_map);
+ unsigned int first_unfound_idx;
+
+ unsigned int uidl_synced:1;
+ unsigned int uidl_sync_failed:1;
+};
+
+static const char *hdr_hash_skip_headers[] = {
+ "Content-Length",
+ "Status",
+ "X-IMAP",
+ "X-IMAPbase",
+ "X-Keywords",
+ "X-Message-Flag",
+ "X-Status",
+ "X-UID",
+ "X-UIDL"
+};
+const char *pop3_migration_plugin_version = DOVECOT_VERSION;
+
+static MODULE_CONTEXT_DEFINE_INIT(pop3_migration_storage_module,
+ &mail_storage_module_register);
+static MODULE_CONTEXT_DEFINE_INIT(pop3_migration_mail_module,
+ &mail_module_register);
+
+static int imap_msg_map_uid_cmp(const struct imap_msg_map *map1,
+ const struct imap_msg_map *map2)
+{
+ if (map1->uid < map2->uid)
+ return -1;
+ if (map1->uid > map2->uid)
+ return 1;
+ return 0;
+}
+
+static int pop3_uidl_map_pop3_seq_cmp(const struct pop3_uidl_map *map1,
+ const struct pop3_uidl_map *map2)
+{
+ if (map1->pop3_seq < map2->pop3_seq)
+ return -1;
+ if (map1->pop3_seq > map2->pop3_seq)
+ return 1;
+ return 0;
+}
+
+static int pop3_uidl_map_hdr_cmp(const struct pop3_uidl_map *map1,
+ const struct pop3_uidl_map *map2)
+{
+ return memcmp(map1->hdr_sha1, map2->hdr_sha1, sizeof(map1->hdr_sha1));
+}
+
+static int imap_msg_map_hdr_cmp(const struct imap_msg_map *map1,
+ const struct imap_msg_map *map2)
+{
+ return memcmp(map1->hdr_sha1, map2->hdr_sha1, sizeof(map1->hdr_sha1));
+}
+
+static int get_hdr_sha1(struct mail *mail, unsigned char sha1[SHA1_RESULTLEN])
+{
+ struct istream *input;
+ const unsigned char *data;
+ size_t size;
+ struct sha1_ctxt sha1_ctx;
+
+ if (mail_get_hdr_stream(mail, NULL, &input) < 0) {
+ i_error("pop3_migration: Failed to get header for msg %u",
+ mail->seq);
+ return -1;
+ }
+ /* hide headers that might change or be different in IMAP vs. POP3 */
+ input = i_stream_create_header_filter(input,
+ HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR,
+ hdr_hash_skip_headers,
+ N_ELEMENTS(hdr_hash_skip_headers),
+ null_header_filter_callback, NULL);
+
+ sha1_init(&sha1_ctx);
+ while (i_stream_read_data(input, &data, &size, 0) > 0) {
+ sha1_loop(&sha1_ctx, data, size);
+ i_stream_skip(input, size);
+ }
+ if (input->stream_errno != 0) {
+ i_error("pop3_migration: Failed to read header for msg %u: %m",
+ mail->seq);
+ i_stream_unref(&input);
+ return -1;
+ }
+ sha1_result(&sha1_ctx, sha1);
+ i_stream_unref(&input);
+ return 0;
+}
+
+static int pop3_mailbox_open(struct mail_storage *storage)
+{
+ struct pop3_migration_mail_storage *mstorage =
+ POP3_MIGRATION_CONTEXT(storage);
+ struct mail_namespace *ns;
+
+ if (mstorage->pop3_box != NULL)
+ return 0;
+
+ ns = mail_namespace_find(storage->user->namespaces,
+ mstorage->pop3_box_vname);
+ if (ns == NULL) {
+ i_error("pop3_migration: Namespace not found for mailbox %s",
+ mstorage->pop3_box_vname);
+ return -1;
+ }
+ mstorage->pop3_box = mailbox_alloc(ns->list, mstorage->pop3_box_vname,
+ MAILBOX_FLAG_READONLY |
+ MAILBOX_FLAG_POP3_SESSION);
+ mstorage->all_mailboxes =
+ mail_user_plugin_getenv(storage->user,
+ "pop3_migration_all_mailboxes") != NULL;
+ return 0;
+}
+
+static int pop3_map_read(struct mail_storage *storage)
+{
+ struct pop3_migration_mail_storage *mstorage =
+ POP3_MIGRATION_CONTEXT(storage);
+ struct mailbox *pop3_box = mstorage->pop3_box;
+ struct mailbox_transaction_context *t;
+ struct mail_search_args *search_args;
+ struct mail_search_context *ctx;
+ struct mail *mail;
+ struct pop3_uidl_map *map;
+ const char *uidl;
+ uoff_t size;
+ int ret = 0;
+
+ if (array_is_created(&mstorage->pop3_uidl_map)) {
+ /* already read these, just reset the imap_uids */
+ array_foreach_modifiable(&mstorage->pop3_uidl_map, map)
+ map->imap_uid_found = FALSE;
+ return 0;
+ }
+ i_array_init(&mstorage->pop3_uidl_map, 128);
+
+ if (mailbox_sync(pop3_box, 0) < 0) {
+ i_error("pop3_migration: Couldn't sync mailbox %s: %s",
+ pop3_box->vname, mailbox_get_last_error(pop3_box, NULL));
+ return -1;
+ }
+
+ t = mailbox_transaction_begin(pop3_box, 0);
+ search_args = mail_search_build_init();
+ mail_search_build_add_all(search_args);
+ ctx = mailbox_search_init(t, search_args, NULL,
+ MAIL_FETCH_VIRTUAL_SIZE, NULL);
+ mail_search_args_unref(&search_args);
+
+ while (mailbox_search_next(ctx, &mail)) {
+ if (mail_get_virtual_size(mail, &size) < 0) {
+ i_error("pop3_migration: Failed to get size for msg %u: %s",
+ mail->seq,
+ mailbox_get_last_error(pop3_box, NULL));
+ ret = -1;
+ break;
+ }
+ if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &uidl) < 0) {
+ i_error("pop3_migration: Failed to get UIDL for msg %u: %s",
+ mail->seq,
+ mailbox_get_last_error(pop3_box, NULL));
+ ret = -1;
+ break;
+ }
+ if (*uidl == '\0') {
+ i_warning("pop3_migration: UIDL for msg %u is empty",
+ mail->seq);
+ continue;
+ }
+
+ map = array_append_space(&mstorage->pop3_uidl_map);
+ map->pop3_seq = mail->seq;
+ map->pop3_uidl = p_strdup(storage->pool, uidl);
+ map->size = size;
+ }
+
+ if (mailbox_search_deinit(&ctx) < 0)
+ ret = -1;
+ (void)mailbox_transaction_commit(&t);
+ return ret;
+}
+
+static int pop3_map_read_hdr_hashes(struct mail_storage *storage,
+ unsigned first_seq)
+{
+ struct pop3_migration_mail_storage *mstorage =
+ POP3_MIGRATION_CONTEXT(storage);
+ struct mailbox_transaction_context *t;
+ struct mail_search_args *search_args;
+ struct mail_search_context *ctx;
+ struct mail *mail;
+ struct pop3_uidl_map *map;
+ int ret = 0;
+
+ if (mstorage->pop3_all_hdr_sha1_set)
+ return 0;
More information about the dovecot-cvs
mailing list