dovecot-2.0: Split dbox (single-dbox) and mdbox (multi-dbox) int...
dovecot at dovecot.org
dovecot at dovecot.org
Wed Oct 7 02:25:08 EEST 2009
details: http://hg.dovecot.org/dovecot-2.0/rev/0bb321c347ae
changeset: 9977:0bb321c347ae
user: Timo Sirainen <tss at iki.fi>
date: Tue Oct 06 19:22:42 2009 -0400
description:
Split dbox (single-dbox) and mdbox (multi-dbox) into separate storage backends.
This cleans up the code, makes it faster and also fixes some bugs.
Super-fast maildir migration code was also dropped, at least for now.
diffstat:
63 files changed, 8328 insertions(+), 7228 deletions(-)
configure.in | 16
src/lib-storage/index/Makefile.am | 2
src/lib-storage/index/dbox-common/Makefile.am | 32
src/lib-storage/index/dbox-common/dbox-file-fix.c | 354 ++++
src/lib-storage/index/dbox-common/dbox-file.c | 755 ++++++++
src/lib-storage/index/dbox-common/dbox-file.h | 194 ++
src/lib-storage/index/dbox-common/dbox-mail.c | 230 ++
src/lib-storage/index/dbox-common/dbox-mail.h | 29
src/lib-storage/index/dbox-common/dbox-save.c | 138 +
src/lib-storage/index/dbox-common/dbox-save.h | 30
src/lib-storage/index/dbox-common/dbox-storage.c | 459 +++++
src/lib-storage/index/dbox-common/dbox-storage.h | 71
src/lib-storage/index/dbox-common/dbox-sync-rebuild.c | 132 +
src/lib-storage/index/dbox-common/dbox-sync-rebuild.h | 27
src/lib-storage/index/dbox-multi/Makefile.am | 38
src/lib-storage/index/dbox-multi/mdbox-file-purge.c | 227 ++
src/lib-storage/index/dbox-multi/mdbox-file.c | 263 +++
src/lib-storage/index/dbox-multi/mdbox-file.h | 27
src/lib-storage/index/dbox-multi/mdbox-mail.c | 205 ++
src/lib-storage/index/dbox-multi/mdbox-map-private.h | 57
src/lib-storage/index/dbox-multi/mdbox-map.c | 1143 +++++++++++++
src/lib-storage/index/dbox-multi/mdbox-map.h | 93 +
src/lib-storage/index/dbox-multi/mdbox-save.c | 353 ++++
src/lib-storage/index/dbox-multi/mdbox-settings.c | 65
src/lib-storage/index/dbox-multi/mdbox-settings.h | 14
src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 777 +++++++++
src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.h | 6
src/lib-storage/index/dbox-multi/mdbox-storage.c | 505 ++++++
src/lib-storage/index/dbox-multi/mdbox-storage.h | 86 +
src/lib-storage/index/dbox-multi/mdbox-sync.c | 349 ++++
src/lib-storage/index/dbox-multi/mdbox-sync.h | 39
src/lib-storage/index/dbox-single/Makefile.am | 32
src/lib-storage/index/dbox-single/sdbox-file.c | 120 +
src/lib-storage/index/dbox-single/sdbox-file.h | 22
src/lib-storage/index/dbox-single/sdbox-mail.c | 109 +
src/lib-storage/index/dbox-single/sdbox-save.c | 296 +++
src/lib-storage/index/dbox-single/sdbox-storage.c | 378 ++++
src/lib-storage/index/dbox-single/sdbox-storage.h | 69
src/lib-storage/index/dbox-single/sdbox-sync-file.c | 67
src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c | 182 ++
src/lib-storage/index/dbox-single/sdbox-sync.c | 291 +++
src/lib-storage/index/dbox-single/sdbox-sync.h | 51
src/lib-storage/index/dbox/Makefile.am | 41
src/lib-storage/index/dbox/dbox-file-fix.c | 352 ----
src/lib-storage/index/dbox/dbox-file-maildir.c | 91 -
src/lib-storage/index/dbox/dbox-file-maildir.h | 9
src/lib-storage/index/dbox/dbox-file.c | 1033 ------------
src/lib-storage/index/dbox/dbox-file.h | 210 --
src/lib-storage/index/dbox/dbox-mail.c | 433 -----
src/lib-storage/index/dbox/dbox-map-private.h | 50
src/lib-storage/index/dbox/dbox-map.c | 1175 --------------
src/lib-storage/index/dbox/dbox-map.h | 97 -
src/lib-storage/index/dbox/dbox-save.c | 515 ------
src/lib-storage/index/dbox/dbox-settings.c | 65
src/lib-storage/index/dbox/dbox-settings.h | 14
src/lib-storage/index/dbox/dbox-storage-rebuild.c | 770 ---------
src/lib-storage/index/dbox/dbox-storage-rebuild.h | 6
src/lib-storage/index/dbox/dbox-storage.c | 925 -----------
src/lib-storage/index/dbox/dbox-storage.h | 123 -
src/lib-storage/index/dbox/dbox-sync-file.c | 368 ----
src/lib-storage/index/dbox/dbox-sync-rebuild.c | 470 -----
src/lib-storage/index/dbox/dbox-sync.c | 402 ----
src/lib-storage/index/dbox/dbox-sync.h | 74
diffs (truncated from 15842 to 300 lines):
diff -r ff4c83871f4d -r 0bb321c347ae configure.in
--- a/configure.in Tue Oct 06 13:24:01 2009 -0400
+++ b/configure.in Tue Oct 06 19:22:42 2009 -0400
@@ -216,12 +216,12 @@ AC_ARG_WITH(gc,
AC_ARG_WITH(storages,
[ --with-storages Build with specified mail storage formats
- (maildir mbox dbox cydir)], [
+ (maildir mbox dbox mdbox cydir)], [
if test "$withval" = "yes" || test "$withval" = "no"; then
AC_MSG_ERROR([--with-storages needs storage list as parameter])
fi
mail_storages="shared `echo "$withval"|sed 's/,/ /g'`" ],
- mail_storages="shared maildir mbox dbox cydir")
+ mail_storages="shared maildir mbox dbox mdbox cydir")
AC_SUBST(mail_storages)
AC_ARG_WITH(sql-drivers,
@@ -2273,7 +2273,9 @@ dnl **
maildir_libs='$(top_builddir)/src/lib-storage/index/maildir/libstorage_maildir.la'
mbox_libs='$(top_builddir)/src/lib-storage/index/mbox/libstorage_mbox.la'
-dbox_libs='$(top_builddir)/src/lib-storage/index/dbox/libstorage_dbox.la'
+dbox_common_libs='$(top_builddir)/src/lib-storage/index/dbox-common/libstorage_dbox_common.la'
+dbox_libs='$(top_builddir)/src/lib-storage/index/dbox-single/libstorage_dbox_single.la'
+mdbox_libs='$(top_builddir)/src/lib-storage/index/dbox-multi/libstorage_dbox_multi.la'
cydir_libs='$(top_builddir)/src/lib-storage/index/cydir/libstorage_cydir.la'
raw_libs='$(top_builddir)/src/lib-storage/index/raw/libstorage_raw.la'
shared_libs='$(top_builddir)/src/lib-storage/index/shared/libstorage_shared.la'
@@ -2287,6 +2289,10 @@ mail_storages=`(for i in $mail_storages;
mail_storages=`(for i in $mail_storages; do echo $i; done)|sort|uniq|xargs echo`
for storage in $mail_storages; do
LINKED_STORAGE_LIBS="$LINKED_STORAGE_LIBS `eval echo \\$${storage}_libs`"
+ if test $storage = dbox || test $storage = mdbox; then
+ LINKED_STORAGE_LIBS="$LINKED_STORAGE_LIBS $dbox_common_libs"
+ dbox_common_libs=""
+ fi
done
AC_SUBST(LINKED_STORAGE_LIBS)
AC_DEFINE_UNQUOTED(MAIL_STORAGES, "$mail_storages", List of compiled in mail storages)
@@ -2480,7 +2486,9 @@ src/lib-storage/index/Makefile
src/lib-storage/index/Makefile
src/lib-storage/index/maildir/Makefile
src/lib-storage/index/mbox/Makefile
-src/lib-storage/index/dbox/Makefile
+src/lib-storage/index/dbox-common/Makefile
+src/lib-storage/index/dbox-multi/Makefile
+src/lib-storage/index/dbox-single/Makefile
src/lib-storage/index/cydir/Makefile
src/lib-storage/index/raw/Makefile
src/lib-storage/index/shared/Makefile
diff -r ff4c83871f4d -r 0bb321c347ae src/lib-storage/index/Makefile.am
--- a/src/lib-storage/index/Makefile.am Tue Oct 06 13:24:01 2009 -0400
+++ b/src/lib-storage/index/Makefile.am Tue Oct 06 19:22:42 2009 -0400
@@ -1,4 +1,4 @@ SUBDIRS = maildir mbox dbox cydir raw sh
-SUBDIRS = maildir mbox dbox cydir raw shared
+SUBDIRS = maildir mbox dbox-common dbox-multi dbox-single cydir raw shared
noinst_LTLIBRARIES = libstorage_index.la
diff -r ff4c83871f4d -r 0bb321c347ae src/lib-storage/index/dbox-common/Makefile.am
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-storage/index/dbox-common/Makefile.am Tue Oct 06 19:22:42 2009 -0400
@@ -0,0 +1,32 @@
+noinst_LTLIBRARIES = libstorage_dbox_common.la
+
+AM_CPPFLAGS = \
+ -I$(top_srcdir)/src/lib \
+ -I$(top_srcdir)/src/lib-settings \
+ -I$(top_srcdir)/src/lib-mail \
+ -I$(top_srcdir)/src/lib-imap \
+ -I$(top_srcdir)/src/lib-index \
+ -I$(top_srcdir)/src/lib-storage \
+ -I$(top_srcdir)/src/lib-storage/index
+
+libstorage_dbox_common_la_SOURCES = \
+ dbox-file.c \
+ dbox-file-fix.c \
+ dbox-mail.c \
+ dbox-save.c \
+ dbox-storage.c \
+ dbox-sync-rebuild.c
+
+headers = \
+ dbox-file.h \
+ dbox-mail.h \
+ dbox-save.h \
+ dbox-storage.h \
+ dbox-sync-rebuild.h
+
+if INSTALL_HEADERS
+ pkginc_libdir=$(pkgincludedir)
+ pkginc_lib_HEADERS = $(headers)
+else
+ noinst_HEADERS = $(headers)
+endif
diff -r ff4c83871f4d -r 0bb321c347ae src/lib-storage/index/dbox-common/dbox-file-fix.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-storage/index/dbox-common/dbox-file-fix.c Tue Oct 06 19:22:42 2009 -0400
@@ -0,0 +1,354 @@
+/* Copyright (c) 2009 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "ostream.h"
+#include "str-find.h"
+#include "hex-binary.h"
+#include "message-size.h"
+#include "dbox-storage.h"
+#include "dbox-file.h"
+
+#include <stdio.h>
+
+#define DBOX_MAIL_FILE_BROKEN_COPY_SUFFIX ".broken"
+
+static int
+dbox_file_find_next_magic(struct dbox_file *file, uoff_t *offset_r, bool *pre_r)
+{
+ struct istream *input = file->input;
+ struct str_find_context *pre_ctx, *post_ctx;
+ uoff_t orig_offset, pre_offset, post_offset;
+ const unsigned char *data;
+ size_t size;
+ int ret;
+
+ *pre_r = FALSE;
+
+ pre_ctx = str_find_init(default_pool, "\n"DBOX_MAGIC_PRE);
+ post_ctx = str_find_init(default_pool, DBOX_MAGIC_POST);
+
+ /* \n isn't part of the DBOX_MAGIC_PRE, but it always preceds it.
+ assume that at this point we've already just read the \n. when
+ scanning for it later we'll need to find the \n though. */
+ str_find_more(pre_ctx, (const unsigned char *)"\n", 1);
+
+ orig_offset = input->v_offset;
+ while ((ret = i_stream_read_data(input, &data, &size, 0)) > 0) {
+ pre_offset = (uoff_t)-1;
+ post_offset = (uoff_t)-1;
+ if (str_find_more(pre_ctx, data, size)) {
+ pre_offset = input->v_offset +
+ str_find_get_match_end_pos(pre_ctx) -
+ (strlen(DBOX_MAGIC_PRE) + 1);
+ *pre_r = TRUE;
+ }
+ if (str_find_more(post_ctx, data, size)) {
+ post_offset = input->v_offset +
+ str_find_get_match_end_pos(post_ctx) -
+ strlen(DBOX_MAGIC_POST);
+ if (pre_offset == (uoff_t)-1 ||
+ post_offset < pre_offset) {
+ pre_offset = post_offset;
+ *pre_r = FALSE;
+ }
+ }
+
+ if (pre_offset != (uoff_t)-1) {
+ if (*pre_r) {
+ /* LF isn't part of the magic */
+ pre_offset++;
+ }
+ *offset_r = pre_offset;
+ break;
+ }
+ i_stream_skip(input, size);
+ }
+ if (ret <= 0) {
+ i_assert(ret == -1);
+ if (input->stream_errno != 0)
+ dbox_file_set_syscall_error(file, "read()");
+ else {
+ ret = 0;
+ *offset_r = input->v_offset;
+ }
+ }
+ i_stream_seek(input, orig_offset);
+ str_find_deinit(&pre_ctx);
+ str_find_deinit(&post_ctx);
+ return ret;
+}
+
+static int
+stream_copy(struct dbox_file *file, struct ostream *output,
+ const char *path, uoff_t count)
+{
+ struct istream *input;
+ off_t bytes;
+
+ input = i_stream_create_limit(file->input, count);
+ bytes = o_stream_send_istream(output, input);
+ i_stream_unref(&input);
+
+ if (bytes < 0) {
+ mail_storage_set_critical(&file->storage->storage,
+ "o_stream_send_istream(%s, %s) failed: %m",
+ file->cur_path, path);
+ return -1;
+ }
+ if ((uoff_t)bytes != count) {
+ mail_storage_set_critical(&file->storage->storage,
+ "o_stream_send_istream(%s) copied only %"
+ PRIuUOFF_T" of %"PRIuUOFF_T" bytes",
+ path, bytes, count);
+ return -1;
+ }
+ return 0;
+}
+
+static void dbox_file_skip_broken_header(struct dbox_file *file)
+{
+ const unsigned int magic_len = strlen(DBOX_MAGIC_PRE);
+ const unsigned char *data;
+ size_t i, size;
+
+ /* if there's LF close to our position, assume that the header ends
+ there. */
+ data = i_stream_get_data(file->input, &size);
+ if (size > file->msg_header_size + 16)
+ size = file->msg_header_size + 16;
+ for (i = 0; i < size; i++) {
+ if (data[i] == '\n') {
+ i_stream_skip(file->input, i);
+ return;
+ }
+ }
+
+ /* skip at least the magic bytes if possible */
+ if (size > magic_len && memcmp(data, DBOX_MAGIC_PRE, magic_len) == 0)
+ i_stream_skip(file->input, magic_len);
+}
+
+static void
+dbox_file_copy_metadata(struct dbox_file *file, struct ostream *output,
+ bool *have_guid_r)
+{
+ const char *line;
+ uoff_t prev_offset = file->input->v_offset;
+
+ *have_guid_r = FALSE;
+ while ((line = i_stream_read_next_line(file->input)) != NULL) {
+ if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') {
+ /* end of metadata */
+ return;
+ }
+ if (*line < 32) {
+ /* broken - possibly a new pre-magic block */
+ i_stream_seek(file->input, prev_offset);
+ return;
+ }
+ if (*line == DBOX_METADATA_VIRTUAL_SIZE) {
+ /* it may be wrong - recreate it */
+ continue;
+ }
+ if (*line == DBOX_METADATA_GUID)
+ *have_guid_r = TRUE;
+ o_stream_send_str(output, line);
+ o_stream_send_str(output, "\n");
+ }
+}
+
+static int
+dbox_file_fix_write_stream(struct dbox_file *file, uoff_t start_offset,
+ const char *temp_path, struct ostream *output)
+{
+ struct dbox_message_header msg_hdr;
+ uoff_t offset, msg_size, hdr_offset, body_offset;
+ bool pre, write_header, have_guid;
+ struct message_size body;
+ struct istream *body_input;
+ uint8_t guid_128[MAIL_GUID_128_SIZE];
+ int ret;
+
+ i_stream_seek(file->input, 0);
+ if (start_offset > 0) {
+ /* copy the valid data */
+ if (stream_copy(file, output, temp_path, start_offset) < 0)
+ return -1;
+ } else {
+ /* the file header is broken. recreate it */
+ if (dbox_file_header_write(file, output) < 0) {
+ dbox_file_set_syscall_error(file, "write()");
+ return -1;
+ }
+ }
+
+ while ((ret = dbox_file_find_next_magic(file, &offset, &pre)) > 0) {
+ msg_size = offset - file->input->v_offset;
+ if (msg_size < 256 && pre) {
+ /* probably some garbage or some broken headers.
+ we most likely don't miss anything by skipping
+ over this data. */
+ i_stream_skip(file->input, msg_size);
+ hdr_offset = file->input->v_offset;
+ ret = dbox_file_read_mail_header(file, &msg_size);
+ if (ret <= 0) {
+ if (ret < 0)
+ return -1;
+ dbox_file_skip_broken_header(file);
+ body_offset = file->input->v_offset;
+ msg_size = (uoff_t)-1;
More information about the dovecot-cvs
mailing list