dovecot-2.0: lib-storage: Added support for saving mail attachme...
dovecot at dovecot.org
dovecot at dovecot.org
Tue Oct 19 20:54:58 EEST 2010
details: http://hg.dovecot.org/dovecot-2.0/rev/28eaaa23f2c6
changeset: 12312:28eaaa23f2c6
user: Timo Sirainen <tss at iki.fi>
date: Tue Oct 19 18:47:17 2010 +0100
description:
lib-storage: Added support for saving mail attachments separately via filesystem API.
Currently this works only with sdbox and mdbox backends.
diffstat:
src/config/settings-get.pl | 1 +
src/lib-storage/index/Makefile.am | 5 +
src/lib-storage/index/cydir/cydir-storage.c | 1 +
src/lib-storage/index/dbox-common/Makefile.am | 3 +
src/lib-storage/index/dbox-common/dbox-attachment.c | 235 +++++++
src/lib-storage/index/dbox-common/dbox-attachment.h | 20 +
src/lib-storage/index/dbox-common/dbox-file.c | 4 +
src/lib-storage/index/dbox-common/dbox-save.c | 15 +-
src/lib-storage/index/dbox-common/dbox-storage.c | 38 +
src/lib-storage/index/dbox-common/dbox-storage.h | 11 +
src/lib-storage/index/dbox-multi/mdbox-purge.c | 111 ++-
src/lib-storage/index/dbox-multi/mdbox-storage.c | 11 +-
src/lib-storage/index/dbox-single/Makefile.am | 1 +
src/lib-storage/index/dbox-single/sdbox-copy.c | 76 ++
src/lib-storage/index/dbox-single/sdbox-file.c | 183 +++++
src/lib-storage/index/dbox-single/sdbox-file.h | 17 +
src/lib-storage/index/dbox-single/sdbox-save.c | 35 +-
src/lib-storage/index/dbox-single/sdbox-storage.c | 112 ++-
src/lib-storage/index/dbox-single/sdbox-storage.h | 1 +
src/lib-storage/index/dbox-single/sdbox-sync.c | 9 +-
src/lib-storage/index/index-attachment.c | 788 +++++++++++++++++++++++++
src/lib-storage/index/index-attachment.h | 40 +
src/lib-storage/index/index-storage.c | 2 +
src/lib-storage/index/istream-attachment.c | 123 +++
src/lib-storage/index/istream-attachment.h | 6 +
src/lib-storage/index/maildir/maildir-storage.c | 1 +
src/lib-storage/index/mbox/mbox-storage.c | 1 +
src/lib-storage/index/raw/raw-storage.c | 1 +
src/lib-storage/mail-storage-private.h | 10 +
src/lib-storage/mail-storage-settings.c | 29 +-
src/lib-storage/mail-storage-settings.h | 4 +
src/lib-storage/test-mailbox.c | 1 +
src/plugins/virtual/virtual-storage.c | 1 +
33 files changed, 1841 insertions(+), 55 deletions(-)
diffs (truncated from 2607 to 300 lines):
diff -r ce5bb3246ffb -r 28eaaa23f2c6 src/config/settings-get.pl
--- a/src/config/settings-get.pl Tue Oct 19 18:30:51 2010 +0100
+++ b/src/config/settings-get.pl Tue Oct 19 18:47:17 2010 +0100
@@ -6,6 +6,7 @@
print '#include "var-expand.h"'."\n";
print '#include "file-lock.h"'."\n";
print '#include "fsync-mode.h"'."\n";
+print '#include "hash-format.h"'."\n";
print '#include "settings-parser.h"'."\n";
print '#include "all-settings.h"'."\n";
print '#include <stddef.h>'."\n";
diff -r ce5bb3246ffb -r 28eaaa23f2c6 src/lib-storage/index/Makefile.am
--- a/src/lib-storage/index/Makefile.am Tue Oct 19 18:30:51 2010 +0100
+++ b/src/lib-storage/index/Makefile.am Tue Oct 19 18:47:17 2010 +0100
@@ -5,13 +5,16 @@
AM_CPPFLAGS = \
-I$(top_srcdir)/src/lib \
-I$(top_srcdir)/src/lib-test \
+ -I$(top_srcdir)/src/lib-fs \
-I$(top_srcdir)/src/lib-mail \
-I$(top_srcdir)/src/lib-imap \
-I$(top_srcdir)/src/lib-index \
-I$(top_srcdir)/src/lib-storage
libstorage_index_la_SOURCES = \
+ istream-attachment.c \
istream-mail-stats.c \
+ index-attachment.c \
index-fetch.c \
index-mail.c \
index-mail-headers.c \
@@ -34,7 +37,9 @@
libstorage_index_la_DEPENDENCIES = @LINKED_STORAGE_LIBS@
headers = \
+ istream-attachment.h \
istream-mail-stats.h \
+ index-attachment.h \
index-mail.h \
index-search-result.h \
index-sort.h \
diff -r ce5bb3246ffb -r 28eaaa23f2c6 src/lib-storage/index/cydir/cydir-storage.c
--- a/src/lib-storage/index/cydir/cydir-storage.c Tue Oct 19 18:30:51 2010 +0100
+++ b/src/lib-storage/index/cydir/cydir-storage.c Tue Oct 19 18:47:17 2010 +0100
@@ -179,6 +179,7 @@
cydir_save_finish,
cydir_save_cancel,
mail_storage_copy,
+ NULL,
index_storage_is_inconsistent
}
};
diff -r ce5bb3246ffb -r 28eaaa23f2c6 src/lib-storage/index/dbox-common/Makefile.am
--- a/src/lib-storage/index/dbox-common/Makefile.am Tue Oct 19 18:30:51 2010 +0100
+++ b/src/lib-storage/index/dbox-common/Makefile.am Tue Oct 19 18:47:17 2010 +0100
@@ -3,6 +3,7 @@
AM_CPPFLAGS = \
-I$(top_srcdir)/src/lib \
-I$(top_srcdir)/src/lib-settings \
+ -I$(top_srcdir)/src/lib-fs \
-I$(top_srcdir)/src/lib-mail \
-I$(top_srcdir)/src/lib-imap \
-I$(top_srcdir)/src/lib-index \
@@ -10,6 +11,7 @@
-I$(top_srcdir)/src/lib-storage/index
libstorage_dbox_common_la_SOURCES = \
+ dbox-attachment.c \
dbox-file.c \
dbox-file-fix.c \
dbox-mail.c \
@@ -18,6 +20,7 @@
dbox-sync-rebuild.c
headers = \
+ dbox-attachment.h \
dbox-file.h \
dbox-mail.h \
dbox-save.h \
diff -r ce5bb3246ffb -r 28eaaa23f2c6 src/lib-storage/index/dbox-common/dbox-attachment.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-storage/index/dbox-common/dbox-attachment.c Tue Oct 19 18:47:17 2010 +0100
@@ -0,0 +1,235 @@
+/* Copyright (c) 2010 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "istream-concat.h"
+#include "str.h"
+#include "istream-attachment.h"
+#include "istream-base64-encoder.h"
+#include "dbox-file.h"
+#include "dbox-save.h"
+#include "dbox-attachment.h"
+
+enum dbox_attachment_decode_option {
+ DBOX_ATTACHMENT_DECODE_OPTION_NONE = '-',
+ DBOX_ATTACHMENT_DECODE_OPTION_BASE64 = 'B',
+ DBOX_ATTACHMENT_DECODE_OPTION_CRLF = 'C'
+};
+
+void dbox_attachment_save_write_metadata(struct mail_save_context *ctx,
+ string_t *str)
+{
+ const ARRAY_TYPE(mail_attachment_extref) *extrefs;
+ const struct mail_attachment_extref *extref;
+ bool add_space = FALSE;
+ unsigned int startpos;
+
+ extrefs = index_attachment_save_get_extrefs(ctx);
+ if (extrefs == NULL || array_count(extrefs) == 0)
+ return;
+
+ str_append_c(str, DBOX_METADATA_EXT_REF);
+ array_foreach(extrefs, extref) {
+ if (!add_space)
+ add_space = TRUE;
+ else
+ str_append_c(str, ' ');
+ str_printfa(str, "%"PRIuUOFF_T" %"PRIuUOFF_T" ",
+ extref->start_offset, extref->size);
+
+ startpos = str_len(str);
+ if (extref->base64_have_crlf)
+ str_append_c(str, DBOX_ATTACHMENT_DECODE_OPTION_CRLF);
+ if (extref->base64_blocks_per_line > 0) {
+ str_printfa(str, "%c%u",
+ DBOX_ATTACHMENT_DECODE_OPTION_BASE64,
+ extref->base64_blocks_per_line * 4);
+ }
+ if (startpos == str_len(str)) {
+ /* make it clear there are no options */
+ str_append_c(str, DBOX_ATTACHMENT_DECODE_OPTION_NONE);
+ }
+ str_append_c(str, ' ');
+ str_append(str, extref->path);
+ }
+ str_append_c(str, '\n');
+}
+
+static bool
+parse_extref_decode_options(const char *str,
+ struct mail_attachment_extref *extref)
+{
+ unsigned int num;
+
+ if (*str == DBOX_ATTACHMENT_DECODE_OPTION_NONE)
+ return str[1] == '\0';
+
+ while (*str != '\0') {
+ switch (*str) {
+ case DBOX_ATTACHMENT_DECODE_OPTION_BASE64:
+ str++; num = 0;
+ while (*str >= '0' && *str <= '9') {
+ num = num*10 + (*str-'0');
+ str++;
+ }
+ if (num == 0 || num % 4 != 0)
+ return FALSE;
+
+ extref->base64_blocks_per_line = num/4;
+ break;
+ case DBOX_ATTACHMENT_DECODE_OPTION_CRLF:
+ extref->base64_have_crlf = TRUE;
+ str++;
+ break;
+ default:
+ return FALSE;
+ }
+ }
+ return TRUE;
+}
+
+static bool
+dbox_attachment_parse_extref_real(const char *line, pool_t pool,
+ ARRAY_TYPE(mail_attachment_extref) *extrefs)
+{
+ struct mail_attachment_extref extref;
+ const char *const *args;
+ unsigned int i, len;
+ uoff_t last_voffset;
+
+ args = t_strsplit(line, " ");
+ len = str_array_length(args);
+ if ((len % 4) != 0)
+ return FALSE;
+
+ last_voffset = 0;
+ for (i = 0; args[i] != NULL; i += 4) {
+ const char *start_offset_str = args[i+0];
+ const char *size_str = args[i+1];
+ const char *decode_options = args[i+2];
+ const char *path = args[i+3];
+
+ memset(&extref, 0, sizeof(extref));
+ if (str_to_uoff(start_offset_str, &extref.start_offset) < 0 ||
+ str_to_uoff(size_str, &extref.size) < 0 ||
+ extref.start_offset < last_voffset ||
+ !parse_extref_decode_options(decode_options, &extref))
+ return FALSE;
+
+ last_voffset += extref.size +
+ (extref.start_offset - last_voffset);
+
+ extref.path = p_strdup(pool, path);
+ array_append(extrefs, &extref, 1);
+ }
+ return TRUE;
+}
+
+bool dbox_attachment_parse_extref(const char *line, pool_t pool,
+ ARRAY_TYPE(mail_attachment_extref) *extrefs)
+{
+ bool ret;
+
+ T_BEGIN {
+ ret = dbox_attachment_parse_extref_real(line, pool, extrefs);
+ } T_END;
+ return ret;
+}
+
+static int
+dbox_attachment_file_get_stream_from(struct dbox_file *file,
+ const char *ext_refs,
+ struct istream **stream_r)
+{
+ ARRAY_TYPE(mail_attachment_extref) extrefs_arr;
+ ARRAY_DEFINE(streams, struct istream *);
+ const struct mail_attachment_extref *extref;
+ struct istream **inputs, *input, *input2;
+ const char *path, *path_suffix;
+ uoff_t root_offset, last_voffset = 0;
+ unsigned int i;
+
+ t_array_init(&extrefs_arr, 16);
+ if (!dbox_attachment_parse_extref_real(ext_refs, pool_datastack_create(),
+ &extrefs_arr))
+ return 0;
+
+ root_offset = file->input->v_offset;
+ t_array_init(&streams, 8);
+ array_foreach(&extrefs_arr, extref) {
+ path_suffix = file->storage->v.get_attachment_path_suffix(file);
+ path = t_strdup_printf("%s/%s%s", file->storage->attachment_dir,
+ extref->path, path_suffix);
+
+ if (extref->start_offset != last_voffset) {
+ uoff_t part_size = extref->start_offset - last_voffset;
+
+ input = i_stream_create_limit(file->input, part_size);
+ array_append(&streams, &input, 1);
+ i_stream_seek(file->input,
+ file->input->v_offset + part_size);
+ last_voffset += part_size;
+ }
+
+ last_voffset += extref->size;
+ input2 = i_stream_create_file(path, IO_BLOCK_SIZE);
+
+ if (extref->base64_blocks_per_line > 0) {
+ input = i_stream_create_base64_encoder(input2,
+ extref->base64_blocks_per_line*4,
+ extref->base64_have_crlf);
+ i_stream_unref(&input2);
+ input2 = input;
+ }
+
+ input = i_stream_create_attachment(input2, extref->size);
+ i_stream_unref(&input2);
+ array_append(&streams, &input, 1);
+ }
+
+ if (file->cur_physical_size != file->input->v_offset-root_offset) {
+ uoff_t trailer_size = file->cur_physical_size -
+ (file->input->v_offset - root_offset);
+
+ input = i_stream_create_limit(file->input, trailer_size);
+ array_append(&streams, &input, 1);
+ (void)array_append_space(&streams);
+ }
+
+ inputs = array_idx_modifiable(&streams, 0);
+ *stream_r = i_stream_create_concat(inputs);
+ for (i = 0; inputs[i] != NULL; i++)
+ i_stream_unref(&inputs[i]);
+ return 1;
+}
+
+int dbox_attachment_file_get_stream(struct dbox_file *file,
+ struct istream **stream_r)
+{
+ const char *ext_refs;
+ int ret;
+
+ /* need to read metadata in case there are external references */
+ if ((ret = dbox_file_metadata_read(file)) <= 0)
+ return ret;
+
+ i_stream_seek(file->input, file->cur_offset + file->msg_header_size);
+
More information about the dovecot-cvs
mailing list