[dovecot-cvs] dovecot/src/lib-storage/index/mbox mbox-save.c, 1.57, 1.58 mbox-storage.c, 1.90, 1.91 mbox-storage.h, 1.27, 1.28 mbox-transaction.c, 1.4, 1.5

cras at dovecot.org cras at dovecot.org
Sun Aug 22 12:17:12 EEST 2004 

Update of /home/cvs/dovecot/src/lib-storage/index/mbox
In directory talvi:/tmp/cvs-serv2400/lib-storage/index/mbox

Modified Files:
	mbox-save.c mbox-storage.c mbox-storage.h mbox-transaction.c 
Log Message:
Changed mail saving API to be nonblocking.



Index: mbox-save.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-save.c,v
retrieving revision 1.57
retrieving revision 1.58
diff -u -d -r1.57 -r1.58
--- mbox-save.c	15 Aug 2004 03:40:32 -0000	1.57
+++ mbox-save.c	22 Aug 2004 09:17:09 -0000	1.58
@@ -3,15 +3,18 @@
 #include "lib.h"
 #include "ioloop.h"
 #include "hostpid.h"
+#include "istream.h"
 #include "ostream.h"
 #include "str.h"
 #include "write-full.h"
+#include "istream-header-filter.h"
+#include "ostream-crlf.h"
+#include "message-parser.h"
 #include "mbox-storage.h"
 #include "mbox-file.h"
 #include "mbox-from.h"
 #include "mbox-lock.h"
 #include "mbox-sync-private.h"
-#include "mail-save.h"
 
 #include <stddef.h>
 #include <stdlib.h>
@@ -21,26 +24,32 @@
 #include <netdb.h>
 
 struct mbox_save_context {
+	struct mail_save_context ctx;
+
 	struct index_mailbox *ibox;
 	struct mail_index_transaction *trans;
-	uoff_t append_offset;
+	uoff_t append_offset, mail_offset;
 
 	string_t *headers;
 	size_t space_end_idx;
-	uint32_t next_uid;
+	uint32_t seq, next_uid;
 
-	struct ostream *output;
-	uoff_t extra_hdr_offset, eoh_offset;
+	struct istream *input;
+	struct ostream *output, *body_output;
+	uoff_t extra_hdr_offset, eoh_offset, eoh_input_offset;
+	char last_char;
 
 	struct index_mail mail;
-
 	const struct mail_full_flags *flags;
+
 	unsigned int synced:1;
+	unsigned int failed:1;
+	unsigned int save_crlf:1;
 };
 
 static char my_hostdomain[256] = "";
 
-static int write_error(struct mbox_save_context *ctx)
+static void write_error(struct mbox_save_context *ctx)
 {
 	if (ENOSPACE(errno)) {
 		mail_storage_set_error(ctx->ibox->box.storage,
@@ -48,8 +57,6 @@
 	} else {
                 mbox_set_syscall_error(ctx->ibox, "write()");
 	}
-
-	return -1;
 }
 
 static int mbox_seek_to_end(struct mbox_save_context *ctx, uoff_t *offset)
@@ -73,8 +80,10 @@
 		return mbox_set_syscall_error(ctx->ibox, "read()");
 
 	if (ch != '\n') {
-		if (write_full(fd, "\n", 1) < 0)
-			return write_error(ctx);
+		if (write_full(fd, "\n", 1) < 0) {
+			write_error(ctx);
+			return -1;
+		}
 		*offset += 1;
 	}
 
@@ -83,16 +92,19 @@
 
 static int mbox_append_lf(struct mbox_save_context *ctx)
 {
-	if (o_stream_send(ctx->output, "\n", 1) < 0)
-		return write_error(ctx);
+	if (o_stream_send(ctx->output, "\n", 1) < 0) {
+		write_error(ctx);
+		return -1;
+	}
 
-	return TRUE;
+	return 0;
 }
 
 static int write_from_line(struct mbox_save_context *ctx, time_t received_date,
 			   const char *from_envelope)
 {
 	const char *line, *name;
+	int ret;
 
 	if (*my_hostdomain == '\0') {
 		struct hostent *hent;
@@ -108,6 +120,7 @@
 		strocpy(my_hostdomain, name, sizeof(my_hostdomain));
 	}
 
+	t_push();
 	if (from_envelope == NULL) {
 		from_envelope = t_strconcat(ctx->ibox->storage->user, "@",
 					    my_hostdomain, NULL);
@@ -116,10 +129,11 @@
 	/* save in local timezone, no matter what it was given with */
 	line = mbox_from_create(from_envelope, received_date);
 
-	if (o_stream_send_str(ctx->output, line) < 0)
-		return write_error(ctx);
+	if ((ret = o_stream_send_str(ctx->output, line)) < 0)
+		write_error(ctx);
+	t_pop();
 
-	return 0;
+	return ret;
 }
 
 static int mbox_write_content_length(struct mbox_save_context *ctx)
@@ -134,7 +148,7 @@
 	/* write Content-Length headers */
 	t_push();
 	str = t_strdup_printf("\nContent-Length: %s",
-			      dec2str(end_offset - (ctx->eoh_offset + 1)));
+			      dec2str(end_offset - ctx->eoh_offset));
 	len = strlen(str);
 
 	if (o_stream_seek(ctx->output, ctx->extra_hdr_offset +
@@ -155,25 +169,6 @@
 	return ret;
 }
 
-static int save_header_callback(const char *name, write_func_t *write_func,
-				void *context)
-{
-	struct mbox_save_context *ctx = context;
-
-	if (name == NULL) {
-		/* write our extra headers */
-		ctx->extra_hdr_offset = ctx->output->offset;
-		if (write_func(ctx->output, str_data(ctx->headers),
-			       str_len(ctx->headers)) < 0)
-			return -1;
-		ctx->eoh_offset = ctx->output->offset;
-		return 1;
-	}
-
-	return bsearch(name, mbox_hide_headers, mbox_hide_headers_count,
-		       sizeof(*mbox_hide_headers), bsearch_strcasecmp) == NULL;
-}
-
 static int mbox_save_init_sync(struct mbox_transaction_context *t)
 {
 	struct mbox_save_context *ctx = t->save_ctx;
@@ -237,36 +232,15 @@
 	str_append_c(ctx->headers, '\n');
 }
 
-int mbox_save(struct mailbox_transaction_context *_t,
-	      const struct mail_full_flags *flags,
-	      time_t received_date, int timezone_offset __attr_unused__,
-	      const char *from_envelope, struct istream *data,
-	      struct mail **mail_r)
+static int
+mbox_save_init_file(struct mbox_save_context *ctx,
+		    struct mbox_transaction_context *t, int want_mail)
 {
-	struct mbox_transaction_context *t =
-		(struct mbox_transaction_context *)_t;
-	struct index_mailbox *ibox = t->ictx.ibox;
-	struct mbox_save_context *ctx = t->save_ctx;
-	enum mail_flags save_flags;
-	keywords_mask_t keywords;
-	uint64_t offset;
-	uint32_t seq = 0;
+	struct index_mailbox *ibox = ctx->ibox;
 	int ret;
 
-	/* FIXME: we could write timezone_offset to From-line.. */
-	if (received_date == (time_t)-1)
-		received_date = ioloop_time;
-
-	if (ctx == NULL) {
-		ctx = t->save_ctx = i_new(struct mbox_save_context, 1);
-		ctx->ibox = ibox;
-		ctx->trans = t->ictx.trans;
-		ctx->append_offset = (uoff_t)-1;
-		ctx->headers = str_new(default_pool, 512);
-	}
-	ctx->flags = flags;
-
 	if (ctx->append_offset == (uoff_t)-1) {
+		/* first appended mail in this transaction */
 		if (ibox->mbox_lock_type != F_WRLCK) {
 			if (mbox_lock(ibox, F_WRLCK, &t->mbox_lock_id) <= 0)
 				return -1;
@@ -277,7 +251,7 @@
 				return -1;
 		}
 
-		if (mail_r == NULL) {
+		if (!want_mail) {
 			/* assign UIDs only if mbox doesn't require syncing */
 			ret = mbox_sync_has_changed(ibox);
 			if (ret < 0)
@@ -295,7 +269,7 @@
 						   0, FALSE);
 	}
 
-	if (!ctx->synced && mail_r != NULL) {
+	if (!ctx->synced && want_mail) {
 		/* we'll need to assign UID for the mail immediately. */
 		if (mbox_sync(ibox, FALSE, FALSE, FALSE) < 0)
 			return -1;
@@ -303,6 +277,58 @@
 			return -1;
 	}
 
+	return 0;
+}
+
+static void save_header_callback(struct message_header_line *hdr,
+				 int *matched __attr_unused__, void *context)
+{
+	struct mbox_save_context *ctx = context;
+
+	if ((hdr == NULL && ctx->eoh_input_offset == (uoff_t)-1) ||
+	    (hdr != NULL && hdr->eoh))
+		ctx->eoh_input_offset = ctx->input->v_offset;
+}
+
+struct mail_save_context *
+mbox_save_init(struct mailbox_transaction_context *_t,
+	       const struct mail_full_flags *flags,
+	       time_t received_date, int timezone_offset __attr_unused__,
+	       const char *from_envelope, struct istream *input, int want_mail)
+{
+	struct mbox_transaction_context *t =
+		(struct mbox_transaction_context *)_t;
+	struct index_mailbox *ibox = t->ictx.ibox;
+	struct mbox_save_context *ctx = t->save_ctx;
+	enum mail_flags save_flags;
+	keywords_mask_t keywords;
+	uint64_t offset;
+
+	/* FIXME: we could write timezone_offset to From-line.. */
+	if (received_date == (time_t)-1)
+		received_date = ioloop_time;
+
+	if (ctx == NULL) {
+		ctx = t->save_ctx = i_new(struct mbox_save_context, 1);
+		ctx->ctx.box = &ibox->box;
+		ctx->ibox = ibox;
+		ctx->trans = t->ictx.trans;
+		ctx->append_offset = (uoff_t)-1;
+		ctx->headers = str_new(default_pool, 512);
+		ctx->save_crlf = getenv("MAIL_SAVE_CRLF") != NULL;
+		ctx->mail_offset = (uoff_t)-1;
+	}
+
+	ctx->failed = FALSE;
+	ctx->seq = 0;
+
+	ctx->flags = flags;
+
+	if (mbox_save_init_file(ctx, t, want_mail) < 0) {
+		ctx->failed = TRUE;
+		return &ctx->ctx;
+	}
+
 	save_flags = (flags->flags & ~MAIL_RECENT) | MAIL_RECENT;
 	str_truncate(ctx->headers, 0);
 	if (ctx->synced) {
@@ -312,13 +338,13 @@
 
 		memset(keywords, 0, INDEX_KEYWORDS_BYTE_COUNT);
 		// FIXME: set keywords
-		mail_index_append(ctx->trans, ctx->next_uid, &seq);
-		mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE,
+		mail_index_append(ctx->trans, ctx->next_uid, &ctx->seq);
+		mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_REPLACE,
 					save_flags, keywords);
 
 		offset = ctx->output->offset == 0 ? 0 :
 			ctx->output->offset - 1;
-		mail_index_update_extra_rec(ctx->trans, seq,
+		mail_index_update_extra_rec(ctx->trans, ctx->seq,
 					    ibox->mbox_extra_idx, &offset);
 		ctx->next_uid++;
 	}
@@ -326,33 +352,159 @@
 				      save_flags ^ MBOX_NONRECENT);
 	mbox_save_append_keyword_headers(ctx, flags->keywords,
 					 flags->keywords_count);
+	str_append_c(ctx->headers, '\n');
 
 	i_assert(ibox->mbox_lock_type == F_WRLCK);
 
-	t_push();
-	if (write_from_line(ctx, received_date, from_envelope) < 0 ||
-	    mail_storage_save(ibox->box.storage, ibox->path, data, ctx->output,
-			      FALSE, getenv("MAIL_SAVE_CRLF") != NULL,
-			      save_header_callback, ctx) < 0 ||
-	    mbox_write_content_length(ctx) < 0 ||
-	    mbox_append_lf(ctx) < 0) {
-		ret = -1;
-	} else {
-		ret = 0;
+	ctx->mail_offset = ctx->output->offset;
+	ctx->eoh_input_offset = (uoff_t)-1;
+	ctx->eoh_offset = (uoff_t)-1;
+	ctx->last_char = '\n';
+
+	if (write_from_line(ctx, received_date, from_envelope) < 0)
+		ctx->failed = TRUE;
+	else {
+		ctx->input =
+			i_stream_create_header_filter(input, TRUE, FALSE,
+						      mbox_hide_headers,
+						      mbox_hide_headers_count,
+						      save_header_callback,
+						      ctx);
+		ctx->body_output = getenv("MAIL_SAVE_CRLF") != NULL ?
+			o_stream_create_crlf(default_pool, ctx->output) :
+			o_stream_create_lf(default_pool, ctx->output);
+	}
+
+	return &ctx->ctx;
+}
+
+int mbox_save_continue(struct mail_save_context *_ctx)
+{
+	struct mbox_save_context *ctx = (struct mbox_save_context *)_ctx;
+	const unsigned char *data;
+	size_t size, hdr_size;
+	ssize_t ret;
+
+	if (ctx->failed)
+		return -1;
+
+	if (ctx->eoh_offset != (uoff_t)-1) {
+		/* writing body */
+		if (o_stream_send_istream(ctx->body_output, ctx->input) < 0) {
+			ctx->failed = TRUE;
+			return -1;
+		}
+		return 0;
+	}
+
+	while ((ret = i_stream_read(ctx->input)) != -1) {
+		if (ret == 0)
+			return 0;
+
+		data = i_stream_get_data(ctx->input, &size);
+		if (ctx->eoh_input_offset != (uoff_t)-1 &&
+		    ctx->input->v_offset + size >= ctx->eoh_input_offset) {
+			/* found end of headers. write the rest of them. */
+			size = ctx->eoh_input_offset - ctx->input->v_offset;
+			if (o_stream_send(ctx->output, data, hdr_size) < 0) {
+				ctx->failed = TRUE;
+				return -1;
+			}
+			if (hdr_size > 0)
+				ctx->last_char = data[hdr_size-1];
+			i_stream_skip(ctx->input, size + 1);
+			break;
+		}
+
+		if (o_stream_send(ctx->output, data, size) < 0) {
+			ctx->failed = TRUE;
+			return -1;
+		}
+		ctx->last_char = data[size-1];
+		i_stream_skip(ctx->input, size);
+	}
+
+	if (ctx->last_char != '\n') {
+		if (o_stream_send(ctx->output, "\n", 1) < 0) {
+			ctx->failed = TRUE;
+			return -1;
+		}
+	}
+
+	/* append our own headers and ending empty line */
+	ctx->extra_hdr_offset = ctx->output->offset;
+	if (o_stream_send(ctx->output, str_data(ctx->headers),
+			  str_len(ctx->headers)) < 0) {
+		ctx->failed = TRUE;
+		return -1;
+	}
+	ctx->eoh_offset = ctx->output->offset;
+
+	/* write body */
+	return ctx->input->eof ? 0 : mbox_save_continue(_ctx);
+}
+
+int mbox_save_finish(struct mail_save_context *_ctx, struct mail **mail_r)
+{
+	struct mbox_save_context *ctx = (struct mbox_save_context *)_ctx;
+
+	if (!ctx->failed) {
+		if (mbox_write_content_length(ctx) < 0 ||
+		    mbox_append_lf(ctx) < 0)
+			ctx->failed = TRUE;
+	}
+
+	if (ctx->input != NULL) {
+		i_stream_unref(ctx->input);
+		ctx->input = NULL;
+	}
+	if (ctx->body_output != NULL) {
+		o_stream_unref(ctx->body_output);
+		ctx->body_output = NULL;
+	}
+
+	if (ctx->failed && ctx->mail_offset != (uoff_t)-1) {
+		/* saving this mail failed - truncate back to beginning of it */
+		if (ftruncate(ctx->ibox->mbox_fd, (off_t)ctx->mail_offset) < 0)
+			mbox_set_syscall_error(ctx->ibox, "ftruncate()");
+		ctx->mail_offset = (uoff_t)-1;
+	}
+
+	if (ctx->failed) {
+		errno = ctx->output->stream_errno;
+		if (ENOSPACE(errno)) {
+			mail_storage_set_error(ctx->ibox->box.storage,
+					       "Not enough disk space");
+		} else if (errno != 0) {
+			mail_storage_set_critical(ctx->ibox->box.storage,
+				"write(%s) failed: %m", ctx->ibox->path);
+		}
+		return -1;
 	}
-	t_pop();
 
 	if (mail_r != NULL) {
-		if (index_mail_next(&ctx->mail, seq) < 0)
+		i_assert(ctx->seq != 0);
+
+		if (index_mail_next(&ctx->mail, ctx->seq) < 0)
 			return -1;
 		*mail_r = &ctx->mail.mail;
 	}
 
-	return ret;
+	return 0;
 }
 
-static void mbox_save_deinit(struct mbox_save_context *ctx)
+void mbox_save_cancel(struct mail_save_context *_ctx)
+{
+	struct mbox_save_context *ctx = (struct mbox_save_context *)_ctx;
+
+	ctx->failed = TRUE;
+	(void)mbox_save_finish(_ctx, NULL);
+}
+
+static void mbox_transaction_save_deinit(struct mbox_save_context *ctx)
 {
+	i_assert(ctx->body_output == NULL);
+
 	if (ctx->mail.pool != NULL)
 		index_mail_deinit(&ctx->mail);
 
@@ -362,7 +514,7 @@
 	i_free(ctx);
 }
 
-int mbox_save_commit(struct mbox_save_context *ctx)
+int mbox_transaction_save_commit(struct mbox_save_context *ctx)
 {
 	int ret = 0;
 
@@ -379,11 +531,11 @@
 		}
 	}
 
-	mbox_save_deinit(ctx);
+	mbox_transaction_save_deinit(ctx);
 	return ret;
 }
 
-void mbox_save_rollback(struct mbox_save_context *ctx)
+void mbox_transaction_save_rollback(struct mbox_save_context *ctx)
 {
 	struct index_mailbox *ibox = ctx->ibox;
 
@@ -399,5 +551,5 @@
 			mbox_set_syscall_error(ibox, "ftruncate()");
 	}
 
-	mbox_save_deinit(ctx);
+	mbox_transaction_save_deinit(ctx);
 }

Index: mbox-storage.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-storage.c,v
retrieving revision 1.90
retrieving revision 1.91
diff -u -d -r1.90 -r1.91
--- mbox-storage.c	22 Jul 2004 21:20:01 -0000	1.90
+++ mbox-storage.c	22 Aug 2004 09:17:09 -0000	1.91
@@ -9,7 +9,7 @@
 #include "mbox-storage.h"
 #include "mbox-lock.h"
 #include "mbox-file.h"
-#include "mail-save.h"
+#include "mail-copy.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -838,7 +838,10 @@
 	index_storage_search_init,
 	index_storage_search_deinit,
 	index_storage_search_next,
-	mbox_save,
+	mbox_save_init,
+	mbox_save_continue,
+	mbox_save_finish,
+	mbox_save_cancel,
 	mail_storage_copy,
 	index_storage_is_inconsistent
 };

Index: mbox-storage.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-storage.h,v
retrieving revision 1.27
retrieving revision 1.28
diff -u -d -r1.27 -r1.28
--- mbox-storage.h	31 Jul 2004 00:54:48 -0000	1.27
+++ mbox-storage.h	22 Aug 2004 09:17:09 -0000	1.28
@@ -40,13 +40,17 @@
 struct mailbox_sync_context *
 mbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);
 
-int mbox_save(struct mailbox_transaction_context *t,
-	      const struct mail_full_flags *flags,
-	      time_t received_date, int timezone_offset,
-	      const char *from_envelope, struct istream *data,
-	      struct mail **mail_r);
-int mbox_save_commit(struct mbox_save_context *ctx);
-void mbox_save_rollback(struct mbox_save_context *ctx);
+struct mail_save_context *
+mbox_save_init(struct mailbox_transaction_context *_t,
+	       const struct mail_full_flags *flags,
+	       time_t received_date, int timezone_offset,
+	       const char *from_envelope, struct istream *input, int want_mail);
+int mbox_save_continue(struct mail_save_context *ctx);
+int mbox_save_finish(struct mail_save_context *ctx, struct mail **mail_r);
+void mbox_save_cancel(struct mail_save_context *ctx);
+
+int mbox_transaction_save_commit(struct mbox_save_context *ctx);
+void mbox_transaction_save_rollback(struct mbox_save_context *ctx);
 
 int mbox_is_valid_mask(const char *mask);
 

Index: mbox-transaction.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-transaction.c,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- mbox-transaction.c	22 Jun 2004 07:36:33 -0000	1.4
+++ mbox-transaction.c	22 Aug 2004 09:17:09 -0000	1.5
@@ -25,7 +25,7 @@
 	int mbox_modified, ret = 0;
 
 	if (t->save_ctx != NULL)
-		ret = mbox_save_commit(t->save_ctx);
+		ret = mbox_transaction_save_commit(t->save_ctx);
 	mbox_modified = t->mbox_modified;
 
 	if (ret == 0) {
@@ -55,7 +55,7 @@
 	struct index_mailbox *ibox = t->ictx.ibox;
 
 	if (t->save_ctx != NULL)
-		mbox_save_rollback(t->save_ctx);
+		mbox_transaction_save_rollback(t->save_ctx);
 
 	if (t->mbox_lock_id != 0)
 		(void)mbox_unlock(ibox, t->mbox_lock_id);

More information about the dovecot-cvs mailing list