diff -r d5cae1f5fb6a src/lib-index/mail-transaction-log-append.c
--- a/src/lib-index/mail-transaction-log-append.c	Sun Mar 16 12:10:57 2008 +0200
+++ b/src/lib-index/mail-transaction-log-append.c	Fri Mar 21 10:35:16 2008 +0200
@@ -3,6 +3,7 @@
 #include "lib.h"
 #include "array.h"
 #include "buffer.h"
+#include "crc32.h"
 #include "write-full.h"
 #include "mail-index-private.h"
 #include "mail-index-view-private.h"
@@ -14,7 +15,6 @@ struct log_append_context {
 	struct mail_index_transaction *trans;
 	buffer_t *output;
 
-	uint32_t first_append_size;
 	bool sync_includes_this;
 };
 
@@ -23,7 +23,6 @@ static void log_append_buffer(struct log
 			      enum mail_transaction_type type)
 {
 	struct mail_transaction_header hdr;
-	uint32_t hdr_size;
 
 	i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
 	i_assert((buf->used % 4) == 0);
@@ -39,17 +38,9 @@ static void log_append_buffer(struct log
 	if ((ctx->trans->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
 		hdr.type |= MAIL_TRANSACTION_EXTERNAL;
 
-	hdr_size = mail_index_uint32_to_offset(sizeof(hdr) + buf->used +
-					       (hdr_buf == NULL ? 0 :
-						hdr_buf->used));
-	if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(ctx->file) &&
-	    ctx->first_append_size == 0) {
-		/* size will be written later once everything
-		   is in disk */
-		ctx->first_append_size = hdr_size;
-	} else {
-		hdr.size = hdr_size;
-	}
+	hdr.size = sizeof(hdr) + buf->used +
+		(hdr_buf == NULL ? 0 : hdr_buf->used);
+	hdr.size = mail_index_uint32_to_offset(hdr.size);
 
 	buffer_append(ctx->output, &hdr, sizeof(hdr));
 	if (hdr_buf != NULL)
@@ -80,6 +71,33 @@ static int log_buffer_move_to_memory(str
 	return 0;
 }
 
+static void log_update_transaction_frame(struct log_append_context *ctx)
+{
+	struct mail_transaction_header hdr;
+	struct mail_transaction_frame frame;
+	const void *data;
+	unsigned int data_offset;
+
+	/* update header */
+	memset(&hdr, 0, sizeof(hdr));
+	hdr.type = MAIL_TRANSACTION_FRAME;
+	if ((ctx->trans->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
+		hdr.type |= MAIL_TRANSACTION_EXTERNAL;
+	hdr.size = mail_index_uint32_to_offset(ctx->output->used);
+	buffer_write(ctx->output, 0, &hdr, sizeof(hdr));
+
+	/* calculate transaction's crc32 and update the frame */
+	data_offset = sizeof(hdr) + sizeof(frame);
+	i_assert(data_offset < ctx->output->used);
+	data = CONST_PTR_OFFSET(ctx->output->data, data_offset);
+
+	memset(&frame, 0, sizeof(frame));
+	frame.crc32 = crc32_data(&hdr, sizeof(hdr));
+	frame.crc32 = crc32_data_more(frame.crc32, data,
+				      ctx->output->used - data_offset);
+	buffer_write(ctx->output, sizeof(hdr), &frame, sizeof(frame));
+}
+
 static int log_buffer_write(struct log_append_context *ctx, bool want_fsync)
 {
 	struct mail_transaction_log_file *file = ctx->file;
@@ -90,7 +108,8 @@ static int log_buffer_write(struct log_a
 		return 0;
 	}
 
-	i_assert(ctx->first_append_size != 0);
+	log_update_transaction_frame(ctx);
+
 	if (pwrite_full(file->fd, ctx->output->data, ctx->output->used,
 			file->sync_offset) < 0) {
 		/* write failure, fallback to in-memory indexes. */
@@ -103,16 +122,6 @@ static int log_buffer_write(struct log_a
 	i_assert(!ctx->sync_includes_this ||
 		 file->sync_offset + ctx->output->used ==
 		 file->max_tail_offset);
-
-	/* now that the whole transaction has been written, rewrite the first
-	   record's size so the transaction becomes visible */
-	if (pwrite_full(file->fd, &ctx->first_append_size,
-			sizeof(uint32_t), file->sync_offset) < 0) {
-		mail_index_file_set_syscall_error(file->log->index,
-						  file->filepath,
-						  "pwrite_full()");
-		return log_buffer_move_to_memory(ctx);
-	}
 
 	if ((want_fsync && !file->log->index->fsync_disable) ||
 	    file->log->index->nfs_flush) {
@@ -508,6 +517,8 @@ mail_transaction_log_append_locked(struc
 	ctx.file = file;
 	ctx.trans = t;
 	ctx.output = buffer_create_dynamic(default_pool, 1024);
+	buffer_append_zero(ctx.output, sizeof(struct mail_transaction_header) +
+			   sizeof(struct mail_transaction_frame));
 
 	/* send all extension introductions and resizes before appends
 	   to avoid resize overhead as much as possible */
diff -r d5cae1f5fb6a src/lib-index/mail-transaction-log-file.c
--- a/src/lib-index/mail-transaction-log-file.c	Sun Mar 16 12:10:57 2008 +0200
+++ b/src/lib-index/mail-transaction-log-file.c	Fri Mar 21 10:35:16 2008 +0200
@@ -3,6 +3,7 @@
 #include "lib.h"
 #include "ioloop.h"
 #include "buffer.h"
+#include "crc32.h"
 #include "file-dotlock.h"
 #include "nfs-workarounds.h"
 #include "read-full.h"
@@ -725,11 +726,69 @@ log_file_track_mailbox_sync_offset(struc
 }
 
 static int
+log_file_check_transaction(struct mail_transaction_log_file *file,
+			   const struct mail_transaction_header *hdr,
+			   uint32_t trans_size)
+{
+	const struct mail_transaction_frame *frame = (const void *)(hdr + 1);
+	const struct mail_transaction_header *subhdr;
+	uint32_t crc32, offset, sub_size;
+
+	if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) !=
+	    MAIL_TRANSACTION_FRAME) {
+		/* for backwards compatibility */
+		if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
+			if (log_file_track_mailbox_sync_offset(file, hdr,
+							       trans_size) < 0)
+				return -1;
+		}
+		return 0;
+	}
+
+	if (trans_size <= sizeof(*hdr) + sizeof(*frame)) {
+		mail_transaction_log_file_set_corrupted(file,
+			"Transaction size too small");
+		return -1;
+	}
+
+	offset = sizeof(*hdr) + sizeof(*frame);
+	crc32 = crc32_data(hdr, sizeof(*hdr));
+	crc32 = crc32_data_more(crc32, frame + 1, trans_size - offset);
+	if (frame->crc32 != crc32) {
+		mail_transaction_log_file_set_corrupted(file,
+			"Transaction CRC32 wrong");
+		return -1;
+	}
+
+	while (offset < trans_size) {
+		subhdr = CONST_PTR_OFFSET(hdr, offset);
+		sub_size = mail_index_offset_to_uint32(subhdr->size);
+		if (sub_size == 0) {
+			mail_transaction_log_file_set_corrupted(file,
+				"Broken transaction internal record size");
+			return -1;
+		}
+		if (offset + sub_size > trans_size) {
+			mail_transaction_log_file_set_corrupted(file,
+				"Transaction internal record size points "
+				"outside transaction");
+			return -1;
+		}
+		if ((subhdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
+			if (log_file_track_mailbox_sync_offset(file, subhdr,
+							       sub_size) < 0)
+				return -1;
+		}
+		offset += sub_size;
+	}
+	return 0;
+}
+
+static int
 mail_transaction_log_file_sync(struct mail_transaction_log_file *file)
 {
         const struct mail_transaction_header *hdr;
 	const void *data;
-	struct stat st;
 	size_t size, avail;
 	uint32_t trans_size = 0;
 
@@ -744,7 +803,7 @@ mail_transaction_log_file_sync(struct ma
 		trans_size = mail_index_offset_to_uint32(hdr->size);
 		if (trans_size == 0) {
 			/* unfinished */
-			return 1;
+			return 0;
 		}
 		if (trans_size < sizeof(*hdr)) {
 			mail_transaction_log_file_set_corrupted(file,
@@ -756,49 +815,18 @@ mail_transaction_log_file_sync(struct ma
 			break;
 
 		/* transaction has been fully written */
-		if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
-			if (log_file_track_mailbox_sync_offset(file, hdr,
-							       trans_size) < 0)
-				return -1;
-		}
+		if (log_file_check_transaction(file, hdr, trans_size) < 0)
+			return -1;
 		file->sync_offset += trans_size;
 		trans_size = 0;
 	}
 
-	if (file->mmap_base != NULL && !file->locked) {
-		/* Now that all the mmaped pages have page faulted, check if
-		   the file had changed while doing that. Only after the last
-		   page has faulted, the size returned by fstat() can be
-		   trusted. Otherwise it might point to a page boundary while
-		   the next page is still being written.
-
-		   Without this check we might see partial transactions,
-		   sometimes causing "Extension record updated without intro
-		   prefix" errors. */
-		if (fstat(file->fd, &st) < 0) {
-			mail_index_file_set_syscall_error(file->log->index,
-							  file->filepath,
-							  "fstat()");
-			return -1;
-		}
-		if ((uoff_t)st.st_size != file->last_size) {
-			file->last_size = st.st_size;
-			return 0;
-		}
-	}
-
 	avail = file->sync_offset - file->buffer_offset;
 	if (avail != size) {
-		/* There's more data than we could sync at the moment. If the
-		   last record's size wasn't valid, we can't know if it will
-		   be updated unless we've locked the log. */
-		if (trans_size != 0) {
-			/* pread()s or the above fstat() check for mmaps should
-			   have guaranteed that this doesn't happen */
-			mail_transaction_log_file_set_corrupted(file,
-				"hdr.size too large (%u)", trans_size);
-			return -1;
-		} else if (file->locked) {
+		/* There's more data than we could sync at the moment.
+		   We can't know if the rest will be added later, unless we've
+		   locked the log. */
+		if (file->locked) {
 			mail_transaction_log_file_set_corrupted(file,
 				"Unexpected garbage at EOF");
 			return -1;
@@ -819,7 +847,7 @@ mail_transaction_log_file_sync(struct ma
 		return -1;
 	}
 
-	return 1;
+	return 0;
 }
 
 static int
@@ -958,10 +986,8 @@ mail_transaction_log_file_read(struct ma
 		return mail_transaction_log_file_read(file, start_offset, TRUE);
 	}
 
-	if ((ret = mail_transaction_log_file_sync(file)) <= 0) {
-		i_assert(ret != 0); /* happens only with mmap */
+	if (mail_transaction_log_file_sync(file) < 0)
 		return -1;
-	}
 
 	i_assert(file->sync_offset >= file->buffer_offset);
 	buffer_set_used_size(file->buffer,
@@ -1045,7 +1071,6 @@ mail_transaction_log_file_map_mmap(struc
 				   uoff_t start_offset)
 {
 	struct stat st;
-	int ret;
 
 	/* we are going to mmap() this file, but it's not necessarily
 	   mmaped currently. */
@@ -1067,30 +1092,20 @@ mail_transaction_log_file_map_mmap(struc
 
 	if ((uoff_t)st.st_size == file->mmap_size) {
 		/* we already have the whole file mmaped */
-		if ((ret = mail_transaction_log_file_sync(file)) < 0)
-			return 0;
-		if (ret > 0)
-			return 1;
-		/* size changed, re-mmap */
+		return mail_transaction_log_file_sync(file) < 0 ? -1 : 1;
 	}
 
-	do {
-		mail_transaction_log_file_munmap(file);
+	mail_transaction_log_file_munmap(file);
 
-		if (file->last_size - start_offset < mmap_get_page_size()) {
-			/* just reading the file is probably faster */
-			return mail_transaction_log_file_read(file,
-							      start_offset,
-							      FALSE);
-		}
+	if (file->last_size - start_offset < mmap_get_page_size()) {
+		/* just reading the file is probably faster */
+		return mail_transaction_log_file_read(file, start_offset,
+						      FALSE);
+	}
 
-		if (mail_transaction_log_file_mmap(file) < 0)
-			return -1;
-		if ((ret = mail_transaction_log_file_sync(file)) < 0)
-			return 0;
-	} while (ret == 0);
-
-	return 1;
+	if (mail_transaction_log_file_mmap(file) < 0)
+		return -1;
+	return mail_transaction_log_file_sync(file) < 0 ? -1 : 1;
 }
 
 int mail_transaction_log_file_map(struct mail_transaction_log_file *file,
diff -r d5cae1f5fb6a src/lib-index/mail-transaction-log.h
--- a/src/lib-index/mail-transaction-log.h	Sun Mar 16 12:10:57 2008 +0200
+++ b/src/lib-index/mail-transaction-log.h	Fri Mar 21 10:35:16 2008 +0200
@@ -28,6 +28,7 @@ enum mail_transaction_type {
 	MAIL_TRANSACTION_EXT_REC_UPDATE		= 0x00000200,
 	MAIL_TRANSACTION_KEYWORD_UPDATE		= 0x00000400,
 	MAIL_TRANSACTION_KEYWORD_RESET		= 0x00000800,
+	MAIL_TRANSACTION_FRAME			= 0x00001000,
 
 	MAIL_TRANSACTION_TYPE_MASK		= 0x0000ffff,
 
@@ -48,6 +49,13 @@ struct mail_transaction_header {
 struct mail_transaction_header {
 	uint32_t size;
 	uint32_t type; /* enum mail_transaction_type */
+};
+
+/* Transaction frame's size extends through all the records within this
+   transaction. crc32 is the CRC32 of the entire transaction, including the
+   frame's header (but excluding the crc32 field itself, of course). */
+struct mail_transaction_frame {
+	uint32_t crc32;
 };
 
 struct mail_transaction_expunge {
