Unfinished patch to move all changing header fields from
dovecot.index.cache to a header in dovecot.index. This way
dovecot.index.cache would only be appended to, never overwritten.

But this is a bit complex change and still requires overwriting until
backwards compatibility is removed. The header update would need to be done
while cache is locked. To avoid deadlocks, transaction log would have to be
locked before cache is locked.

This change adds also possibility of relative offsets to 
mail_cache_record.prev_offset, which could eventually allow appending to
the cache file without it being locked.

diff -r debecd057f9c src/lib-index/mail-cache-compress.c
--- a/src/lib-index/mail-cache-compress.c	Sat Nov 03 20:25:17 2012 +0200
+++ b/src/lib-index/mail-cache-compress.c	Sat Nov 03 20:37:05 2012 +0200
@@ -148,10 +148,29 @@
 	mail_cache_header_fields_get(cache, ctx->buffer);
 }
 
+static void
+mail_cache_index_header_add_fields(struct mail_cache *cache, buffer_t *ihdr_buf)
+{
+	unsigned int i, field;
+	uint8_t dec;
+	uint32_t last_used;
+
+	for (i = 0; i < cache->file_fields_count; i++) {
+		field = cache->file_field_map[i];
+		last_used = cache->fields[field].field.last_used;
+		buffer_append(ihdr_buf, &last_used, sizeof(last_used));
+	}
+	for (i = 0; i < cache->file_fields_count; i++) {
+		field = cache->file_field_map[i];
+		dec = cache->fields[field].field.decision;
+		buffer_append(ihdr_buf, &dec, sizeof(dec));
+	}
+}
+
 static int
 mail_cache_copy(struct mail_cache *cache, struct mail_index_transaction *trans,
 		int fd, uint32_t *file_seq_r,
-		ARRAY_TYPE(uint32_t) *ext_offsets)
+		ARRAY_TYPE(uint32_t) *ext_offsets, buffer_t *ihdr_buf)
 {
         struct mail_cache_copy_context ctx;
 	struct mail_cache_lookup_iterate_ctx iter;
@@ -161,6 +180,7 @@
 	const struct mail_index_header *idx_hdr;
 	struct mail_cache_header hdr;
 	struct mail_cache_record cache_rec;
+	struct mail_cache_index_header *ihdr;
 	struct ostream *output;
 	uint32_t message_count, seq, first_new_seq, ext_offset;
 	unsigned int i, used_fields_count, orig_fields_count, record_count;
@@ -259,11 +279,12 @@
 			/* nothing cached */
 			ext_offset = 0;
 		} else {
-			cache_rec.size = ctx.buffer->used;
+			cache_rec.size_and_flags = ctx.buffer->used;
 			ext_offset = output->offset;
 			buffer_write(ctx.buffer, 0, &cache_rec,
 				     sizeof(cache_rec));
-			o_stream_nsend(output, ctx.buffer->data, cache_rec.size);
+			o_stream_nsend(output, ctx.buffer->data,
+				       ctx.buffer->used);
 			record_count++;
 		}
 
@@ -271,9 +292,15 @@
 	}
 	i_assert(orig_fields_count == cache->fields_count);
 
-	hdr.record_count = record_count;
-	hdr.field_header_offset = mail_index_uint32_to_offset(output->offset);
+	ihdr = buffer_append_space_unsafe(ihdr_buf, sizeof(*ihdr));
+	ihdr->fields_offset = output->offset;
+	ihdr->record_count = record_count;
+
+	hdr.backwards_compat_record_count = record_count;
+	hdr.backwards_compat_field_header_offset =
+		mail_index_uint32_to_offset(output->offset);
 	mail_cache_compress_get_fields(&ctx, used_fields_count);
+	mail_cache_index_header_add_fields(cache, ihdr_buf);
 	o_stream_nsend(output, ctx.buffer->data, ctx.buffer->used);
 
 	hdr.backwards_compat_used_file_size = output->offset;
@@ -350,6 +377,7 @@
 	ARRAY_TYPE(uint32_t) ext_offsets;
 	const uint32_t *offsets;
 	const void *data;
+	buffer_t *ihdr_buf;
 	unsigned int i, count;
 	int fd, ret;
 
@@ -387,7 +415,9 @@
 	mail_index_fchown(cache->index, fd,
 			  file_dotlock_get_lock_path(dotlock));
 
-	if (mail_cache_copy(cache, trans, fd, &file_seq, &ext_offsets) < 0) {
+	ihdr_buf = buffer_create_dynamic(pool_datastack_create(), 256);
+	if (mail_cache_copy(cache, trans, fd, &file_seq, &ext_offsets,
+			    ihdr_buf) < 0) {
 		/* the fields may have been updated in memory already.
 		   reverse those changes by re-reading them from file. */
 		if (mail_cache_header_fields_read(cache) < 0)
@@ -414,6 +444,11 @@
 	/* once we're sure that the compression was successful,
 	   update the offsets */
 	mail_index_ext_reset(trans, cache->ext_id, file_seq, TRUE);
+	mail_index_ext_resize(trans, cache->ext_id, ihdr_buf->used,
+			      sizeof(uint32_t), sizeof(uint32_t));
+	mail_index_update_header_ext(trans, cache->ext_id, 0,
+				     ihdr_buf->data, ihdr_buf->used);
+
 	offsets = array_get(&ext_offsets, &count);
 	for (i = 0; i < count; i++) {
 		if (offsets[i] != 0) {
diff -r debecd057f9c src/lib-index/mail-cache-fields.c
--- a/src/lib-index/mail-cache-fields.c	Sat Nov 03 20:25:17 2012 +0200
+++ b/src/lib-index/mail-cache-fields.c	Sat Nov 03 20:37:05 2012 +0200
@@ -203,6 +203,7 @@
 {
 	const struct mail_cache_header_fields *field_hdr;
 	struct mail_cache_header_fields tmp_field_hdr;
+	const struct mail_cache_index_header *ihdr;
 	const void *data;
 	uint32_t offset = 0, next_offset;
 	unsigned int next_count = 0;
@@ -215,11 +216,18 @@
 		return 0;
 	}
 
+	mail_cache_index_hdr_refresh(cache);
+	i_assert(cache->ihdr->used > 0);
+	ihdr = cache->ihdr->data;
+
 	/* find the latest header */
 	offset = 0;
 	next_offset = cache->last_field_header_offset != 0 ?
 		cache->last_field_header_offset :
-		mail_index_offset_to_uint32(cache->hdr->field_header_offset);
+		mail_index_offset_to_uint32(cache->hdr->backwards_compat_field_header_offset);
+	if (next_offset < ihdr->fields_offset)
+		next_offset = ihdr->fields_offset;
+
 	while (next_offset != 0) {
 		if (next_offset == offset) {
 			mail_cache_set_corrupted(cache,
@@ -258,7 +266,7 @@
 		}
 
 		next_offset =
-			mail_index_offset_to_uint32(field_hdr->next_offset);
+			mail_index_offset_to_uint32(field_hdr->backwards_compat_next_offset);
 		next_count++;
 	}
 
@@ -511,6 +519,11 @@
 {
 	int ret;
 
+	// FIXME: update index header
+
+	if (!cache->backwards_compatibility)
+		return 0;
+
 	if (cache->locked) {
 		T_BEGIN {
 			ret = mail_cache_header_fields_update_locked(cache);
@@ -588,10 +601,10 @@
 
 	if (*offset_r == 0) {
 		*offset_r = offsetof(struct mail_cache_header,
-				     field_header_offset);
+				     backwards_compat_field_header_offset);
 	} else {
 		*offset_r += offsetof(struct mail_cache_header_fields,
-				      next_offset);
+				      backwards_compat_next_offset);
 	}
 	return 0;
 }
diff -r debecd057f9c src/lib-index/mail-cache-lookup.c
--- a/src/lib-index/mail-cache-lookup.c	Sat Nov 03 20:25:17 2012 +0200
+++ b/src/lib-index/mail-cache-lookup.c	Sat Nov 03 20:37:05 2012 +0200
@@ -15,6 +15,7 @@
 {
 	const struct mail_cache_record *rec;
 	const void *data;
+	uint32_t rec_size;
 
 	i_assert(offset != 0);
 
@@ -34,20 +35,21 @@
 		return -1;
 	}
 	rec = data;
+	rec_size = rec->size_and_flags & MAIL_CACHE_RECORD_SIZE_MASK;
 
-	if (rec->size < sizeof(*rec)) {
+	if (rec_size < sizeof(*rec)) {
 		mail_cache_set_corrupted(cache, "invalid record size");
 		return -1;
 	}
-	if (rec->size > CACHE_PREFETCH) {
+	if (rec_size > CACHE_PREFETCH) {
 		/* larger than we guessed. map the rest of the record. */
-		if (mail_cache_map(cache, offset, rec->size, &data) < 0)
+		if (mail_cache_map(cache, offset, rec_size, &data) < 0)
 			return -1;
 		rec = data;
 	}
 
-	if (rec->size > cache->mmap_length ||
-	    offset + rec->size > cache->mmap_length) {
+	if (rec_size > cache->mmap_length ||
+	    offset + rec_size > cache->mmap_length) {
 		mail_cache_set_corrupted(cache, "record points outside file");
 		return -1;
 	}
@@ -170,12 +172,24 @@
 mail_cache_lookup_iter_next_record(struct mail_cache_lookup_iterate_ctx *ctx)
 {
 	struct mail_cache_view *view = ctx->view;
+	uint32_t rec_size;
 
 	if (ctx->stop)
 		return ctx->failed ? -1 : 0;
 
-	if (ctx->rec != NULL)
-		ctx->offset = ctx->rec->prev_offset;
+	if (ctx->rec != NULL) {
+		if ((ctx->rec->size_and_flags & MAIL_CACHE_RECORD_FLAG_REL_PREVOFFSET) == 0)
+			ctx->offset = ctx->rec->prev_roffset;
+		else if (ctx->rec->prev_roffset == 0)
+			ctx->offset = 0;
+		else if (ctx->rec->prev_roffset >= ctx->offset) {
+			mail_cache_set_corrupted(view->cache,
+				"invalid relative prev_offset");
+			return -1;
+		} else {
+			ctx->offset -= ctx->rec->prev_roffset;
+		}
+	}
 	if (ctx->offset == 0) {
 		/* end of this record list. check newly appended data. */
 		if (ctx->appends_checked ||
@@ -194,8 +208,9 @@
 	/* look up the next record */
 	if (mail_cache_get_record(view->cache, ctx->offset, &ctx->rec) < 0)
 		return -1;
+	rec_size = ctx->rec->size_and_flags & MAIL_CACHE_RECORD_SIZE_MASK;
 	if (mail_cache_track_loops(&view->loop_track, ctx->offset,
-				   ctx->rec->size)) {
+				   rec_size)) {
 		mail_cache_set_corrupted(view->cache,
 					 "record list is circular");
 		return -1;
@@ -203,7 +218,7 @@
 	ctx->remap_counter = view->cache->remap_counter;
 
 	ctx->pos = sizeof(*ctx->rec);
-	ctx->rec_size = ctx->rec->size;
+	ctx->rec_size = rec_size;
 	return 1;
 }
 
@@ -256,15 +271,15 @@
 
 	field_idx = cache->file_field_map[file_field];
 	data_size = cache->fields[field_idx].field.field_size;
-	if (data_size == (unsigned int)-1 &&
-	    ctx->pos + sizeof(uint32_t) <= ctx->rec->size) {
+	if (data_size == -1U &&
+	    ctx->pos + sizeof(uint32_t) <= ctx->rec_size) {
 		/* variable size field. get its size from the file. */
 		data_size = *((const uint32_t *)
 			      CONST_PTR_OFFSET(ctx->rec, ctx->pos));
 		ctx->pos += sizeof(uint32_t);
 	}
 
-	if (ctx->rec->size - ctx->pos < data_size) {
+	if (ctx->rec_size - ctx->pos < data_size) {
 		mail_cache_set_corrupted(cache,
 			"record continues outside its allocated size");
 		return -1;
diff -r debecd057f9c src/lib-index/mail-cache-private.h
--- a/src/lib-index/mail-cache-private.h	Sat Nov 03 20:25:17 2012 +0200
+++ b/src/lib-index/mail-cache-private.h	Sat Nov 03 20:37:05 2012 +0200
@@ -35,7 +35,7 @@
 	((cache)->hdr == NULL)
 
 struct mail_cache_header {
-	/* version is increased only when you can't have backwards
+	/* major version is increased only when you can't have backwards
 	   compatibility. */
 	uint8_t major_version;
 	uint8_t compat_sizeof_uoff_t;
@@ -45,32 +45,32 @@
 	uint32_t indexid;
 	uint32_t file_seq;
 
-	uint32_t continued_record_count;
+	uint32_t backwards_compat_continued_record_count;
 
 	/* NOTE: old versions used this for hole offset, so we can't fully
 	   rely on it */
-	uint32_t record_count;
+	uint32_t backwards_compat_record_count;
 	uint32_t backwards_compat_used_file_size;
-	uint32_t deleted_record_count;
+	uint32_t backwards_compat_deleted_record_count;
 
-	uint32_t field_header_offset;
+	uint32_t backwards_compat_field_header_offset;
 };
 
 struct mail_cache_header_fields {
-	uint32_t next_offset;
+	uint32_t backwards_compat_next_offset;
 	uint32_t size;
 	uint32_t fields_count;
 
 #if 0
 	/* last time the field was accessed. not updated more often than
 	   once a day. */
-	uint32_t last_used[fields_count];
+	uint32_t backwards_compat_last_used[fields_count];
 	/* (uint32_t)-1 for variable sized fields */
 	uint32_t size[fields_count];
 	/* enum mail_cache_field_type */
 	uint8_t type[fields_count];
 	/* enum mail_cache_decision_type */
-	uint8_t decision[fields_count];
+	uint8_t backwards_compat_decision[fields_count];
 	/* NUL-separated list of field names */
 	char name[fields_count][];
 #endif
@@ -87,12 +87,34 @@
 #define MAIL_CACHE_FIELD_NAMES(count) \
 	(MAIL_CACHE_FIELD_DECISION(count) + sizeof(uint8_t) * (count))
 
+#define MAIL_CACHE_RECORD_SIZE_MASK		0x0fffffff
+#define MAIL_CACHE_RECORD_FLAG_REL_PREVOFFSET	0x10000000
+
 struct mail_cache_record {
-	uint32_t prev_offset;
-	uint32_t size; /* full record size, including this header */
+	/* offset to previous record. if REL_PREVOFFSET flag is set, the real
+	   offset is this many bytes backwards from the current record offset */
+	uint32_t prev_roffset;
+	/* full record size, including this header. */
+	uint32_t size_and_flags;
 	/* array of { uint32_t field; [ uint32_t size; ] { .. } } */
 };
 
+struct mail_cache_index_header {
+	/* offset to latest mail_cache_header_fields */
+	uint32_t fields_offset;
+
+	uint32_t record_count;
+	uint32_t continued_record_count;
+	uint32_t deleted_record_count;
+#if 0
+	/* last time the field was accessed. not updated more often than
+	   once a day. */
+	uint32_t last_used[fields_count];
+	/* enum mail_cache_decision_type */
+	uint8_t decision[fields_count];
+#endif
+};
+
 struct mail_cache_field_private {
 	struct mail_cache_field field;
 
@@ -113,6 +135,7 @@
 
 	ino_t st_ino;
 	dev_t st_dev;
+	buffer_t *ihdr; /* struct mail_cache_index_header */
 
 	size_t mmap_length;
 	/* a) mmaping the whole file */
@@ -156,9 +179,11 @@
 	unsigned int locked:1;
 	unsigned int last_lock_failed:1;
 	unsigned int hdr_modified:1;
+	unsigned int locked_hdr_refresh:1;
 	unsigned int field_header_write_pending:1;
 	unsigned int compressing:1;
 	unsigned int map_with_read:1;
+	unsigned int backwards_compatibility:1;
 };
 
 struct mail_cache_loop_track {
@@ -248,6 +273,7 @@
 		   const void **data_r);
 void mail_cache_file_close(struct mail_cache *cache);
 int mail_cache_reopen(struct mail_cache *cache);
+bool mail_cache_index_hdr_refresh(struct mail_cache *cache);
 
 void mail_cache_delete(struct mail_cache *cache);
 
@@ -261,6 +287,7 @@
 int mail_cache_expunge_handler(struct mail_index_sync_map_ctx *sync_ctx,
 			       uint32_t seq, const void *data,
 			       void **sync_context, void *context);
+void mail_cache_sync_lost_handler(struct mail_index *index);
 
 void mail_cache_set_syscall_error(struct mail_cache *cache,
 				  const char *function);
diff -r debecd057f9c src/lib-index/mail-cache-transaction.c
--- a/src/lib-index/mail-cache-transaction.c	Sat Nov 03 20:25:17 2012 +0200
+++ b/src/lib-index/mail-cache-transaction.c	Sat Nov 03 20:37:05 2012 +0200
@@ -130,6 +130,7 @@
 void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx **_ctx)
 {
 	struct mail_cache_transaction_ctx *ctx = *_ctx;
+	struct mail_cache_index_header *ihdr;
 
 	*_ctx = NULL;
 
@@ -137,7 +138,11 @@
 		/* we already wrote to the cache file. we can't (or don't want
 		   to) delete that data, so just mark it as deleted space */
 		if (mail_cache_transaction_lock(ctx) > 0) {
-			ctx->cache->hdr_copy.deleted_record_count +=
+			ihdr = buffer_get_space_unsafe(ctx->cache->ihdr, 0,
+						       sizeof(*ihdr));
+			ihdr->deleted_record_count += ctx->records_written;
+
+			ctx->cache->hdr_copy.backwards_compat_deleted_record_count +=
 				ctx->records_written;
 			(void)mail_cache_unlock(ctx->cache);
 		}
@@ -263,7 +268,8 @@
 		i_assert(ctx->cache_data == NULL ||
 			 ctx->cache_data->used == 0);
 		ctx->cache_file_seq = cache->hdr->file_seq;
-	} else if (ctx->cache_file_seq != cache->hdr->file_seq) {
+	} else if (!mail_cache_index_hdr_refresh(cache) ||
+		   ctx->cache_file_seq != cache->hdr->file_seq) {
 		if (mail_cache_unlock(cache) < 0)
 			return -1;
 		mail_cache_transaction_reset(ctx);
@@ -279,21 +285,19 @@
 	struct mail_cache *cache = ctx->cache;
 	const struct mail_cache_record *rec = ctx->cache_data->data;
 	const uint32_t *seqs;
-	uint32_t i, seq_count;
+	uint32_t i, seq_count, rec_size;
 
 	mail_index_ext_using_reset_id(ctx->trans, ctx->cache->ext_id,
 				      ctx->cache_file_seq);
 
-	/* write the cache_offsets to index file. records' prev_offset
-	   is updated to point to old cache record when index is being
-	   synced. */
 	seqs = array_get(&ctx->cache_data_seq, &seq_count);
 	for (i = 0; i < seq_count; i++) {
 		mail_index_update_ext(ctx->trans, seqs[i], cache->ext_id,
 				      &write_offset, NULL);
 
-		write_offset += rec->size;
-		rec = CONST_PTR_OFFSET(rec, rec->size);
+		rec_size = rec->size_and_flags & MAIL_CACHE_RECORD_SIZE_MASK;
+		write_offset += rec_size;
+		rec = CONST_PTR_OFFSET(rec, rec_size);
 	}
 	return 0;
 }
@@ -303,12 +307,14 @@
 			uint32_t write_offset)
 {
 	struct mail_index_map *map;
+	struct mail_cache_index_header *ihdr;
 	struct mail_cache_record *rec;
 	const uint32_t *seqs, *prev_offsetp;
 	ARRAY_TYPE(uint32_t) seq_offsets;
-	uint32_t i, seq_count, reset_id, prev_offset, *offsetp;
+	uint32_t i, seq_count, reset_id, rec_size, prev_offset, *offsetp;
 	const void *data;
 	bool expunged;
+	unsigned int continue_count = 0;
 
 	i_assert(ctx->min_seq != 0);
 
@@ -318,9 +324,15 @@
 	for (i = 0; i < seq_count; i++) {
 		offsetp = array_idx_modifiable(&seq_offsets,
 					       seqs[i] - ctx->min_seq);
-		if (*offsetp != 0)
+		if (*offsetp != 0) {
 			prev_offset = *offsetp;
-		else {
+			if (!ctx->cache->backwards_compatibility) {
+				/* the offset is relative */
+				prev_offset = write_offset - prev_offset;
+				rec->size_and_flags |=
+					MAIL_CACHE_RECORD_FLAG_REL_PREVOFFSET;
+			}
+		} else {
 			mail_index_lookup_ext_full(ctx->view->trans_view, seqs[i],
 						   ctx->cache->ext_id, &map,
 						   &data, &expunged);
@@ -345,18 +357,24 @@
 
 		if (prev_offset != 0) {
 			/* link this record to previous one */
-			rec->prev_offset = prev_offset;
-			ctx->cache->hdr_copy.continued_record_count++;
-		} else {
-			ctx->cache->hdr_copy.record_count++;
+			rec->prev_roffset = prev_offset;
+			continue_count++;
 		}
 		*offsetp = write_offset;
 
-		write_offset += rec->size;
-		rec = PTR_OFFSET(rec, rec->size);
+		rec_size = rec->size_and_flags & MAIL_CACHE_RECORD_SIZE_MASK;
+		write_offset += rec_size;
+		rec = PTR_OFFSET(rec, rec_size);
+	}
+	if (continue_count > 0) {
+		ihdr = buffer_get_space_unsafe(ctx->cache->ihdr, 0,
+					       sizeof(*ihdr));
+		ihdr->continued_record_count += continue_count;
+		ctx->cache->hdr_copy.backwards_compat_continued_record_count +=
+			continue_count;
+		ctx->cache->hdr_modified = TRUE;
 	}
 	array_free(&seq_offsets);
-	ctx->cache->hdr_modified = TRUE;
 	return 0;
 }
 
@@ -364,8 +382,8 @@
 mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
 {
 	struct stat st;
-	uint32_t write_offset = 0;
-	int ret;
+	uint32_t write_offset;
+	int ret = 0;
 
 	i_assert(!ctx->cache->locked);
 
@@ -382,19 +400,24 @@
 	i_assert(ctx->cache_data != NULL);
 	i_assert(ctx->last_rec_pos <= ctx->cache_data->used);
 
-	/* we need to get the final write offset for linking records */
-	if (fstat(ctx->cache->fd, &st) < 0) {
-		if (!ESTALE_FSTAT(errno))
-			mail_cache_set_syscall_error(ctx->cache, "fstat()");
+	if (!ctx->cache->backwards_compatibility)
+		write_offset = 0;
+	else {
+		/* we need to get the final write offset for linking records */
+		if (fstat(ctx->cache->fd, &st) < 0) {
+			if (!ESTALE_FSTAT(errno))
+				mail_cache_set_syscall_error(ctx->cache, "fstat()");
+			return -1;
+		}
+		if ((uint32_t)-1 < st.st_size + ctx->last_rec_pos) {
+			mail_cache_set_corrupted(ctx->cache,
+						 "Cache file too large");
+			ret = -1;
+		}
+		write_offset = st.st_size;
+	}
+	if (mail_cache_link_records(ctx, write_offset) < 0)
 		ret = -1;
-	} else if ((uint32_t)-1 < st.st_size + ctx->last_rec_pos) {
-		mail_cache_set_corrupted(ctx->cache, "Cache file too large");
-		ret = -1;
-	} else {
-		write_offset = st.st_size;
-		if (mail_cache_link_records(ctx, write_offset) < 0)
-			ret = -1;
-	}
 
 	/* write to cache file */
 	if (ret < 0 ||
@@ -426,18 +449,20 @@
 {
 	struct mail_cache_record *rec;
 	void *data;
-	size_t size;
+	size_t size, rec_size;
 
 	data = buffer_get_modifiable_data(ctx->cache_data, &size);
-	rec = PTR_OFFSET(data, ctx->last_rec_pos);
-	rec->size = size - ctx->last_rec_pos;
-	i_assert(rec->size > sizeof(*rec));
+	rec_size = size - ctx->last_rec_pos;
+	i_assert(rec_size > sizeof(*rec));
 
-	if (rec->size > MAIL_CACHE_RECORD_MAX_SIZE) {
+	if (rec_size > MAIL_CACHE_RECORD_MAX_SIZE) {
 		buffer_set_used_size(ctx->cache_data, ctx->last_rec_pos);
 		return;
 	}
 
+	rec = PTR_OFFSET(data, ctx->last_rec_pos);
+	rec->size_and_flags = rec_size;
+
 	if (ctx->min_seq > ctx->prev_seq || ctx->min_seq == 0)
 		ctx->min_seq = ctx->prev_seq;
 	array_append(&ctx->cache_data_seq, &ctx->prev_seq, 1);
@@ -495,6 +520,7 @@
 			       const buffer_t *buffer)
 {
 	struct mail_cache *cache = ctx->cache;
+	struct mail_cache_index_header *ihdr;
 	uint32_t offset, hdr_offset;
 
 	i_assert(cache->locked);
@@ -509,6 +535,12 @@
 			return -1;
 		}
 	}
+	ihdr = buffer_get_space_unsafe(ctx->cache->ihdr, 0, sizeof(*ihdr));
+	ihdr->fields_offset = offset;
+
+	if (!ctx->cache->backwards_compatibility)
+		return 0;
+
 	/* find offset to the previous header's "next_offset" field */
 	if (mail_cache_header_fields_get_next_offset(cache, &hdr_offset) < 0)
 		return -1;
@@ -519,11 +551,11 @@
 		return -1;
 
 	if (hdr_offset == offsetof(struct mail_cache_header,
-				   field_header_offset)) {
+				   backwards_compat_field_header_offset)) {
 		/* we're adding the first field. hdr_copy needs to be kept
 		   in sync so unlocking won't overwrite it. */
-		cache->hdr_copy.field_header_offset = hdr_offset;
-		cache->hdr_ro_copy.field_header_offset = hdr_offset;
+		cache->hdr_copy.backwards_compat_field_header_offset = hdr_offset;
+		cache->hdr_ro_copy.backwards_compat_field_header_offset = hdr_offset;
 	}
 	return 0;
 }
@@ -741,6 +773,8 @@
 
 void mail_cache_delete(struct mail_cache *cache)
 {
+	struct mail_cache_index_header *ihdr;
+
 	i_assert(cache->locked);
 
 	/* we'll only update the deleted record count in the header. we can't
@@ -748,8 +782,11 @@
 	   using the data. also it's actually useful as old index views are
 	   still able to ask cached data for messages that have already been
 	   expunged. */
-	cache->hdr_copy.deleted_record_count++;
-	if (cache->hdr_copy.record_count > 0)
-		cache->hdr_copy.record_count--;
+	ihdr = buffer_get_space_unsafe(cache->ihdr, 0, sizeof(*ihdr));
+	ihdr->deleted_record_count++;
+
+	cache->hdr_copy.backwards_compat_deleted_record_count++;
+	if (cache->hdr_copy.backwards_compat_record_count > 0)
+		cache->hdr_copy.backwards_compat_record_count--;
 	cache->hdr_modified = TRUE;
 }
diff -r debecd057f9c src/lib-index/mail-cache.c
--- a/src/lib-index/mail-cache.c	Sat Nov 03 20:25:17 2012 +0200
+++ b/src/lib-index/mail-cache.c	Sat Nov 03 20:37:05 2012 +0200
@@ -65,6 +65,7 @@
 	cache->hdr = NULL;
 	cache->mmap_length = 0;
 	cache->last_field_header_offset = 0;
+	buffer_set_used_size(cache->ihdr, 0);
 
 	if (cache->file_lock != NULL)
 		file_lock_free(&cache->file_lock);
@@ -145,9 +146,8 @@
 
 int mail_cache_reopen(struct mail_cache *cache)
 {
-	struct mail_index_view *view;
-	const struct mail_index_ext *ext;
 	const void *data;
+	unsigned int retry;
 
 	i_assert(!cache->locked);
 
@@ -156,39 +156,40 @@
 		return 0;
 	}
 
-	mail_cache_file_close(cache);
+	for (retry = 0; retry < 3; retry++) {
+		mail_cache_file_close(cache);
+		cache->fd = nfs_safe_open(cache->filepath,
+					  cache->index->readonly ?
+					  O_RDONLY : O_RDWR);
+		if (cache->fd == -1) {
+			if (errno == ENOENT)
+				cache->need_compress_file_seq = 0;
+			else
+				mail_cache_set_syscall_error(cache, "open()");
+			return -1;
+		}
+		mail_cache_init_file_cache(cache);
 
-	cache->fd = nfs_safe_open(cache->filepath,
-				  cache->index->readonly ? O_RDONLY : O_RDWR);
-	if (cache->fd == -1) {
-		if (errno == ENOENT)
-			cache->need_compress_file_seq = 0;
-		else
-			mail_cache_set_syscall_error(cache, "open()");
-		return -1;
+		if (mail_cache_map(cache, 0, 0, &data) < 0)
+			return -1;
+		/* the mapping should have updated index header, except if
+		   the reset_id doesn't match the cache file. retry opening
+		   and refreshing a few times to see if it helps. */
+		if (cache->ihdr->used != 0)
+			break;
 	}
 
-	mail_cache_init_file_cache(cache);
-
-	if (mail_cache_map(cache, 0, 0, &data) < 0)
-		return -1;
-
-	if (mail_cache_header_fields_read(cache) < 0)
-		return -1;
-
-	view = mail_index_view_open(cache->index);
-	ext = mail_index_view_get_ext(view, cache->ext_id);
-	if (ext == NULL || cache->hdr->file_seq != ext->reset_id) {
+	if (cache->ihdr->used == 0) {
 		/* still different - maybe a race condition or maybe the
 		   file_seq really is corrupted. either way, this shouldn't
 		   happen often so we'll just mark cache to be compressed
 		   later which fixes this. */
 		cache->need_compress_file_seq = cache->hdr->file_seq;
-		mail_index_view_close(&view);
 		return 0;
 	}
 
-	mail_index_view_close(&view);
+	if (mail_cache_header_fields_read(cache) < 0)
+		return -1;
 	i_assert(!MAIL_CACHE_IS_UNUSABLE(cache));
 	return 1;
 }
@@ -196,41 +197,26 @@
 static void mail_cache_update_need_compress(struct mail_cache *cache)
 {
 	const struct mail_cache_header *hdr = cache->hdr;
+	const struct mail_cache_index_header *ihdr = cache->ihdr->data;
 	struct stat st;
-	unsigned int msg_count;
 	unsigned int records_count, cont_percentage, delete_percentage;
 	bool want_compress = FALSE;
 
-	if (hdr->minor_version == 0) {
-		/* compress to get ourself into the new header version */
-		cache->need_compress_file_seq = hdr->file_seq;
+	if (cache->ihdr->used == 0) {
+		/* old cache format, compress to update it */
+		cache->need_compress_file_seq = cache->hdr->file_seq;
 		return;
 	}
+	records_count = I_MAX(ihdr->record_count, 1);
 
-	msg_count = cache->index->map->rec_map->records_count;
-	if (msg_count == 0)
-		records_count = 1;
-	else if (hdr->record_count == 0 || hdr->record_count > msg_count*2) {
-		/* probably not the real record_count, but hole offset that
-		   Dovecot <=v2.1 versions used to use in this position.
-		   we already checked that minor_version>0, but this could
-		   happen if old Dovecot was used to access mailbox after
-		   it had been updated. */
-		records_count = I_MAX(msg_count, 1);
-		cache->hdr_copy.record_count = msg_count;
-		cache->hdr_modified = TRUE;
-	} else {
-		records_count = hdr->record_count;
-	}
-
-	cont_percentage = hdr->continued_record_count * 100 / records_count;
+	cont_percentage = ihdr->continued_record_count * 100 / records_count;
 	if (cont_percentage >= MAIL_CACHE_COMPRESS_CONTINUED_PERCENTAGE) {
 		/* too many continued rows, compress */
 		want_compress = TRUE;
 	}
 
-	delete_percentage = hdr->deleted_record_count * 100 /
-		(records_count + hdr->deleted_record_count);
+	delete_percentage = ihdr->deleted_record_count * 100 /
+		(records_count + ihdr->deleted_record_count);
 	if (delete_percentage >= MAIL_CACHE_COMPRESS_DELETE_PERCENTAGE) {
 		/* too many deleted records, compress */
 		want_compress = TRUE;
@@ -245,7 +231,6 @@
 		if (st.st_size >= MAIL_CACHE_COMPRESS_MIN_SIZE)
 			cache->need_compress_file_seq = hdr->file_seq;
 	}
-
 }
 
 static bool mail_cache_verify_header(struct mail_cache *cache,
@@ -280,6 +265,93 @@
 	return TRUE;
 }
 
+bool mail_cache_index_hdr_refresh(struct mail_cache *cache)
+{
+	struct mail_index_view *view;
+	const void *data;
+	size_t data_size;
+	uint32_t reset_id;
+	bool ret;
+
+	if (cache->locked_hdr_refresh)
+		return TRUE;
+	cache->locked_hdr_refresh = cache->locked;
+
+	/* get the latest offset from index */
+	(void)mail_index_refresh(cache->index);
+	view = mail_index_view_open(cache->index);
+	if (mail_index_ext_get_reset_id(view, cache->index->map, cache->ext_id,
+					&reset_id) &&
+	    reset_id == cache->hdr->file_seq) {
+		mail_index_map_get_header_ext(view, cache->index->map,
+					      cache->ext_id, &data, &data_size);
+		buffer_set_used_size(cache->ihdr, 0);
+		if (data_size > 0)
+			buffer_append(cache->ihdr, data, data_size);
+		else {
+			/* upgrading from old cache file format. our callers
+			   depend on us having at least something in the
+			   header */
+			buffer_append_zero(cache->ihdr,
+				sizeof(struct mail_cache_index_header));
+		}
+		ret = TRUE;
+	} else {
+		ret = FALSE;
+	}
+	mail_index_view_close(&view);
+	return ret;
+}
+
+static int mail_cache_index_hdr_write(struct mail_cache *cache)
+{
+	struct mail_index_view *view;
+	const void *data;
+	const unsigned char *old_data, *new_data;
+	size_t i, common_size, data_size, start_offset, end_offset;
+
+	if (!mail_cache_index_hdr_refresh(cache)) {
+		/* this */
+		i_error("Cache file %s was unexpectedly reset", cache->filepath);
+		return -1;
+	}
+
+	view = mail_index_view_open(cache->index);
+	mail_index_get_header_ext(view, cache->ext_id, &data, &data_size);
+	old_data = data;
+	new_data = cache->ihdr->data;
+
+	/* find the first byte that is different */
+	common_size = I_MIN(data_size, cache->ihdr->used);
+	for (i = start_offset = 0; i < common_size; i++, start_offset++) {
+		if (old_data[i] != new_data[i])
+			break;
+	}
+	/* find the last byte that is different */
+	if (cache->ihdr->used > data_size)
+		end_offset = cache->ihdr->used;
+	else {
+		end_offset = start_offset;
+		for (; i < common_size; i++) {
+			if (old_data[i] != new_data[i])
+				end_offset = i;
+		}
+	}
+
+	if (cache->ihdr->used != data_size) {
+		mail_index_ext_resize(trans, cache->ext_id, cache->ihdr->used,
+				      sizeof(uint32_t), sizeof(uint32_t));
+	}
+	if (start_offset < cache->ihdr->used) {
+		mail_index_update_header_ext(trans, cache->ext_id, start_offset,
+					     cache->ihdr->data,
+					     end_offset - start_offset + 1);
+	}
+
+	mail_index_view_close(&view);
+	return 0;
+}
+
 static int
 mail_cache_map_finish(struct mail_cache *cache, uoff_t offset, size_t size,
 		      const void *hdr_data, bool copy_hdr)
@@ -305,6 +377,7 @@
 			       sizeof(cache->hdr_ro_copy));
 			cache->hdr = &cache->hdr_ro_copy;
 		}
+		(void)mail_cache_index_hdr_refresh(cache);
 		mail_cache_update_need_compress(cache);
 	} else {
 		i_assert(cache->hdr != NULL);
@@ -440,6 +513,7 @@
 static int mail_cache_try_open(struct mail_cache *cache)
 {
 	const void *data;
+	int ret;
 
 	cache->opened = TRUE;
 
@@ -457,12 +531,17 @@
 		mail_cache_set_syscall_error(cache, "open()");
 		return -1;
 	}
-
 	mail_cache_init_file_cache(cache);
 
-	if (mail_cache_map(cache, 0, sizeof(struct mail_cache_header),
-			   &data) < 0)
+	if (mail_cache_map(cache, 0, 0, &data) < 0)
 		return -1;
+	if (cache->ihdr->used == 0) {
+		/* index's reset_id doesn't match the cache file. see if
+		   reopening the cache file helps. */
+		if ((ret = mail_cache_reopen(cache)) == 0)
+			cache->need_compress_file_seq = cache->hdr->file_seq;
+		return ret;
+	}
 	return 1;
 }
 
@@ -493,6 +572,7 @@
 	cache->field_pool = pool_alloconly_create("Cache fields", 2048);
 	hash_table_create(&cache->field_name_hash, cache->field_pool, 0,
 			  strcase_hash, strcasecmp);
+	cache->ihdr = buffer_create_dynamic(default_pool, 128);
 
 	cache->dotlock_settings.use_excl_lock =
 		(index->flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0;
@@ -509,10 +589,16 @@
 		(cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) != 0;
 
 	cache->ext_id =
-		mail_index_ext_register(index, "cache", 0,
+		mail_index_ext_register(index, "cache",
+					sizeof(struct mail_cache_index_header),
 					sizeof(uint32_t), sizeof(uint32_t));
 	mail_index_register_expunge_handler(index, cache->ext_id, FALSE,
 					    mail_cache_expunge_handler, cache);
+
+	if (cache->file_cache != NULL) {
+		mail_index_register_sync_lost_handler(index,
+			mail_cache_sync_lost_handler);
+	}
 	return cache;
 }
 
@@ -549,6 +635,7 @@
 
 	hash_table_destroy(&cache->field_name_hash);
 	pool_unref(&cache->field_pool);
+	buffer_free(&cache->ihdr);
 	i_free(cache->field_file_map);
 	i_free(cache->file_field_map);
 	i_free(cache->fields);
@@ -679,7 +766,7 @@
 			file_cache_invalidate(cache->file_cache, 0,
 					      sizeof(struct mail_cache_header));
 		}
-		if (mail_cache_map(cache, 0, 0, &data) > 0)
+		if (mail_cache_map(cache, 0, 0, &data) == 0)
 			cache->hdr_copy = *cache->hdr;
 		else {
 			(void)mail_cache_unlock(cache);
@@ -703,6 +790,7 @@
 
 int mail_cache_unlock(struct mail_cache *cache)
 {
+	struct mail_index_transaction *trans;
 	int ret = 0;
 
 	i_assert(cache->locked);
@@ -719,10 +807,14 @@
 
 	if (cache->hdr_modified) {
 		cache->hdr_modified = FALSE;
-		if (mail_cache_write(cache, &cache->hdr_copy,
-				     sizeof(cache->hdr_copy), 0) < 0)
+		if (mail_cache_index_hdr_write(cache, &trans) < 0)
 			ret = -1;
-		cache->hdr_ro_copy = cache->hdr_copy;
+		if (cache->backwards_compatibility) {
+			if (mail_cache_write(cache, &cache->hdr_copy,
+					     sizeof(cache->hdr_copy), 0) < 0)
+				ret = -1;
+			cache->hdr_ro_copy = cache->hdr_copy;
+		}
 		mail_cache_update_need_compress(cache);
 	}
 
@@ -732,7 +824,14 @@
 	}
 
 	cache->locked = FALSE;
+	cache->locked_hdr_refresh = FALSE;
 	mail_cache_unlock_file(cache);
+
+	if (trans != NULL) {
+		/* commit only after unlocking cache file to avoid deadlocks */
+		if (mail_index_transaction_commit(&trans) < 0)
+			ret = -1;
+	}
 	return ret;
 }
 
