[dovecot-cvs] dovecot/src/lib-index mail-cache.c, 1.74, 1.75 mail-index-lock.c, 1.48, 1.49 mail-index-private.h, 1.60, 1.61 mail-index.c, 1.216, 1.217 mail-transaction-log-append.c, 1.15, 1.16 mail-transaction-log.c, 1.103, 1.104 mail-transaction-log.h, 1.28, 1.29

[cras at dovecot.org]

Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv14852

Modified Files:
	mail-cache.c mail-index-lock.c mail-index-private.h 
	mail-index.c mail-transaction-log-append.c 
	mail-transaction-log.c mail-transaction-log.h 
Log Message:
Beginnings of fallbacking to in-memory indexes when write fails with "out of
disk space" error.



Index: mail-cache.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-cache.c,v
retrieving revision 1.74
retrieving revision 1.75
diff -u -d -r1.74 -r1.75
--- mail-cache.c	6 Jan 2006 14:21:21 -0000	1.74
+++ mail-cache.c	6 Jan 2006 23:48:57 -0000	1.75
@@ -373,7 +373,8 @@
 
 	i_assert(!cache->locked);
 
-	if (MAIL_CACHE_IS_UNUSABLE(cache))
+	if (MAIL_CACHE_IS_UNUSABLE(cache) ||
+	    MAIL_INDEX_IS_IN_MEMORY(cache->index))
 		return 0;
 
 	view = mail_index_view_open(cache->index);

Index: mail-index-lock.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-lock.c,v
retrieving revision 1.48
retrieving revision 1.49
diff -u -d -r1.48 -r1.49
--- mail-index-lock.c	6 Jan 2006 21:16:43 -0000	1.48
+++ mail-index-lock.c	6 Jan 2006 23:48:57 -0000	1.49
@@ -38,8 +38,10 @@
 {
 	int ret;
 
-	if (MAIL_INDEX_IS_IN_MEMORY(index))
+	if (index->fd == -1) {
+		i_assert(MAIL_INDEX_IS_IN_MEMORY(index));
 		return 1;
+	}
 
 	if (timeout_secs != 0)
 		alarm(MAIL_INDEX_LOCK_WAIT_TIME);

Index: mail-index-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-private.h,v
retrieving revision 1.60
retrieving revision 1.61
diff -u -d -r1.60 -r1.61
--- mail-index-private.h	6 Jan 2006 12:57:06 -0000	1.60
+++ mail-index-private.h	6 Jan 2006 23:48:57 -0000	1.61
@@ -256,6 +256,7 @@
 void mail_index_view_transaction_unref(struct mail_index_view *view);
 
 void mail_index_set_inconsistent(struct mail_index *index);
+int mail_index_move_to_memory(struct mail_index *index);
 
 int mail_index_set_error(struct mail_index *index, const char *fmt, ...)
 	__attr_format__(2, 3);

Index: mail-index.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.216
retrieving revision 1.217
diff -u -d -r1.216 -r1.217
--- mail-index.c	6 Jan 2006 22:31:40 -0000	1.216
+++ mail-index.c	6 Jan 2006 23:48:57 -0000	1.217
@@ -1410,8 +1410,11 @@
 			lock_id = 0;
 		}
 		if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
-			if (mail_index_create(index, &hdr) < 0)
-				return -1;
+			if (mail_index_create(index, &hdr) < 0) {
+				/* fallback to in-memory index */
+				mail_index_move_to_memory(index);
+				mail_index_create_in_memory(index, &hdr);
+			}
 		} else {
 			mail_index_create_in_memory(index, &hdr);
 		}
@@ -1691,6 +1694,12 @@
 	index->indexid = 0;
 }
 
+int mail_index_move_to_memory(struct mail_index *index)
+{
+	i_free_and_null(index->dir);
+	return mail_transaction_log_move_to_memory(index->log);
+}
+
 void mail_index_mark_corrupted(struct mail_index *index)
 {
 	struct mail_index_header hdr;

Index: mail-transaction-log-append.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log-append.c,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- mail-transaction-log-append.c	6 Jan 2006 19:12:03 -0000	1.15
+++ mail-transaction-log-append.c	6 Jan 2006 23:48:57 -0000	1.16
@@ -17,6 +17,7 @@
 	struct mail_transaction_header hdr;
 	const void *data, *hdr_data;
 	size_t size, hdr_data_size;
+	uoff_t offset;
 	uint32_t hdr_size;
 
 	i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
@@ -45,38 +46,56 @@
 	hdr_size = mail_index_uint32_to_offset(sizeof(hdr) + size +
 					       hdr_data_size);
 	if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
-		if (file->first_append_size == 0) {
-			/* size will be written later once everything is in
-			   disk */
-			file->first_append_size = hdr_size;
-		} else {
-			hdr.size = hdr_size;
-		}
-		if (pwrite_full(file->fd, &hdr, sizeof(hdr),
-				file->sync_offset) < 0)
-			return -1;
-		file->sync_offset += sizeof(hdr);
+		do {
+			offset = file->sync_offset;
+			if (file->first_append_size == 0) {
+				/* size will be written later once everything
+				   is in disk */
+				file->first_append_size = hdr_size;
+			} else {
+				hdr.size = hdr_size;
+			}
+			if (pwrite_full(file->fd, &hdr, sizeof(hdr),
+					offset) < 0)
+				break;
+			offset += sizeof(hdr);
 
-		if (hdr_data_size > 0) {
-			if (pwrite_full(file->fd, hdr_data, hdr_data_size,
-					file->sync_offset) < 0)
-				return -1;
-			file->sync_offset += hdr_data_size;
-		}
+			if (hdr_data_size > 0) {
+				if (pwrite_full(file->fd, hdr_data,
+						hdr_data_size, offset) < 0)
+					break;
+				offset += hdr_data_size;
+			}
 
-		if (pwrite_full(file->fd, data, size, file->sync_offset) < 0)
+			if (pwrite_full(file->fd, data, size, offset) < 0)
+				break;
+
+			file->sync_offset = offset + size;
+			return 0;
+		} while (0);
+
+		/* write failure. */
+		if (!ENOSPACE(errno)) {
+			mail_index_file_set_syscall_error(file->log->index,
+							  file->filepath,
+							  "pwrite_full()");
 			return -1;
-		file->sync_offset += size;
-	} else {
-		hdr.size = hdr_size;
+		}
 
-		i_assert(file->buffer_offset + file->buffer->used ==
-			 file->sync_offset);
-		buffer_append(file->buffer, &hdr, sizeof(hdr));
-		buffer_append(file->buffer, hdr_data, hdr_data_size);
-		buffer_append(file->buffer, data, size);
-		file->sync_offset = file->buffer_offset + file->buffer->used;
+		/* not enough space. fallback to in-memory indexes. */
+		if (mail_index_move_to_memory(file->log->index) < 0)
+			return -1;
+		i_assert(MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file));
 	}
+
+	hdr.size = hdr_size;
+
+	i_assert(file->buffer_offset + file->buffer->used ==
+		 file->sync_offset);
+	buffer_append(file->buffer, &hdr, sizeof(hdr));
+	buffer_append(file->buffer, hdr_data, hdr_data_size);
+	buffer_append(file->buffer, data, size);
+	file->sync_offset = file->buffer_offset + file->buffer->used;
 	return 0;
 }
 
@@ -470,10 +489,6 @@
 					MAIL_TRANSACTION_EXPUNGE, t->external);
 	}
 
-	if (ret < 0) {
-		mail_index_file_set_syscall_error(log->index, file->filepath,
-						  "pwrite()");
-	}
 	if (t->post_hdr_changed && ret == 0) {
 		ret = log_append_buffer(file,
 					log_get_hdr_update_buffer(t, FALSE),
@@ -481,8 +496,8 @@
 					t->external);
 	}
 
-	if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
-		if (ret == 0 && file->first_append_size != 0) {
+	if (ret == 0 && file->first_append_size != 0) {
+		if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
 			/* synced - rewrite first record's header */
 			ret = pwrite_full(file->fd, &file->first_append_size,
 					  sizeof(uint32_t), append_offset);
@@ -490,6 +505,12 @@
 				mail_index_file_set_syscall_error(log->index,
 					file->filepath, "pwrite()");
 			}
+		} else {
+			/* changed into in-memory buffer in the middle */
+			buffer_write(file->buffer,
+				     append_offset - file->buffer_offset,
+				     &file->first_append_size,
+				     sizeof(file->first_append_size));
 		}
 	}
 

Index: mail-transaction-log.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.c,v
retrieving revision 1.103
retrieving revision 1.104
diff -u -d -r1.103 -r1.104
--- mail-transaction-log.c	6 Jan 2006 22:31:40 -0000	1.103
+++ mail-transaction-log.c	6 Jan 2006 23:48:57 -0000	1.104
@@ -30,6 +30,9 @@
 static struct mail_transaction_log_file *
 mail_transaction_log_file_open_or_create(struct mail_transaction_log *log,
 					 const char *path);
+static int
+mail_transaction_log_file_read(struct mail_transaction_log_file *file,
+			       uoff_t offset);
 
 void
 mail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,
@@ -237,8 +240,13 @@
 			   MAIL_TRANSACTION_LOG_SUFFIX, NULL);
 	log->head = mail_transaction_log_file_open_or_create(log, path);
 	if (log->head == NULL) {
-		mail_transaction_log_close(log);
-		return NULL;
+		/* fallback to in-memory indexes */
+		if (mail_index_move_to_memory(index) < 0) {
+			mail_transaction_log_close(log);
+			return NULL;
+		}
+		log->head = mail_transaction_log_file_open_or_create(log, path);
+		i_assert(log->head != NULL);
 	}
 
 	if (index->fd != -1 &&
@@ -268,16 +276,16 @@
 }
 
 static void
-mail_transaction_log_file_close(struct mail_transaction_log_file *file)
+mail_transaction_log_file_free(struct mail_transaction_log_file *file)
 {
+	mail_transaction_log_file_unlock(file);
+
 	if (file == file->log->head)
 		file->log->head = NULL;
 	if (file == file->log->tail)
 		file->log->tail = file->next;
 
-	mail_transaction_log_file_unlock(file);
-
-	if (file->buffer != NULL)
+	if (file->buffer != NULL) 
 		buffer_free(file->buffer);
 
 	if (file->mmap_base != NULL) {
@@ -300,6 +308,44 @@
 	i_free(file);
 }
 
+int mail_transaction_log_move_to_memory(struct mail_transaction_log *log)
+{
+	struct mail_transaction_log_file *file = log->head;
+
+	if (file == NULL || MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
+		return 0;
+
+	/* read the whole file to memory. we might currently be appending
+	   data into it, so we want to read it up to end of file */
+        file->buffer_offset = 0;
+
+	if (file->buffer != NULL) {
+		buffer_free(file->buffer);
+		file->buffer = NULL;
+	}
+
+	if (file->mmap_base != NULL) {
+		if (munmap(file->mmap_base, file->mmap_size) < 0) {
+			mail_index_file_set_syscall_error(file->log->index,
+							  file->filepath,
+							  "munmap()");
+		}
+		file->mmap_base = NULL;
+	}
+
+	if (mail_transaction_log_file_read(file, 0) <= 0)
+		return -1;
+
+	/* after we've read the file into memory, make it into in-memory
+	   log file */
+	if (close(file->fd) < 0) {
+		mail_index_file_set_syscall_error(file->log->index,
+						  file->filepath, "close()");
+	}
+	file->fd = -1;
+	return 0;
+}
+
 static int
 mail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,
 				   int head)
@@ -610,7 +656,7 @@
 
 	ret = mail_transaction_log_file_read_hdr(file, head);
 	if (ret < 0) {
-		mail_transaction_log_file_close(file);
+		mail_transaction_log_file_free(file);
 		return -1;
 	}
 
@@ -660,7 +706,7 @@
 		}
 	}
 	if (ret <= 0) {
-		mail_transaction_log_file_close(file);
+		mail_transaction_log_file_free(file);
 		return NULL;
 	}
 
@@ -733,7 +779,7 @@
 	if (ret <= 0) {
 		/* error / corrupted */
 		if (ret == 0)
-			mail_transaction_log_file_close(file);
+			mail_transaction_log_file_free(file);
 		return NULL;
 	}
 
@@ -750,7 +796,7 @@
                         p = &(*p)->next;
 		else {
 			next = (*p)->next;
-			mail_transaction_log_file_close(*p);
+			mail_transaction_log_file_free(*p);
 			*p = next;
 		}
 	}
@@ -905,7 +951,7 @@
 				i_error("unlink(%s) failed: %m",
 					file->filepath);
 			}
-			mail_transaction_log_file_close(file);
+			mail_transaction_log_file_free(file);
 			return 0;
 		}
 		return -1;

Index: mail-transaction-log.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.h,v
retrieving revision 1.28
retrieving revision 1.29
diff -u -d -r1.28 -r1.29
--- mail-transaction-log.h	6 Jan 2006 22:31:40 -0000	1.28
+++ mail-transaction-log.h	6 Jan 2006 23:48:57 -0000	1.29
@@ -109,6 +109,8 @@
 mail_transaction_log_open_or_create(struct mail_index *index);
 void mail_transaction_log_close(struct mail_transaction_log *log);
 
+int mail_transaction_log_move_to_memory(struct mail_transaction_log *log);
+
 struct mail_transaction_log_view *
 mail_transaction_log_view_open(struct mail_transaction_log *log);
 void mail_transaction_log_view_close(struct mail_transaction_log_view *view);