[dovecot-cvs] dovecot/src/lib-index mail-index-lock.c, 1.9, 1.10 mail-index-private.h, 1.9, 1.10 mail-index-sync.c, 1.4, 1.5 mail-index-view-private.h, 1.3, 1.4 mail-index-view-sync.c, 1.2, 1.3 mail-index-view.c, 1.4, 1.5 mail-transaction-log-view.c, 1.5, 1.6 mail-transaction-log.c, 1.11, 1.12 mail-transaction-log.h, 1.2, 1.3

Sat May 1 17:30:40 EEST 2004

Update of /home/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv19443/lib-index

Modified Files:
	mail-index-lock.c mail-index-private.h mail-index-sync.c 
	mail-index-view-private.h mail-index-view-sync.c 
	mail-index-view.c mail-transaction-log-view.c 
	mail-transaction-log.c mail-transaction-log.h 
Log Message:
fixes



Index: mail-index-lock.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-lock.c,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10

--- mail-index-lock.c	28 Apr 2004 20:51:22 -0000	1.9
+++ mail-index-lock.c	1 May 2004 14:30:37 -0000	1.10
@@ -113,17 +113,16 @@
 	}
 }
 
-static int mail_index_lock_mprotect(struct mail_index *index, int lock_type)
+int mail_index_map_lock_mprotect(struct mail_index *index,
+				 struct mail_index_map *map, int lock_type)
 {
 	int prot;
 
-	if (index->map != NULL &&
-	    !MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
+	if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
 		prot = lock_type == F_UNLCK ? PROT_NONE :
 			lock_type == F_WRLCK ? (PROT_READ|PROT_WRITE) :
 			PROT_READ;
-		if (mprotect(index->map->mmap_base,
-			     index->map->mmap_size, prot) < 0) {
+		if (mprotect(map->mmap_base, map->mmap_size, prot) < 0) {
 			mail_index_set_syscall_error(index, "mprotect()");
 			return -1;
 		}
@@ -131,6 +130,14 @@
 	return 0;
 }
 
+static int mail_index_lock_mprotect(struct mail_index *index, int lock_type)
+{
+	if (index->map == NULL)
+		return 0;
+
+	return mail_index_map_lock_mprotect(index, index->map, lock_type);
+}
+
 static int mail_index_lock(struct mail_index *index, int lock_type,
 			   unsigned int timeout_secs, int update_index,
 			   unsigned int *lock_id_r)
@@ -142,14 +149,23 @@
 	if (lock_type == F_RDLCK && index->lock_type != F_UNLCK) {
 		index->shared_lock_count++;
 		*lock_id_r = index->lock_id;
-		return 1;
-	}
-	if (lock_type == F_WRLCK && index->lock_type == F_WRLCK) {
+		ret = 1;
+	} else if (lock_type == F_WRLCK && index->lock_type == F_WRLCK) {
 		index->excl_lock_count++;
 		*lock_id_r = index->lock_id + 1;
-		return 1;
+		ret = 1;
+	} else {
+		ret = 0;
 	}
 
+	if (update_index && index->excl_lock_count == 0) {
+		if (mail_index_has_changed(index) < 0)
+			return -1;
+	}
+
+	if (ret > 0)
+		return 1;
+
 	if (index->fcntl_locks_disable) {
 		/* FIXME: exclusive locking will rewrite the index file every
 		   time. shouldn't really be needed.. reading doesn't require
@@ -169,11 +185,6 @@
 		return 1;
 	}
 
-	if (update_index) {
-		if (mail_index_has_changed(index) < 0)
-			return -1;
-	}
-
 	ret = file_wait_lock_full(index->fd, lock_type, timeout_secs,
 				  NULL, NULL);
 	if (ret <= 0) {

Index: mail-index-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-private.h,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- mail-index-private.h	29 Apr 2004 12:59:16 -0000	1.9
+++ mail-index-private.h	1 May 2004 14:30:37 -0000	1.10
@@ -103,6 +103,8 @@
 void mail_index_unlock(struct mail_index *index, unsigned int lock_id);
 /* Returns 1 if given lock_id is valid, 0 if not. */
 int mail_index_is_locked(struct mail_index *index, unsigned int lock_id);
+int mail_index_map_lock_mprotect(struct mail_index *index,
+				 struct mail_index_map *map, int lock_type);
 
 /* Map index file to memory, replacing the previous mapping for index.
    Returns 1 = ok, 0 = corrupted, -1 = error. If index needs fscking, it

Index: mail-index-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-sync.c,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- mail-index-sync.c	28 Apr 2004 18:52:06 -0000	1.4
+++ mail-index-sync.c	1 May 2004 14:30:37 -0000	1.5
@@ -344,6 +344,16 @@
 		   it's quite unlikely this expunge was caused by some bug. */
 		uint32_t uid1, uid2;
 
+		if (next_exp->seq1 > ctx->view->map->records_count ||
+		    next_exp->seq2 > ctx->view->map->records_count) {
+			mail_transaction_log_view_set_corrupted(
+				ctx->view->log_view, "Expunge range %u..%u "
+				"larger than message count %u",
+				next_exp->seq1, next_exp->seq2,
+				ctx->view->map->records_count);
+			return -1;
+		}
+
 		if (mail_index_lookup_uid(ctx->view, next_exp->seq1, &uid1) < 0)
 			return -1;
 		if (mail_index_lookup_uid(ctx->view, next_exp->seq2, &uid2) < 0)

Index: mail-index-view-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-view-private.h,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- mail-index-view-private.h	28 Apr 2004 02:42:34 -0000	1.3
+++ mail-index-view-private.h	1 May 2004 14:30:37 -0000	1.4
@@ -7,6 +7,7 @@
 	struct mail_index *index;
         struct mail_transaction_log_view *log_view;
 
+	unsigned int indexid;
 	struct mail_index_map *map;
 
 	uint32_t log_file_seq;
@@ -19,6 +20,7 @@
 	unsigned int inconsistent:1;
 	unsigned int syncing:1;
 	unsigned int external:1;
+	unsigned int map_protected:1;
 };
 
 int mail_index_view_lock(struct mail_index_view *view);

Index: mail-index-view-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-view-sync.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mail-index-view-sync.c	28 Apr 2004 00:21:00 -0000	1.2
+++ mail-index-view-sync.c	1 May 2004 14:30:37 -0000	1.3
@@ -160,6 +160,7 @@
 
 	map->records_count++;
 	map->hdr_copy.messages_count++;
+	map->hdr_copy.next_uid = rec->uid+1;
 
 	mail_index_header_update_counts(&map->hdr_copy, 0, rec->flags);
 	mail_index_header_update_lowwaters(&map->hdr_copy, rec);

Index: mail-index-view.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-view.c,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- mail-index-view.c	28 Apr 2004 02:42:34 -0000	1.4
+++ mail-index-view.c	1 May 2004 14:30:37 -0000	1.5
@@ -14,6 +14,7 @@
 	view->index = index;
 	view->log_view = mail_transaction_log_view_open(index->log);
 
+	view->indexid = index->indexid;
 	view->map = index->map;
 	view->map->refcount++;
 
@@ -33,8 +34,27 @@
 	i_free(view);
 }
 
+static int mail_index_view_map_protect(struct mail_index_view *view)
+{
+	/* not head mapping, no need to lock */
+	if (!view->map_protected) {
+		if (mail_index_map_lock_mprotect(view->index, view->map,
+						 F_RDLCK) < 0)
+			return -1;
+		view->map_protected = TRUE;
+	}
+	return 0;
+}
+
 int mail_index_view_lock_head(struct mail_index_view *view, int update_index)
 {
+	unsigned int lock_id;
+
+	if (view->map != view->index->map) {
+		if (mail_index_view_map_protect(view) < 0)
+			return -1;
+	}
+
 	if (!mail_index_is_locked(view->index, view->lock_id)) {
 		if (mail_index_lock_shared(view->index, update_index,
 					   &view->lock_id) < 0)
@@ -45,13 +65,17 @@
 			return -1;
 		}
 
-		if (view->index->indexid != view->map->hdr->indexid) {
+		if (view->index->indexid != view->indexid) {
 			/* index was rebuilt */
 			view->inconsistent = TRUE;
 			return -1;
 		}
 	} else if (update_index) {
-		// FIXME: check if we need to reopen it!
+		if (mail_index_lock_shared(view->index, TRUE, &lock_id) < 0)
+			return -1;
+
+		mail_index_unlock(view->index, view->lock_id);
+		view->lock_id = lock_id;
 	}
 
 	return 0;
@@ -63,7 +87,8 @@
 		return -1;
 
 	if (view->map != view->index->map) {
-		/* not head mapping, no need to lock */
+		if (mail_index_view_map_protect(view) < 0)
+			return -1;
 		return 0;
 	}
 
@@ -72,6 +97,12 @@
 
 void mail_index_view_unlock(struct mail_index_view *view)
 {
+	if (view->map_protected) {
+		(void)mail_index_map_lock_mprotect(view->index, view->map,
+						   F_UNLCK);
+		view->map_protected = FALSE;
+	}
+
 	if (view->lock_id != 0) {
 		mail_index_unlock(view->index, view->lock_id);
 		view->lock_id = 0;

Index: mail-transaction-log-view.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-log-view.c,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- mail-transaction-log-view.c	29 Apr 2004 00:08:00 -0000	1.5
+++ mail-transaction-log-view.c	1 May 2004 14:30:37 -0000	1.6
@@ -90,10 +90,10 @@
 	ret = mail_transaction_log_file_find(view->log, min_file_seq, &file);
 	if (ret <= 0) {
 		if (ret == 0 &&
-		    min_file_seq == view->log->tail->hdr.file_seq-1 &&
+		    min_file_seq == view->log->tail->hdr.prev_file_seq &&
 		    min_file_offset == view->log->tail->hdr.prev_file_offset) {
 			/* we can skip this */
-			min_file_seq++;
+			min_file_seq = view->log->tail->hdr.file_seq;
 			min_file_offset =
 				sizeof(struct mail_transaction_log_header);
 			ret = mail_transaction_log_file_find(view->log,
@@ -197,19 +197,26 @@
 			     const void **data_r)
 {
 	const struct mail_transaction_header *hdr;
-	struct mail_transaction_log_file *file = view->file;
+	struct mail_transaction_log_file *file;
 	const struct mail_transaction_type_map *type_rec;
 	const void *data;
 	unsigned int record_size;
 	size_t file_size;
 
-	view->prev_file_seq = file->hdr.file_seq;
-	view->prev_file_offset = view->file_offset;
+	for (;;) {
+		file = view->file;
+
+		view->prev_file_seq = file->hdr.file_seq;
+		view->prev_file_offset = view->file_offset;
+
+		if (view->file_offset != file->hdr.used_size)
+			break;
 
-	if (view->file_offset == file->hdr.used_size) {
 		view->file = file->next;
 		view->file_offset = sizeof(struct mail_transaction_log_header);
-		return 0;
+
+		if (view->file == NULL)
+			return 0;
 	}
 
 	data = buffer_get_data(file->buffer, &file_size);

Index: mail-transaction-log.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-log.c,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -d -r1.11 -r1.12
--- mail-transaction-log.c	29 Apr 2004 12:49:27 -0000	1.11
+++ mail-transaction-log.c	1 May 2004 14:30:37 -0000	1.12
@@ -63,7 +63,7 @@
 
 #define INDEX_HAS_MISSING_LOGS(index, file) \
 	((file)->hdr.file_seq != (index)->hdr->log_file_seq && \
-	 ((file)->hdr.file_seq != (index)->hdr->log_file_seq+1 || \
+	 ((file)->hdr.prev_file_seq != (index)->hdr->log_file_seq || \
 	  (file)->hdr.prev_file_offset != (index)->hdr->log_file_offset))
 
 
@@ -78,6 +78,8 @@
 		return -1;
 
 	file = log->head;
+	file->refcount++;
+
 	ret = mail_index_lock_shared(index, TRUE, &lock_id);
 	if (ret == 0) {
 		ret = mail_index_map(index, FALSE);
@@ -88,7 +90,11 @@
 			ret = mail_transaction_log_rotate(log);
 		}
 	}
-	(void)mail_transaction_log_file_lock(file, F_UNLCK);
+
+	if (--file->refcount == 0)
+		mail_transaction_logs_clean(log);
+	else
+		(void)mail_transaction_log_file_lock(file, F_UNLCK);
 	return ret;
 }
 
@@ -330,11 +336,16 @@
 	hdr.indexid = index->indexid;
 	hdr.used_size = sizeof(hdr);
 
-	if (index->fd != -1)
+	if (index->fd != -1) {
+		hdr.prev_file_seq = index->hdr->log_file_seq;
 		hdr.prev_file_offset = index->hdr->log_file_offset;
+	}
 	hdr.file_seq = index->hdr->log_file_seq+1;
 
-	i_assert(log->head == NULL || hdr.file_seq > log->head->hdr.file_seq);
+	if (log->head != NULL && hdr.file_seq <= log->head->hdr.file_seq) {
+		/* make sure the sequence grows */
+		hdr.file_seq = log->head->hdr.file_seq+1;
+	}
 
 	if (write_full(fd, &hdr, sizeof(hdr)) < 0) {
 		mail_index_file_set_syscall_error(index, path, "write_full()");
@@ -439,14 +450,15 @@
 
 void mail_transaction_logs_clean(struct mail_transaction_log *log)
 {
-	struct mail_transaction_log_file **p;
+	struct mail_transaction_log_file **p, *next;
 
 	for (p = &log->tail; *p != NULL; ) {
 		if ((*p)->refcount != 0)
                         p = &(*p)->next;
 		else {
-                        mail_transaction_log_file_close(*p);
-			*p = (*p)->next;
+			next = (*p)->next;
+			mail_transaction_log_file_close(*p);
+			*p = next;
 		}
 	}
 }
@@ -455,7 +467,7 @@
 {
 	struct mail_transaction_log_file *file;
 	struct stat st;
-	int fd;
+	int fd, lock_type;
 
 	if (fstat(log->head->fd, &st) < 0) {
 		mail_index_file_set_syscall_error(log->index,
@@ -473,11 +485,17 @@
 	if (file == NULL)
 		return -1;
 
-	if (log->head != NULL) {
-		if (--log->head->refcount == 0)
-			mail_transaction_logs_clean(log);
+	lock_type = log->head->lock_type;
+	if (lock_type != F_UNLCK) {
+		if (mail_transaction_log_file_lock(file, lock_type) < 0)
+			return -1;
 	}
 
+	if (--log->head->refcount == 0)
+		mail_transaction_logs_clean(log);
+	else
+		(void)mail_transaction_log_file_lock(log->head, F_UNLCK);
+
 	log->head = file;
 	return 0;
 }
@@ -988,6 +1006,7 @@
 				uoff_t *log_file_offset_r)
 {
 	struct mail_index_view *view = t->view;
+	struct mail_index *index;
 	struct mail_transaction_log *log;
 	struct mail_transaction_log_file *file;
 	size_t offset;
@@ -1000,7 +1019,8 @@
 		return 0;
 	}
 
-	log = mail_index_view_get_index(view)->log;
+	index = mail_index_view_get_index(view);
+	log = index->log;
 
 	if (log->index->log_locked) {
 		i_assert(view->external);
@@ -1008,6 +1028,19 @@
 		if (mail_transaction_log_lock_head(log) < 0)
 			return -1;
 	}
+
+	if (log->head->hdr.file_seq == index->hdr->log_file_seq &&
+	    log->head->hdr.used_size > MAIL_TRANSACTION_LOG_ROTATE_SIZE) {
+		/* everything synced in index, we can rotate. */
+		if (mail_transaction_log_rotate(log) < 0) {
+			if (!log->index->log_locked) {
+				(void)mail_transaction_log_file_lock(log->head,
+								     F_UNLCK);
+			}
+			return -1;
+		}
+	}
+
 	file = log->head;
 	append_offset = file->hdr.used_size;
 

Index: mail-transaction-log.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-log.h,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mail-transaction-log.h	29 Apr 2004 00:08:00 -0000	1.2
+++ mail-transaction-log.h	1 May 2004 14:30:37 -0000	1.3
@@ -2,10 +2,12 @@
 #define __MAIL_TRANSACTION_LOG_H
 
 #define MAIL_TRANSACTION_LOG_PREFIX ".log"
+#define MAIL_TRANSACTION_LOG_ROTATE_SIZE (1024*128)
 
 struct mail_transaction_log_header {
 	uint32_t indexid;
 	uint32_t file_seq;
+	uint32_t prev_file_seq;
 	uint32_t prev_file_offset;
 	uint32_t used_size;
 };

