[dovecot-cvs] dovecot/src/lib-storage/index/maildir maildir-copy.c, 1.46, 1.47 maildir-save.c, 1.75, 1.76 maildir-storage.c, 1.132, 1.133 maildir-storage.h, 1.55, 1.56 maildir-sync.c, 1.84, 1.85 maildir-sync.h, NONE, 1.1 maildir-transaction.c, 1.13, 1.14 maildir-util.c, 1.16, 1.17

[tss at dovecot.org]

Update of /var/lib/cvs/dovecot/src/lib-storage/index/maildir
In directory talvi:/tmp/cvs-serv28938/lib-storage/index/maildir

Modified Files:
	maildir-copy.c maildir-save.c maildir-storage.c 
	maildir-storage.h maildir-sync.c maildir-transaction.c 
	maildir-util.c 
Added Files:
	maildir-sync.h 
Log Message:
Mailbox list indexing and related changes. Currently works only with
maildir and mmap_disable=no. This allows doing STATUS to synced mailboxes
without opening their index files at all.



Index: maildir-copy.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/maildir/maildir-copy.c,v
retrieving revision 1.46
retrieving revision 1.47
diff -u -d -r1.46 -r1.47
--- maildir-copy.c	17 Jun 2006 15:57:04 -0000	1.46
+++ maildir-copy.c	25 Nov 2006 22:17:42 -0000	1.47
@@ -7,6 +7,7 @@
 #include "maildir-storage.h"
 #include "maildir-uidlist.h"
 #include "maildir-keywords.h"
+#include "maildir-sync.h"
 #include "index-mail.h"
 #include "mail-copy.h"
 

Index: maildir-save.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/maildir/maildir-save.c,v
retrieving revision 1.75
retrieving revision 1.76
diff -u -d -r1.75 -r1.76
--- maildir-save.c	7 Nov 2006 21:22:08 -0000	1.75
+++ maildir-save.c	25 Nov 2006 22:17:42 -0000	1.76
@@ -12,6 +12,7 @@
 #include "index-mail.h"
 #include "maildir-storage.h"
 #include "maildir-uidlist.h"
+#include "maildir-sync.h"
 
 #include <stdio.h>
 #include <stdlib.h>

Index: maildir-storage.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/maildir/maildir-storage.c,v
retrieving revision 1.132
retrieving revision 1.133
diff -u -d -r1.132 -r1.133
--- maildir-storage.c	16 Nov 2006 00:16:34 -0000	1.132
+++ maildir-storage.c	25 Nov 2006 22:17:42 -0000	1.133
@@ -10,6 +10,7 @@
 #include "maildir-storage.h"
 #include "maildir-uidlist.h"
 #include "maildir-keywords.h"
+#include "maildir-sync.h"
 #include "index-mail.h"
 
 #include <stdio.h>
@@ -147,6 +148,8 @@
 
 	if (maildir_get_list_settings(&list_set, data, flags) < 0)
 		return NULL;
+	list_set.mail_storage_flags = &flags;
+	list_set.mail_storage_lock_method = &lock_method;
 
 	pool = pool_alloconly_create("storage", 512);
 	storage = p_new(pool, struct maildir_storage, 1);
@@ -727,7 +730,7 @@
 
 	mask = t_strdup_printf("%s%c*", oldname,
 			       mailbox_list_get_hierarchy_sep(storage->list));
-	iter = mailbox_list_iter_init(storage->list, "", mask,
+	iter = mailbox_list_iter_init(storage->list, mask,
 				      MAILBOX_LIST_ITER_FAST_FLAGS);
 	while ((info = mailbox_list_iter_next(iter)) != NULL) {
 		const char *name;

Index: maildir-storage.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/maildir/maildir-storage.h,v
retrieving revision 1.55
retrieving revision 1.56
diff -u -d -r1.55 -r1.56
--- maildir-storage.h	16 Nov 2006 00:16:34 -0000	1.55
+++ maildir-storage.h	25 Nov 2006 22:17:42 -0000	1.56
@@ -42,8 +42,6 @@
 struct timeval;
 struct maildir_save_context;
 struct maildir_copy_context;
-struct maildir_keywords_sync_ctx;
-struct maildir_index_sync_context;
 
 struct maildir_storage {
 	struct index_storage storage;
@@ -55,6 +53,11 @@
 	unsigned int stat_dirs:1;
 };
 
+enum maildir_dirty_flags {
+	MAILDIR_DIRTY_NEW = 0x01,
+	MAILDIR_DIRTY_CUR = 0x02
+};
+
 struct maildir_mailbox {
 	struct index_mailbox ibox;
 	struct maildir_storage *storage;
@@ -64,8 +67,9 @@
 	/* maildir sync: */
 	struct maildir_uidlist *uidlist;
 	struct maildir_keywords *keywords;
-	time_t last_new_mtime, last_cur_mtime, last_new_sync_time;
+	time_t last_new_mtime, last_cur_mtime;
 	time_t dirty_cur_time;
+	enum maildir_dirty_flags last_dirty_flags;
 
         mode_t mail_create_mode;
 	unsigned int private_flags_mask;
@@ -91,19 +95,6 @@
 		       mode_t mode, const char **fname_r);
 bool maildir_filename_get_size(const char *fname, char type, uoff_t *size_r);
 
-int maildir_sync_is_synced(struct maildir_mailbox *mbox);
-
-struct mailbox_sync_context *
-maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);
-int maildir_storage_sync_force(struct maildir_mailbox *mbox);
-
-int maildir_sync_index_begin(struct maildir_mailbox *mbox,
-			     struct maildir_index_sync_context **ctx_r);
-int maildir_sync_index(struct maildir_index_sync_context *sync_ctx,
-		       bool partial);
-int maildir_sync_index_finish(struct maildir_index_sync_context **sync_ctx,
-			      bool failed, bool cancel);
-
 void maildir_transaction_created(struct mail_index_transaction *t);
 void maildir_transaction_class_init(void);
 void maildir_transaction_class_deinit(void);
@@ -136,17 +127,6 @@
 int maildir_transaction_copy_commit(struct maildir_copy_context *ctx);
 void maildir_transaction_copy_rollback(struct maildir_copy_context *ctx);
 
-int maildir_sync_last_commit(struct maildir_mailbox *mbox);
-
-int maildir_filename_get_flags(struct maildir_keywords_sync_ctx *ctx,
-			       const char *fname, enum mail_flags *flags_r,
-			       ARRAY_TYPE(keyword_indexes) *keywords);
-struct maildir_keywords_sync_ctx *
-maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx);
-const char *maildir_filename_set_flags(struct maildir_keywords_sync_ctx *ctx,
-				       const char *fname, enum mail_flags flags,
-				       ARRAY_TYPE(keyword_indexes) *keywords);
-
 unsigned int maildir_hash(const void *p);
 int maildir_cmp(const void *p1, const void *p2);
 

Index: maildir-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/maildir/maildir-sync.c,v
retrieving revision 1.84
retrieving revision 1.85
diff -u -d -r1.84 -r1.85
--- maildir-sync.c	7 Nov 2006 21:22:08 -0000	1.84
+++ maildir-sync.c	25 Nov 2006 22:17:42 -0000	1.85
@@ -179,6 +179,7 @@
 #include "maildir-storage.h"
 #include "maildir-uidlist.h"
 #include "maildir-keywords.h"
+#include "maildir-sync.h"
 
 #include <stdio.h>
 #include <stddef.h>
@@ -186,8 +187,6 @@
 #include <dirent.h>
 #include <sys/stat.h>
 
-#define MAILDIR_SYNC_SECS 1
-
 #define MAILDIR_FILENAME_FLAG_FOUND 128
 
 /* When rename()ing many files from new/ to cur/, it's possible that next
@@ -774,17 +773,29 @@
 }
 
 static void
-maildir_sync_update_from_header(struct maildir_mailbox *mbox)
+maildir_sync_update_from_header(struct maildir_mailbox *mbox,
+				struct mail_index_header *hdr_r)
 {
-	uint64_t value;
+	struct mail_index_view *view;
+	const struct mail_index_header *hdr;
+
+	/* open a new view so we get the latest header */
+	view = mail_index_view_open(mbox->ibox.index);
+	hdr = mail_index_get_header(view);
 
 	/* FIXME: ugly, replace with extension header */
-	value = mail_index_get_header(mbox->ibox.view)->sync_size;
-	mbox->last_new_mtime = value & 0xffffffff;
-	mbox->last_new_sync_time = value >> 32;
+	mbox->last_new_mtime = hdr->sync_size & 0xffffffff;
+	mbox->last_dirty_flags = (hdr->sync_size >> 32) &
+		(MAILDIR_DIRTY_NEW | MAILDIR_DIRTY_CUR);
 
-	mbox->last_cur_mtime =
-		mail_index_get_header(mbox->ibox.view)->sync_stamp;
+	mbox->last_cur_mtime = hdr->sync_stamp;
+
+	if ((mbox->last_dirty_flags & MAILDIR_DIRTY_CUR) != 0 &&
+	    mbox->dirty_cur_time < mbox->last_cur_mtime)
+		mbox->dirty_cur_time = mbox->last_cur_mtime;
+
+	*hdr_r = *hdr;
+	mail_index_view_close(&view);
 }
 
 static int
@@ -793,6 +804,7 @@
 			 bool *new_changed_r, bool *cur_changed_r)
 {
 	struct index_mailbox *ibox = &mbox->ibox;
+	struct mail_index_header hdr;
 	struct stat st;
 	time_t new_mtime, cur_mtime;
 
@@ -817,7 +829,7 @@
 
 	   FIXME: For now we're using sync_size field as the new/ dir's stamp.
 	   Pretty ugly.. */
-        maildir_sync_update_from_header(mbox);
+        maildir_sync_update_from_header(mbox, &hdr);
 	if ((mbox->dirty_cur_time == 0 && cur_mtime != mbox->last_cur_mtime) ||
 	    (new_mtime != mbox->last_new_mtime)) {
 		/* check if the index has been updated.. */
@@ -826,18 +838,22 @@
 			return -1;
 		}
 
-		maildir_sync_update_from_header(mbox);
+		maildir_sync_update_from_header(mbox, &hdr);
 	}
 
 	/* If we're removing recent flags, always sync new/ directory if
 	   it has mails. */
 	if (new_mtime != mbox->last_new_mtime ||
-	    new_mtime >= mbox->last_new_sync_time - MAILDIR_SYNC_SECS ||
-	    (!ibox->keep_recent &&
-	     mail_index_get_header(ibox->view)->recent_messages_count > 0)) {
+	    ((mbox->last_dirty_flags & MAILDIR_DIRTY_NEW) != 0 &&
+	     new_mtime < ioloop_time - MAILDIR_SYNC_SECS) ||
+	    (!ibox->keep_recent && hdr.recent_messages_count > 0)) {
 		*new_changed_r = TRUE;
 		mbox->last_new_mtime = new_mtime;
-		mbox->last_new_sync_time = ioloop_time;
+
+		if (new_mtime < ioloop_time - MAILDIR_SYNC_SECS)
+			mbox->last_dirty_flags &= ~MAILDIR_DIRTY_NEW;
+		else
+			mbox->last_dirty_flags |= MAILDIR_DIRTY_NEW;
 	}
 
 	if (cur_mtime != mbox->last_cur_mtime ||
@@ -847,9 +863,13 @@
 		*cur_changed_r = TRUE;
 		mbox->last_cur_mtime = cur_mtime;
 
-		mbox->dirty_cur_time =
-			cur_mtime >= ioloop_time - MAILDIR_SYNC_SECS ?
-			cur_mtime : 0;
+		if (cur_mtime < ioloop_time - MAILDIR_SYNC_SECS) {
+			mbox->last_dirty_flags &= ~MAILDIR_DIRTY_CUR;
+			mbox->dirty_cur_time = 0;
+		} else {
+			mbox->last_dirty_flags |= MAILDIR_DIRTY_CUR;
+			mbox->dirty_cur_time = cur_mtime;
+		}
 	}
 
 	return 0;
@@ -947,7 +967,6 @@
 	ARRAY_TYPE(keyword_indexes) idx_keywords;
 	uint32_t uid_validity, next_uid;
 	uint64_t value;
-	time_t old_new_sync_time;
 	int ret = 0;
 	bool full_rescan = FALSE;
 
@@ -1179,8 +1198,10 @@
 		mbox->syncing_commit = FALSE;
 	}
 
-	if (mbox->dirty_cur_time == 0 &&
-	    mbox->last_cur_mtime != (time_t)hdr->sync_stamp) {
+	if (mbox->dirty_cur_time != 0)
+		mbox->last_dirty_flags |= MAILDIR_DIRTY_CUR;
+
+	if (mbox->last_cur_mtime != (time_t)hdr->sync_stamp) {
 		uint32_t sync_stamp = mbox->last_cur_mtime;
 
 		mail_index_update_header(trans,
@@ -1189,20 +1210,12 @@
 	}
 
 	/* FIXME: use a header extension instead of sync_size.. */
-	value = mbox->last_new_mtime;
-	old_new_sync_time = hdr->sync_size >> 32;
-	if (mbox->last_new_mtime >= old_new_sync_time - MAILDIR_SYNC_SECS) {
-		value |= (uint64_t)mbox->last_new_sync_time << 32;
-	} else {
-		value |= (uint64_t)old_new_sync_time << 32;
-	}
+	value = mbox->last_new_mtime |
+		((uint64_t)mbox->last_dirty_flags << 32);
 	if (value != hdr->sync_size) {
-		uint64_t sync_stamp = mbox->last_new_mtime |
-			((uint64_t)mbox->last_new_sync_time << 32);
-
 		mail_index_update_header(trans,
 			offsetof(struct mail_index_header, sync_size),
-			&sync_stamp, sizeof(sync_stamp), TRUE);
+			&value, sizeof(value), TRUE);
 	}
 
 	if (hdr->uid_validity == 0) {
@@ -1384,6 +1397,11 @@
 	struct maildir_sync_context *ctx;
 	int ret = 0;
 
+	if (!box->opened) {
+		if (index_storage_mailbox_open(&mbox->ibox) < 0)
+			return index_mailbox_sync_init(box, 0, TRUE);
+	}
+
 	if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
 	    mbox->ibox.sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <=
 	    ioloop_time) {

--- NEW FILE: maildir-sync.h ---
#ifndef __MAILDIR_SYNC_H
#define __MAILDIR_SYNC_H

#define MAILDIR_SYNC_SECS 1

struct maildir_mailbox;

struct maildir_keywords_sync_ctx;
struct maildir_index_sync_context;

int maildir_sync_is_synced(struct maildir_mailbox *mbox);

struct mailbox_sync_context *
maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);
int maildir_storage_sync_force(struct maildir_mailbox *mbox);

int maildir_sync_index_begin(struct maildir_mailbox *mbox,
			     struct maildir_index_sync_context **ctx_r);
int maildir_sync_index(struct maildir_index_sync_context *sync_ctx,
		       bool partial);
int maildir_sync_index_finish(struct maildir_index_sync_context **sync_ctx,
			      bool failed, bool cancel);

int maildir_sync_last_commit(struct maildir_mailbox *mbox);

int maildir_filename_get_flags(struct maildir_keywords_sync_ctx *ctx,
			       const char *fname, enum mail_flags *flags_r,
			       ARRAY_TYPE(keyword_indexes) *keywords);
struct maildir_keywords_sync_ctx *
maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx);
const char *maildir_filename_set_flags(struct maildir_keywords_sync_ctx *ctx,
				       const char *fname, enum mail_flags flags,
				       ARRAY_TYPE(keyword_indexes) *keywords);

#endif

Index: maildir-transaction.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/maildir/maildir-transaction.c,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -d -r1.13 -r1.14
--- maildir-transaction.c	15 Oct 2006 12:59:45 -0000	1.13
+++ maildir-transaction.c	25 Nov 2006 22:17:42 -0000	1.14
@@ -3,6 +3,7 @@
 #include "lib.h"
 #include "array.h"
 #include "maildir-storage.h"
+#include "maildir-sync.h"
 
 static void (*next_hook_mail_index_transaction_created)
 	(struct mail_index_transaction *t) = NULL;
@@ -54,7 +55,9 @@
 {
 	struct mailbox *box = MAIL_STORAGE_INDEX(t->view->index);
 
-	if (strcmp(box->storage->name, MAILDIR_STORAGE_NAME) == 0) {
+	/* index can be for mailbox list index, in which case box=NULL */
+	if (box != NULL &&
+	    strcmp(box->storage->name, MAILDIR_STORAGE_NAME) == 0) {
 		struct maildir_mailbox *mbox = (struct maildir_mailbox *)box;
 		struct maildir_transaction_context *mt;
 

Index: maildir-util.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/maildir/maildir-util.c,v
retrieving revision 1.16
retrieving revision 1.17
diff -u -d -r1.16 -r1.17
--- maildir-util.c	27 Jun 2006 22:51:47 -0000	1.16
+++ maildir-util.c	25 Nov 2006 22:17:43 -0000	1.17
@@ -7,6 +7,7 @@
 #include "maildir-storage.h"
 #include "maildir-uidlist.h"
 #include "maildir-keywords.h"
+#include "maildir-sync.h"
 
 #include <unistd.h>
 #include <fcntl.h>