dovecot: dovecot-uidlist can now be updated by appending to it. ...
dovecot at dovecot.org
dovecot at dovecot.org
Mon Jul 9 05:44:46 EEST 2007
details: http://hg.dovecot.org/dovecot/rev/fdc7e47ccea3
changeset: 5933:fdc7e47ccea3
user: Timo Sirainen <tss at iki.fi>
date: Mon Jul 09 05:43:03 2007 +0300
description:
dovecot-uidlist can now be updated by appending to it. It's recreated only
if there have been enough expunged messages.
diffstat:
1 file changed, 196 insertions(+), 88 deletions(-)
src/lib-storage/index/maildir/maildir-uidlist.c | 284 +++++++++++++++--------
diffs (truncated from 472 to 300 lines):
diff -r 6ac8d6c93d34 -r fdc7e47ccea3 src/lib-storage/index/maildir/maildir-uidlist.c
--- a/src/lib-storage/index/maildir/maildir-uidlist.c Mon Jul 09 04:40:54 2007 +0300
+++ b/src/lib-storage/index/maildir/maildir-uidlist.c Mon Jul 09 05:43:03 2007 +0300
@@ -27,6 +27,8 @@
/* how many seconds to wait before overriding uidlist.lock */
#define UIDLIST_LOCK_STALE_TIMEOUT (60*2)
+#define UIDLIST_COMPRESS_PERCENTAGE 75
+
#define UIDLIST_IS_LOCKED(uidlist) \
((uidlist)->lock_count > 0)
@@ -44,6 +46,7 @@ struct maildir_uidlist {
int fd;
dev_t fd_dev;
ino_t fd_ino;
+ off_t fd_size;
unsigned int lock_count;
@@ -57,8 +60,11 @@ struct maildir_uidlist {
unsigned int version;
unsigned int uid_validity, next_uid, prev_read_uid, last_seen_uid;
+ unsigned int read_records_count;
uint32_t first_recent_uid;
-
+ uoff_t last_read_offset;
+
+ unsigned int recreate:1;
unsigned int initial_read:1;
unsigned int initial_sync:1;
};
@@ -193,6 +199,7 @@ static void maildir_uidlist_close(struct
uidlist->fd = -1;
uidlist->fd_ino = 0;
}
+ uidlist->last_read_offset = 0;
}
void maildir_uidlist_deinit(struct maildir_uidlist *uidlist)
@@ -249,13 +256,6 @@ static int maildir_uidlist_next(struct m
return 1;
}
uidlist->last_seen_uid = uid;
-
- if (uid >= uidlist->next_uid) {
- mail_storage_set_critical(&uidlist->mbox->storage->storage,
- "UID larger than next_uid in file %s (%u >= %u)",
- uidlist->path, uid, uidlist->next_uid);
- return 0;
- }
while (*line == ' ') line++;
@@ -281,29 +281,76 @@ static int maildir_uidlist_next(struct m
return 1;
}
+static int maildir_uidlist_read_header(struct maildir_uidlist *uidlist,
+ struct istream *input)
+{
+ unsigned int uid_validity, next_uid;
+ const char *line;
+
+ line = i_stream_read_next_line(input);
+ if (line == NULL) {
+ /* I/O error / empty file */
+ return input->stream_errno == 0 ? 0 : -1;
+ }
+
+ if (sscanf(line, "%u %u %u", &uidlist->version,
+ &uid_validity, &next_uid) != 3 ||
+ uidlist->version < 1 || uidlist->version > 2) {
+ /* broken file */
+ mail_storage_set_critical(&uidlist->mbox->storage->storage,
+ "Corrupted header in file %s (version = %u)",
+ uidlist->path, uidlist->version);
+ return 0;
+ }
+ if (uid_validity == 0 || next_uid == 0) {
+ mail_storage_set_critical(&uidlist->mbox->storage->storage,
+ "%s: Broken header (uidvalidity = %u, next_uid=%u)",
+ uidlist->path, uid_validity, next_uid);
+ return 0;
+ }
+
+ uidlist->uid_validity = uid_validity;
+ uidlist->next_uid = next_uid;
+ return 1;
+}
+
static int
maildir_uidlist_update_read(struct maildir_uidlist *uidlist,
bool *retry_r, bool try_retry)
{
struct mail_storage *storage = &uidlist->mbox->storage->storage;
const char *line;
- unsigned int uid_validity, next_uid;
+ unsigned int orig_next_uid;
struct istream *input;
struct stat st;
+ uoff_t last_read_offset;
int fd, ret;
*retry_r = FALSE;
- maildir_uidlist_close(uidlist);
-
- fd = nfs_safe_open(uidlist->path, O_RDONLY);
- if (fd == -1) {
- if (errno != ENOENT) {
+ if (uidlist->fd == -1) {
+ fd = nfs_safe_open(uidlist->path, O_RDWR);
+ if (fd == -1) {
+ if (errno != ENOENT) {
+ mail_storage_set_critical(storage,
+ "open(%s) failed: %m", uidlist->path);
+ return -1;
+ }
+ return 0;
+ }
+ last_read_offset = 0;
+ } else {
+ /* the file was updated */
+ fd = uidlist->fd;
+ if (lseek(fd, 0, SEEK_SET) < 0) {
mail_storage_set_critical(storage,
- "open(%s) failed: %m", uidlist->path);
+ "lseek(%s) failed: %m", uidlist->path);
return -1;
}
- return 0;
+ uidlist->fd = -1;
+ uidlist->fd_ino = 0;
+ last_read_offset = uidlist->last_read_offset;
+ uidlist->last_read_offset = 0;
}
if (fstat(fd, &st) < 0) {
@@ -325,49 +372,38 @@ maildir_uidlist_update_read(struct maild
st.st_size/8));
}
- uidlist->version = 0;
-
input = i_stream_create_file(fd, default_pool, 4096, FALSE);
-
- /* get header */
- line = i_stream_read_next_line(input);
- if (line == NULL) {
- /* I/O error / empty file */
- ret = input->stream_errno == 0 ? 0 : -1;
- } else if (sscanf(line, "%u %u %u", &uidlist->version,
- &uid_validity, &next_uid) != 3 ||
- uidlist->version < 1 || uidlist->version > 2) {
- /* broken file */
- mail_storage_set_critical(storage,
- "Corrupted header in file %s (version = %u)",
- uidlist->path, uidlist->version);
- ret = 0;
- } else if (uid_validity == uidlist->uid_validity &&
- next_uid < uidlist->next_uid) {
- mail_storage_set_critical(storage,
- "%s: next_uid was lowered (%u -> %u)",
- uidlist->path, uidlist->next_uid, next_uid);
- ret = 0;
- } else if (uid_validity == 0 || next_uid == 0) {
- mail_storage_set_critical(storage,
- "%s: Broken header (uidvalidity = %u, next_uid=%u)",
- uidlist->path, uid_validity, next_uid);
- ret = 0;
- } else {
- uidlist->uid_validity = uid_validity;
- uidlist->next_uid = next_uid;
+ i_stream_seek(input, uidlist->last_read_offset);
+
+ orig_next_uid = uidlist->next_uid;
+ ret = input->v_offset != 0 ? 1 :
+ maildir_uidlist_read_header(uidlist, input);
+ if (ret > 0) {
uidlist->prev_read_uid = 0;
uidlist->change_counter++;
+ uidlist->read_records_count = 0;
ret = 1;
while ((line = i_stream_read_next_line(input)) != NULL) {
+ uidlist->read_records_count++;
if (!maildir_uidlist_next(uidlist, line)) {
ret = 0;
break;
}
}
- if (input->stream_errno != 0)
+ if (input->stream_errno != 0)
ret = -1;
+
+ if (uidlist->next_uid <= uidlist->prev_read_uid)
+ uidlist->next_uid = uidlist->prev_read_uid + 1;
+ if (uidlist->next_uid < orig_next_uid) {
+ mail_storage_set_critical(storage,
+ "%s: next_uid was lowered (%u -> %u)",
+ uidlist->path, orig_next_uid,
+ uidlist->next_uid);
+ uidlist->recreate = TRUE;
+ uidlist->next_uid = orig_next_uid;
+ }
}
if (ret == 0) {
@@ -378,6 +414,8 @@ maildir_uidlist_update_read(struct maild
uidlist->fd = fd;
uidlist->fd_dev = st.st_dev;
uidlist->fd_ino = st.st_ino;
+ uidlist->fd_size = st.st_size;
+ uidlist->last_read_offset = input->v_offset;
} else {
/* I/O error */
if (input->stream_errno == ESTALE && try_retry)
@@ -397,41 +435,62 @@ maildir_uidlist_update_read(struct maild
return ret;
}
-int maildir_uidlist_refresh(struct maildir_uidlist *uidlist)
+static int
+maildir_uidlist_has_changed(struct maildir_uidlist *uidlist, bool *recreated_r)
{
struct mail_storage *storage = &uidlist->mbox->storage->storage;
struct stat st;
+
+ *recreated_r = FALSE;
+
+ /* FIXME: nfs attribute cache flush */
+ if (nfs_safe_stat(uidlist->path, &st) < 0) {
+ if (errno != ENOENT) {
+ mail_storage_set_critical(storage,
+ "stat(%s) failed: %m", uidlist->path);
+ return -1;
+ }
+ return 0;
+ }
+
+ if (st.st_ino != uidlist->fd_ino ||
+ !CMP_DEV_T(st.st_dev, uidlist->fd_dev)) {
+ /* file recreated */
+ *recreated_r = TRUE;
+ return 1;
+ } else if (st.st_size != uidlist->fd_size) {
+ /* file modified but not recreated */
+ return 1;
+ } else {
+ /* unchanged */
+ return 0;
+ }
+}
+
+int maildir_uidlist_refresh(struct maildir_uidlist *uidlist)
+{
unsigned int i;
- bool retry;
+ bool retry, recreated;
int ret;
if (uidlist->fd != -1) {
- if (nfs_safe_stat(uidlist->path, &st) < 0) {
- if (errno != ENOENT) {
- mail_storage_set_critical(storage,
- "stat(%s) failed: %m", uidlist->path);
- return -1;
- }
- return 0;
- }
-
- if (st.st_ino == uidlist->fd_ino &&
- CMP_DEV_T(st.st_dev, uidlist->fd_dev)) {
- /* unchanged */
- return 1;
- }
+ ret = maildir_uidlist_has_changed(uidlist, &recreated);
+ if (ret <= 0)
+ return ret;
+
+ if (recreated)
+ maildir_uidlist_close(uidlist);
}
for (i = 0; ; i++) {
ret = maildir_uidlist_update_read(uidlist, &retry,
i < UIDLIST_ESTALE_RETRY_COUNT);
- if (!retry) {
- if (ret >= 0)
- uidlist->initial_read = TRUE;
- break;
- }
+ if (!retry)
+ break;
/* ESTALE - try reopening and rereading */
}
+ if (ret >= 0)
+ uidlist->initial_read = TRUE;
More information about the dovecot-cvs
mailing list