[dovecot-cvs] dovecot/src/lib-index/maildir maildir-uidlist.c,NONE,1.1 maildir-uidlist.h,NONE,1.1 Makefile.am,1.2,1.3 maildir-build.c,1.19,1.20 maildir-index.c,1.27,1.28 maildir-index.h,1.15,1.16 maildir-rebuild.c,1.9,1.10 maildir-sync.c,1.30,1.31
cras at procontrol.fi
cras at procontrol.fi
Thu Apr 10 00:10:04 EEST 2003
Update of /home/cvs/dovecot/src/lib-index/maildir
In directory danu:/tmp/cvs-serv22327/lib-index/maildir
Modified Files:
Makefile.am maildir-build.c maildir-index.c maildir-index.h
maildir-rebuild.c maildir-sync.c
Added Files:
maildir-uidlist.c maildir-uidlist.h
Log Message:
Rewritten maildir syncing. Uses dovecot-uidlist file to store UIDs
permanently.
--- NEW FILE: maildir-uidlist.c ---
/* Copyright (C) 2003 Timo Sirainen */
#include "lib.h"
#include "ioloop.h"
#include "istream.h"
#include "str.h"
#include "write-full.h"
#include "mail-index.h"
#include "mail-index-util.h"
#include "maildir-index.h"
#include "maildir-uidlist.h"
#include <stdio.h>
#include <sys/stat.h>
/* how many seconds to wait before overriding uidlist.lock */
#define UIDLIST_LOCK_STALE_TIMEOUT (60*5)
int maildir_uidlist_try_lock(struct mail_index *index)
{
struct stat st;
const char *path;
int fd, i;
i_assert(!INDEX_IS_UIDLIST_LOCKED(index));
path = t_strconcat(index->mailbox_path,
"/" MAILDIR_UIDLIST_NAME ".lock", NULL);
for (i = 0; i < 2; i++) {
fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644);
if (fd != -1)
break;
if (errno != EEXIST) {
index_file_set_syscall_error(index, path, "open()");
return -1;
}
/* exists, is it stale? */
if (stat(path, &st) < 0) {
if (errno == ENOENT) {
/* try again */
continue;
}
index_file_set_syscall_error(index, path, "stat()");
return -1;
}
if (st.st_mtime < ioloop_time - UIDLIST_LOCK_STALE_TIMEOUT) {
if (unlink(path) < 0 && errno != ENOENT) {
return index_file_set_syscall_error(index, path,
"unlink()");
}
/* try again */
continue;
}
return 0;
}
index->maildir_lock_fd = fd;
return 1;
}
void maildir_uidlist_unlock(struct mail_index *index)
{
const char *path;
if (!INDEX_IS_UIDLIST_LOCKED(index))
return;
path = t_strconcat(index->mailbox_path,
"/" MAILDIR_UIDLIST_NAME ".lock", NULL);
if (unlink(path) < 0 && errno != ENOENT)
index_file_set_syscall_error(index, path, "unlink()");
if (close(index->maildir_lock_fd) < 0)
index_file_set_syscall_error(index, path, "close()");
index->maildir_lock_fd = -1;
}
struct maildir_uidlist *maildir_uidlist_open(struct mail_index *index)
{
const char *path, *line;
struct maildir_uidlist *uidlist;
unsigned int version;
int fd;
path = t_strconcat(index->mailbox_path, "/" MAILDIR_UIDLIST_NAME, NULL);
fd = open(path, O_RDONLY);
if (fd == -1) {
if (errno != ENOENT)
index_file_set_syscall_error(index, path, "open()");
return NULL;
}
uidlist = i_new(struct maildir_uidlist, 1);
uidlist->index = index;
uidlist->fname = i_strdup(path);
uidlist->input = i_stream_create_file(fd, default_pool, 4096, TRUE);
/* get header */
line = i_stream_read_next_line(uidlist->input);
if (line == NULL || sscanf(line, "%u %u %u", &version,
&uidlist->uid_validity,
&uidlist->next_uid) != 3 ||
version != 1) {
/* broken file */
(void)unlink(path);
maildir_uidlist_close(uidlist);
return NULL;
}
return uidlist;
}
int maildir_uidlist_next(struct maildir_uidlist *uidlist,
struct maildir_uidlist_rec *uid_rec)
{
const char *line;
unsigned int uid;
memset(uid_rec, 0, sizeof(*uid_rec));
line = i_stream_read_next_line(uidlist->input);
if (line == NULL)
return 0;
uid = 0;
while (*line >= '0' && *line <= '9') {
uid = uid*10 + (*line - '0');
line++;
}
if (uid == 0 || *line != ' ') {
/* invalid file */
index_set_error(uidlist->index, "Invalid data in file %s",
uidlist->fname);
(void)unlink(uidlist->fname);
return -1;
}
if (uid <= uidlist->last_read_uid) {
index_set_error(uidlist->index,
"UIDs not ordered in file %s (%u > %u)",
uidlist->fname, uid, uidlist->last_read_uid);
(void)unlink(uidlist->fname);
return -1;
}
if (uid >= uidlist->next_uid) {
index_set_error(uidlist->index,
"UID larger than next_uid in file %s "
"(%u >= %u)", uidlist->fname,
uid, uidlist->next_uid);
(void)unlink(uidlist->fname);
return -1;
}
while (*line == ' ') line++;
uid_rec->uid = uid;
uid_rec->filename = line;
return 1;
}
void maildir_uidlist_close(struct maildir_uidlist *uidlist)
{
i_stream_unref(uidlist->input);
i_free(uidlist->fname);
i_free(uidlist);
}
int maildir_uidlist_rewrite(struct mail_index *index)
{
struct mail_index_record *rec;
const char *temp_path, *db_path, *p, *fname;
string_t *str;
size_t len;
int failed = FALSE;
i_assert(INDEX_IS_UIDLIST_LOCKED(index));
if (index->lock_type == MAIL_LOCK_UNLOCK) {
if (!index->set_lock(index, MAIL_LOCK_SHARED))
return FALSE;
}
temp_path = t_strconcat(index->mailbox_path,
"/" MAILDIR_UIDLIST_NAME ".lock", NULL);
str = t_str_new(4096);
str_printfa(str, "1 %u %u\n",
index->header->uid_validity, index->header->next_uid);
rec = index->lookup(index, 1);
while (rec != NULL) {
fname = maildir_get_location(index, rec);
if (fname == NULL)
break;
p = strchr(fname, ':');
len = p == NULL ? strlen(fname) : (size_t)(p-fname);
if (str_len(str) + MAX_INT_STRLEN + len + 2 >= 4096) {
/* flush buffer */
if (write_full(index->maildir_lock_fd,
str_data(str), str_len(str)) < 0) {
index_file_set_syscall_error(index, temp_path,
"write_full()");
break;
}
str_truncate(str, 0);
}
str_printfa(str, "%u ", rec->uid);
str_append_n(str, fname, len);
str_append_c(str, '\n');
rec = index->next(index, rec);
}
if (write_full(index->maildir_lock_fd,
str_data(str), str_len(str)) < 0) {
index_file_set_syscall_error(index, temp_path, "write_full()");
failed = TRUE;
}
if (fdatasync(index->maildir_lock_fd) < 0) {
index_file_set_syscall_error(index, temp_path, "fdatasync()");
failed = TRUE;
}
if (close(index->maildir_lock_fd) < 0) {
index_file_set_syscall_error(index, temp_path, "close()");
failed = TRUE;
}
index->maildir_lock_fd = -1;
if (rec == NULL) {
db_path = t_strconcat(index->mailbox_path,
"/" MAILDIR_UIDLIST_NAME, NULL);
if (rename(temp_path, db_path) < 0) {
index_set_error(index, "rename(%s, %s) failed: %m",
temp_path, db_path);
failed = TRUE;
}
}
if (failed)
(void)unlink(temp_path);
return !failed;
}
--- NEW FILE: maildir-uidlist.h ---
#ifndef __MAILDIR_UIDLIST_H
#define __MAILDIR_UIDLIST_H
#define INDEX_IS_UIDLIST_LOCKED(index) \
((index)->maildir_lock_fd != -1)
#define MAILDIR_UIDLIST_NAME "dovecot-uidlist"
struct maildir_uidlist {
struct mail_index *index;
char *fname;
struct istream *input;
unsigned int uid_validity, next_uid, last_read_uid;
unsigned int rewrite:1;
};
struct maildir_uidlist_rec {
unsigned int uid;
const char *filename;
};
int maildir_uidlist_try_lock(struct mail_index *index);
void maildir_uidlist_unlock(struct mail_index *index);
int maildir_uidlist_rewrite(struct mail_index *index);
struct maildir_uidlist *maildir_uidlist_open(struct mail_index *index);
void maildir_uidlist_close(struct maildir_uidlist *uidlist);
/* Returns -1 if error, 0 if end of file or 1 if found.
uid_rec.uid is also set to 0 at EOF. This function does sanity checks so
you can be sure that uid_rec.uid is always growing and smaller than
uidlist->next_uid. */
int maildir_uidlist_next(struct maildir_uidlist *uidlist,
struct maildir_uidlist_rec *uid_rec);
#endif
Index: Makefile.am
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/maildir/Makefile.am,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- Makefile.am 11 Feb 2003 15:07:31 -0000 1.2
+++ Makefile.am 9 Apr 2003 20:10:01 -0000 1.3
@@ -13,6 +13,7 @@
maildir-open.c \
maildir-rebuild.c \
maildir-sync.c \
+ maildir-uidlist.c \
maildir-update.c
noinst_HEADERS = \
Index: maildir-build.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/maildir/maildir-build.c,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -d -r1.19 -r1.20
--- maildir-build.c 5 Mar 2003 01:41:36 -0000 1.19
+++ maildir-build.c 9 Apr 2003 20:10:01 -0000 1.20
@@ -1,6 +1,7 @@
/* Copyright (C) 2002 Timo Sirainen */
#include "lib.h"
+#include "str.h"
#include "maildir-index.h"
#include "mail-index-data.h"
#include "mail-index-util.h"
@@ -103,64 +104,59 @@
return ret;
}
-int maildir_index_build_dir(struct mail_index *index, const char *source_dir,
- const char *dest_dir)
+int maildir_index_build_dir(struct mail_index *index,
+ const char *source_dir, const char *dest_dir,
+ DIR *dirp, struct dirent *d)
{
- DIR *dirp;
const char *final_dir;
- struct dirent *d;
- struct stat st;
- char sourcepath[PATH_MAX], destpath[PATH_MAX];
+ string_t *sourcepath, *destpath;
int failed;
+ i_assert(index->maildir_lock_fd != -1);
i_assert(index->lock_type != MAIL_LOCK_SHARED);
- i_assert(source_dir != NULL);
- dirp = opendir(source_dir);
- if (dirp == NULL) {
- return index_file_set_syscall_error(index, source_dir,
- "opendir()");
- }
+ sourcepath = t_str_new(PATH_MAX);
+ destpath = t_str_new(PATH_MAX);
final_dir = dest_dir != NULL ? dest_dir : source_dir;
failed = FALSE;
- while (!failed && (d = readdir(dirp)) != NULL) {
+ for (; d != NULL && !failed; d = readdir(dirp)) {
if (d->d_name[0] == '.')
continue;
if (dest_dir != NULL) {
- /* move the file into dest_dir - abort everything if it
- already exists, as that should never happen */
- if (str_path(sourcepath, sizeof(sourcepath),
- source_dir, d->d_name) < 0) {
- index_set_error(index, "Path too long: %s/%s",
- source_dir, d->d_name);
- failed = TRUE;
- break;
- }
- if (str_path(destpath, sizeof(destpath),
- dest_dir, d->d_name) < 0) {
- index_set_error(index, "Path too long: %s/%s",
- dest_dir, d->d_name);
- failed = TRUE;
- break;
- }
- if (stat(destpath, &st) == 0) {
- index_set_error(index, "Can't move mail %s to "
- "%s: file already exists",
- sourcepath, destpath);
- failed = TRUE;
- break;
- }
+ /* rename() has the problem that it might overwrite
+ some mails, but that happens only with a broken
+ client that has created non-unique base name.
- /* race condition here - ignore it as the chance of it
- happening is pretty much zero */
+ Alternative would be link() + unlink(), but that's
+ racy when multiple clients try to move the mail from
+ new/ to cur/:
- if (rename(sourcepath, destpath) < 0) {
+ a) One of the clients uses slightly different
+ filename (eg. sets flags)
+
+ b) Third client changes mail's flag between
+ client1's unlink() and client2's link() calls.
+
+ Checking first if file exists with stat() is pretty
+ useless as well. It requires that we also stat the
+ file in new/, to make sure that the dest file isn't
+ actually the same file which someone _just_ had
+ rename()d. */
+ str_truncate(sourcepath, 0);
+ str_truncate(destpath, 0);
+
+ str_printfa(sourcepath, "%s/%s", source_dir, d->d_name);
+ str_printfa(destpath, "%s/%s", dest_dir, d->d_name);
+
+ if (rename(str_c(sourcepath), str_c(destpath)) < 0 &&
+ errno != ENOENT) {
index_set_error(index, "maildir build: "
"rename(%s, %s) failed: %m",
- sourcepath, destpath);
+ str_c(sourcepath),
+ str_c(destpath));
failed = TRUE;
break;
}
@@ -172,7 +168,5 @@
t_pop();
}
- if (closedir(dirp) < 0)
- index_file_set_syscall_error(index, source_dir, "closedir()");
return !failed;
}
Index: maildir-index.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/maildir/maildir-index.c,v
retrieving revision 1.27
retrieving revision 1.28
diff -u -d -r1.27 -r1.28
--- maildir-index.c 30 Mar 2003 12:48:37 -0000 1.27
+++ maildir-index.c 9 Apr 2003 20:10:01 -0000 1.28
@@ -1,6 +1,8 @@
/* Copyright (C) 2002 Timo Sirainen */
#include "lib.h"
+#include "ioloop.h"
+#include "hostpid.h"
#include "str.h"
#include "maildir-index.h"
#include "mail-index-data.h"
@@ -8,6 +10,8 @@
#include <stdio.h>
#include <sys/stat.h>
+#include <sys/time.h>
+#include <time.h>
extern struct mail_index maildir_index;
@@ -31,6 +35,53 @@
return fname;
}
+const char *maildir_generate_tmp_filename(const struct timeval *tv)
+{
+ static unsigned int create_count = 0;
+
+ return t_strdup_printf("%s.P%sQ%uM%s.%s",
+ dec2str(tv->tv_sec), my_pid, create_count++,
+ dec2str(tv->tv_usec), my_hostname);
+}
+
+int maildir_create_tmp(struct mail_index *index, const char *dir,
+ const char **fname)
+{
+ const char *path, *tmp_fname;
+ struct stat st;
+ struct timeval *tv, tv_now;
+ pool_t pool;
+ int fd;
+
+ tv = &ioloop_timeval;
+ pool = pool_alloconly_create("maildir_tmp", 4096);
+ for (;;) {
+ p_clear(pool);
+ tmp_fname = maildir_generate_tmp_filename(tv);
+
+ path = p_strconcat(pool, dir, "/", tmp_fname, NULL);
+ if (stat(path, &st) < 0 && errno == ENOENT) {
+ /* doesn't exist */
+ fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0600);
+ if (fd != -1 || errno != EEXIST)
+ break;
+ }
+
+ /* wait and try again - very unlikely */
+ sleep(2);
+ tv = &tv_now;
+ if (gettimeofday(&tv_now, NULL) < 0)
+ i_fatal("gettimeofday(): %m");
+ }
+
+ *fname = t_strdup(path);
+ if (fd == -1)
+ index_file_set_syscall_error(index, path, "open()");
+
+ pool_unref(pool);
+ return fd;
+}
+
enum mail_flags maildir_filename_get_flags(const char *fname,
enum mail_flags default_flags)
{
@@ -163,6 +214,7 @@
index = i_new(struct mail_index, 1);
memcpy(index, &maildir_index, sizeof(struct mail_index));
+ index->maildir_lock_fd = -1;
index->mailbox_path = i_strdup(maildir);
mail_index_init(index, dir);
return index;
Index: maildir-index.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/maildir/maildir-index.h,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- maildir-index.h 9 Mar 2003 11:57:35 -0000 1.15
+++ maildir-index.h 9 Apr 2003 20:10:01 -0000 1.16
@@ -1,6 +1,7 @@
#ifndef __MAILDIR_INDEX_H
#define __MAILDIR_INDEX_H
+#include <dirent.h>
#include "mail-index.h"
/* ":2,DFRST" - leave the 2 extra for other clients' additions */
@@ -8,6 +9,11 @@
struct mail_index *maildir_index_alloc(const char *dir, const char *maildir);
+/* Return new filename base to save into tmp/ */
+const char *maildir_generate_tmp_filename(const struct timeval *tv);
+int maildir_create_tmp(struct mail_index *index, const char *dir,
+ const char **path);
+
const char *maildir_get_location(struct mail_index *index,
struct mail_index_record *rec);
enum mail_flags maildir_filename_get_flags(const char *fname,
@@ -21,8 +27,9 @@
int maildir_index_append_file(struct mail_index *index, const char *dir,
const char *fname);
-int maildir_index_build_dir(struct mail_index *index, const char *source_dir,
- const char *dest_dir);
+int maildir_index_build_dir(struct mail_index *index,
+ const char *source_dir, const char *dest_dir,
+ DIR *dirp, struct dirent *d);
struct istream *maildir_open_mail(struct mail_index *index,
struct mail_index_record *rec,
Index: maildir-rebuild.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/maildir/maildir-rebuild.c,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- maildir-rebuild.c 5 Jan 2003 13:09:52 -0000 1.9
+++ maildir-rebuild.c 9 Apr 2003 20:10:01 -0000 1.10
@@ -11,11 +11,6 @@
int maildir_index_rebuild(struct mail_index *index)
{
- struct stat st;
- const char *cur_dir, *new_dir;
-
- i_assert(index->lock_type != MAIL_LOCK_SHARED);
-
if (!mail_index_set_lock(index, MAIL_LOCK_EXCLUSIVE))
return FALSE;
@@ -30,6 +25,7 @@
changed */
index->indexid = index->header->indexid;
index->inconsistent = TRUE;
+ index->rebuilding = TRUE;
if (msync(index->mmap_base,
sizeof(struct mail_index_header), MS_SYNC) < 0)
@@ -39,23 +35,12 @@
if (!mail_index_data_reset(index->data))
return FALSE;
- /* rebuild cur/ directory */
- cur_dir = t_strconcat(index->mailbox_path, "/cur", NULL);
- if (!maildir_index_build_dir(index, cur_dir, NULL))
- return FALSE;
-
- /* also see if there's new mail */
- new_dir = t_strconcat(index->mailbox_path, "/new", NULL);
- if (!maildir_index_build_dir(index, new_dir, cur_dir))
+ /* read the mails by syncing */
+ if (!index->sync_and_lock(index, MAIL_LOCK_UNLOCK, NULL))
return FALSE;
- /* update sync stamp */
- if (stat(cur_dir, &st) < 0)
- return index_file_set_syscall_error(index, cur_dir, "stat()");
-
- index->file_sync_stamp = st.st_mtime;
-
/* rebuild is complete - remove the flag */
index->header->flags &= ~(MAIL_INDEX_FLAG_REBUILD|MAIL_INDEX_FLAG_FSCK);
+ index->rebuilding = FALSE;
return TRUE;
}
Index: maildir-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/maildir/maildir-sync.c,v
retrieving revision 1.30
retrieving revision 1.31
diff -u -d -r1.30 -r1.31
--- maildir-sync.c 9 Mar 2003 11:57:35 -0000 1.30
+++ maildir-sync.c 9 Apr 2003 20:10:01 -0000 1.31
@@ -1,9 +1,11 @@
-/* Copyright (C) 2002 Timo Sirainen */
+/* Copyright (C) 2002-2003 Timo Sirainen */
#include "lib.h"
-#include "ioloop.h"
+#include "buffer.h"
#include "hash.h"
+#include "ioloop.h"
#include "maildir-index.h"
+#include "maildir-uidlist.h"
#include "mail-index-data.h"
#include "mail-index-util.h"
@@ -14,185 +16,294 @@
#include <utime.h>
#include <sys/stat.h>
-static int maildir_index_sync_file(struct mail_index *index,
+enum maildir_file_action {
+ MAILDIR_FILE_ACTION_EXPUNGE,
+ MAILDIR_FILE_ACTION_UPDATE_FLAGS,
+ MAILDIR_FILE_ACTION_UPDATE_CONTENT,
+ MAILDIR_FILE_ACTION_NEW,
+ MAILDIR_FILE_ACTION_NONE
+};
+
+struct maildir_hash_context {
+ struct mail_index *index;
+ struct mail_index_record *new_mail;
+
+ int failed;
+};
+
+struct maildir_hash_rec {
+ struct mail_index_record *rec;
+ enum maildir_file_action action;
+};
+
+static int maildir_update_filename(struct mail_index *index,
struct mail_index_record *rec,
- unsigned int seq, const char *fname,
- const char *path, int fname_changed)
+ const char *new_fname)
{
struct mail_index_update *update;
+
+ update = index->update_begin(index, rec);
+ index->update_field(update, DATA_FIELD_LOCATION, new_fname, 0);
+ return index->update_end(update);
+}
+
+static int maildir_update_flags(struct mail_index *index,
+ struct mail_index_record *rec,
+ unsigned int seq, const char *new_fname)
+{
enum mail_flags flags;
- int failed;
- i_assert(fname != NULL);
- i_assert(path != NULL);
+ flags = maildir_filename_get_flags(new_fname, rec->msg_flags);
+ if (flags != rec->msg_flags) {
+ if (!index->update_flags(index, rec, seq, flags, TRUE))
+ return FALSE;
+ }
- if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
+ return TRUE;
+}
+
+static int is_file_content_changed(struct mail_index *index,
+ struct mail_index_record *rec,
+ const char *dir, const char *fname)
+{
+#define DATA_HDR_SIZE (DATA_HDR_HEADER_SIZE | DATA_HDR_BODY_SIZE)
+ struct mail_index_data_record_header *data_hdr;
+ struct stat st;
+ const char *path;
+
+ if ((rec->data_fields & DATA_HDR_INTERNAL_DATE) == 0 &&
+ (rec->data_fields & DATA_HDR_SIZE) != DATA_HDR_SIZE) {
+ /* nothing in cache, we can't know if it's changed */
return FALSE;
+ }
- failed = FALSE;
- update = index->update_begin(index, rec);
+ t_push();
+ path = t_strdup_printf("%s/%s", dir, fname);
- if (fname_changed)
- index->update_field(update, DATA_FIELD_LOCATION, fname, 0);
+ if (stat(path, &st) < 0) {
+ if (errno != ENOENT)
+ index_file_set_syscall_error(index, path, "stat()");
+ t_pop();
+ return FALSE;
+ }
+ t_pop();
- if (!index->update_end(update))
- failed = TRUE;
+ data_hdr = mail_index_data_lookup_header(index->data, rec);
+ if (data_hdr == NULL)
+ return FALSE;
- /* update flags after filename has been updated, so it can be
- compared correctly */
- flags = maildir_filename_get_flags(fname, rec->msg_flags);
- if (!failed && flags != rec->msg_flags) {
- if (!index->update_flags(index, rec, seq, flags, TRUE))
- failed = TRUE;
+ if ((rec->data_fields & DATA_HDR_INTERNAL_DATE) != 0 &&
+ st.st_mtime != data_hdr->internal_date)
+ return TRUE;
+
+ if ((rec->data_fields & DATA_HDR_SIZE) == DATA_HDR_SIZE &&
+ (uoff_t)st.st_size != data_hdr->body_size + data_hdr->header_size)
+ return TRUE;
+
+ return FALSE;
+}
+
+/* a char* hash function from ASU -- from glib */
+static unsigned int maildir_hash(const void *p)
+{
+ const unsigned char *s = p;
+ unsigned int g, h = 0;
+
+ while (*s != ':' && *s != '\0') {
+ h = (h << 4) + *s;
+ if ((g = h & 0xf0000000UL)) {
+ h = h ^ (g >> 24);
+ h = h ^ g;
+ }
+ s++;
}
- return !failed;
+ return h;
}
-static int maildir_index_sync_files(struct mail_index *index, const char *dir,
- struct hash_table *files,
- int check_content_changes)
+static int maildir_cmp(const void *p1, const void *p2)
{
- struct mail_index_record *rec;
- struct mail_index_data_record_header *data_hdr;
- struct stat st;
- const char *fname, *base_fname, *value;
- char path[PATH_MAX];
- unsigned int seq;
- int fname_changed;
+ const char *s1 = p1, *s2 = p2;
- i_assert(dir != NULL);
+ while (*s1 == *s2 && *s1 != ':' && *s1 != '\0') {
+ s1++; s2++;
+ }
+ if ((*s1 == '\0' || *s1 == ':') &&
+ (*s2 == '\0' || *s2 == '\0'))
+ return 0;
+ return *s1 - *s2;
+}
- rec = index->lookup(index, 1);
- for (seq = 1; rec != NULL; rec = index->next(index, rec)) {
- fname = maildir_get_location(index, rec);
- if (fname == NULL)
- return FALSE;
+static void uidlist_hash_get_filenames(void *key, void *value, void *context)
+{
+ buffer_t *buf = context;
+ struct maildir_hash_rec *hash_rec = value;
- t_push();
+ if (hash_rec->action == MAILDIR_FILE_ACTION_NEW)
+ buffer_append(buf, (const void *) &key, sizeof(key));
+}
- /* get the filename without the ":flags" part */
- base_fname = t_strcut(fname, ':');
+static int maildir_sync_uidlist(struct mail_index *index, const char *dir,
+ struct maildir_uidlist *uidlist,
+ struct hash_table *files, pool_t pool,
+ unsigned int new_count)
+{
+ struct mail_index_record *rec;
+ struct maildir_hash_rec *hash_rec;
+ struct maildir_uidlist_rec uid_rec;
+ const char *fname, **new_files;
+ void *orig_key, *orig_value;
+ unsigned int seq, last_uid, i;
+ buffer_t *buf;
- value = hash_lookup(files, base_fname);
- if (value != NULL)
- hash_remove(files, base_fname);
+ seq = 0;
+ rec = index->lookup(index, 1);
- t_pop();
+ if (uidlist == NULL)
+ memset(&uid_rec, 0, sizeof(uid_rec));
+ else {
+ if (maildir_uidlist_next(uidlist, &uid_rec) < 0)
+ return FALSE;
+ }
- if (value == NULL) {
- /* mail is expunged */
- if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
- return FALSE;
+ while (rec != NULL) {
+ seq++;
- if (!index->expunge(index, rec, seq, TRUE))
+ /* skip over the expunged records in uidlist */
+ while (uid_rec.uid != 0 && uid_rec.uid < rec->uid) {
+ uidlist->rewrite = TRUE;
+ if (!maildir_uidlist_next(uidlist, &uid_rec))
return FALSE;
- continue;
}
- /* file still exists */
- if (str_path(path, sizeof(path), dir, value) < 0) {
- index_set_error(index, "Path too long: %s/%s",
- dir, value);
+ fname = maildir_get_location(index, rec);
+ if (fname == NULL) {
+ hash_destroy(files);
return FALSE;
}
- if (check_content_changes) {
- if (stat(path, &st) < 0) {
- index_file_set_syscall_error(index, path,
- "stat()");
- return FALSE;
- }
-
- data_hdr = mail_index_data_lookup_header(index->data,
- rec);
- if (data_hdr != NULL &&
- (st.st_mtime != data_hdr->internal_date ||
- (uoff_t)st.st_size !=
- data_hdr->body_size + data_hdr->header_size)) {
- /* file changed. IMAP doesn't allow that, so
- we have to treat it as a new message. */
- if (!index->set_lock(index,
- MAIL_LOCK_EXCLUSIVE))
- return FALSE;
+ hash_rec = hash_lookup(files, fname);
+ if (hash_rec == NULL) {
+ index_set_corrupted(index, "Unexpectedly lost file "
+ "%s from hash", fname);
+ return FALSE;
+ }
- if (!index->expunge(index, rec, seq, TRUE))
- return FALSE;
- continue;
- }
+ if (uid_rec.uid != 0 &&
+ maildir_cmp(fname, uid_rec.filename) != 0) {
+ index_set_corrupted(index, "Filename mismatch for UID "
+ "%u: %s vs %s", rec->uid, fname,
+ uid_rec.filename);
+ return FALSE;
}
- /* changed - update */
- fname_changed = strcmp(value, fname) != 0;
- if (fname_changed) {
- if (!maildir_index_sync_file(index, rec, seq, value,
- path, fname_changed))
+ switch (hash_rec->action) {
+ case MAILDIR_FILE_ACTION_EXPUNGE:
+ if (!index->expunge(index, rec, seq, TRUE))
+ return FALSE;
+ seq--;
+ break;
+ case MAILDIR_FILE_ACTION_UPDATE_FLAGS:
+ if (!maildir_update_flags(index, rec, seq, fname))
+ return FALSE;
+ break;
+ case MAILDIR_FILE_ACTION_UPDATE_CONTENT:
+ if (!index->expunge(index, rec, seq, TRUE))
return FALSE;
+ seq--;
+ hash_rec->action = MAILDIR_FILE_ACTION_NEW;
+ new_count++;
+ break;
+ case MAILDIR_FILE_ACTION_NONE:
+ break;
+ default:
+ i_unreached();
}
- seq++;
+ if (uid_rec.uid != 0) {
+ if (maildir_uidlist_next(uidlist, &uid_rec) < 0)
+ return FALSE;
+ }
+ rec = index->next(index, rec);
}
- if (seq-1 != index->header->messages_count) {
+ if (seq != index->header->messages_count) {
index_set_corrupted(index, "Wrong messages_count in header "
- "(%u != %u)", seq-1,
+ "(%u != %u)", seq,
index->header->messages_count);
return FALSE;
}
- return TRUE;
-}
+ /* if there's mails with UIDs in uidlist, write them */
+ last_uid = 0;
+ while (uid_rec.uid != 0) {
+ if (!hash_lookup_full(files, uid_rec.filename,
+ &orig_key, &orig_value)) {
+ /* expunged */
+ if (uidlist != NULL)
+ uidlist->rewrite = TRUE;
+ } else {
+ hash_rec = orig_value;
+ i_assert(hash_rec->action == MAILDIR_FILE_ACTION_NEW);
-struct hash_append_context {
- struct mail_index *index;
- const char *dir;
- int failed;
-};
+ /* make sure we set the same UID for it. */
+ i_assert(index->header->next_uid <= uid_rec.uid);
+ index->header->next_uid = uid_rec.uid;
-static void maildir_index_hash_append_file(void *key __attr_unused__,
- void *value, void *context)
-{
- struct hash_append_context *ctx = context;
+ hash_rec->action = MAILDIR_FILE_ACTION_NONE;
+ new_count--;
- t_push();
- if (!maildir_index_append_file(ctx->index, ctx->dir, value)) {
- ctx->failed = TRUE;
- hash_foreach_stop();
+ if (!maildir_index_append_file(index, dir, orig_key))
+ return FALSE;
+ }
+
+ if (maildir_uidlist_next(uidlist, &uid_rec) < 0)
+ return FALSE;
}
- t_pop();
-}
-static int maildir_index_append_files(struct mail_index *index, const char *dir,
- struct hash_table *files)
-{
- struct hash_append_context ctx;
+ if (new_count == 0) {
+ /* all done */
+ return TRUE;
+ }
- ctx.failed = FALSE;
- ctx.index = index;
- ctx.dir = dir;
- hash_foreach(files, maildir_index_hash_append_file, &ctx);
+ if (uidlist != NULL)
+ uidlist->rewrite = TRUE;
- return !ctx.failed;
+ /* then there's the completely new mails. sort them by the filename
+ so we should get them to same order as they were created. */
+ buf = buffer_create_static_hard(pool, new_count * sizeof(const char *));
+ hash_foreach(files, uidlist_hash_get_filenames, buf);
+ i_assert(buffer_get_used_size(buf) / sizeof(const char *) <= new_count);
+
+ new_files = buffer_get_modifyable_data(buf, NULL);
+ qsort(new_files, new_count, sizeof(const char *),
+ (int (*)(const void *, const void *)) strcmp);
+
+ /* and finally write */
+ for (i = 0; i < new_count; i++) {
+ if (!maildir_index_append_file(index, dir, new_files[i]))
+ return FALSE;
+ }
+
+ return TRUE;
}
-static int maildir_index_sync_dir(struct mail_index *index, const char *dir)
+static int maildir_index_sync_dir(struct mail_index *index, const char *dir,
+ struct maildir_uidlist *uidlist)
{
pool_t pool;
struct hash_table *files;
+ struct mail_index_record *rec;
+ struct maildir_hash_rec *hash_rec;
DIR *dirp;
struct dirent *d;
- char *key, *value, *p;
- unsigned int count;
+ const char *fname;
+ void *orig_key, *orig_value;
+ unsigned int new_count;
int failed, check_content_changes;
i_assert(dir != NULL);
-
- /* get exclusive lock always, this way the index file's timestamp
- is updated even if there's no changes, which is useful to make
- sure the cur/ directory isn't scanned all the time when it's
- timestamp has changed but hasn't had any other changes. */
- if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
- return FALSE;
+ i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
if (index->header->messages_count >= INT_MAX/32) {
index_set_corrupted(index, "Header says %u messages",
@@ -200,131 +311,260 @@
return FALSE;
}
- /* we need to find out the new and the deleted files. do this by
- first building a hash of what files really exist, then go through
- the index and after updated/removed the index, remove the file
- from hash, so finally the hash should contain only the new
- files which will be added then. */
+ /* read current messages in index into hash */
+ pool = pool_alloconly_create("maildir sync", 16384);
+ files = hash_create(default_pool, pool, index->header->messages_count*2,
+ maildir_hash, maildir_cmp);
+
+ rec = index->lookup(index, 1);
+ while (rec != NULL) {
+ fname = maildir_get_location(index, rec);
+ if (fname == NULL) {
+ hash_destroy(files);
+ return FALSE;
+ }
+ hash_rec = p_new(pool, struct maildir_hash_rec, 1);
+ hash_rec->rec = rec;
+ hash_rec->action = MAILDIR_FILE_ACTION_EXPUNGE;
+ hash_insert(files, (void *) fname, hash_rec);
+
+ rec = index->next(index, rec);
+ }
+
+ /* Do we want to check changes in file contents? This slows down
+ things as we need to do extra stat() for all files. */
+ check_content_changes = getenv("MAILDIR_CHECK_CONTENT_CHANGES") != NULL;
+
dirp = opendir(dir);
if (dirp == NULL)
return index_file_set_syscall_error(index, dir, "opendir()");
- count = index->header->messages_count + 16;
- pool = pool_alloconly_create("Maildir sync", count*30);
- files = hash_create(default_pool, pool, index->header->messages_count*2,
- str_hash, (hash_cmp_callback_t *)strcmp);
-
+ new_count = 0; failed = FALSE;
while ((d = readdir(dirp)) != NULL) {
if (d->d_name[0] == '.')
continue;
- /* hash key is the file name without the ":flags" part */
- p = strrchr(d->d_name, ':');
- if (p == d->d_name)
- continue;
+ if (!hash_lookup_full(files, d->d_name,
+ &orig_key, &orig_value)) {
+ hash_rec = p_new(pool, struct maildir_hash_rec, 1);
+ } else {
+ hash_rec = orig_value;
+ if (hash_rec->action != MAILDIR_FILE_ACTION_EXPUNGE) {
+ /* FIXME: duplicate */
+ continue;
+ }
+ }
- value = p_strdup(pool, d->d_name);
- key = p == NULL ? value : p_strdup_until(pool, d->d_name, p);
- hash_insert(files, key, value);
+ if (hash_rec->rec == NULL) {
+ /* new message */
+ new_count++;
+ hash_rec->action = MAILDIR_FILE_ACTION_NEW;
+ hash_insert(files, p_strdup(pool, d->d_name), hash_rec);
+ } else if (check_content_changes &&
+ is_file_content_changed(index, rec,
+ dir, d->d_name)) {
+ /* file content changed, treat it as new message */
+ hash_rec->action = MAILDIR_FILE_ACTION_UPDATE_CONTENT;
+
+ /* make sure filename is not invalidated by expunge
+ later. the file name may have changed also. */
+ hash_insert(files, p_strdup(pool, d->d_name), hash_rec);
+ } else if (strcmp(orig_key, d->d_name) != 0) {
+ /* update filename now, flags later */
+ hash_rec->action = MAILDIR_FILE_ACTION_UPDATE_FLAGS;
+ if (!maildir_update_filename(index, hash_rec->rec,
+ d->d_name)) {
+ failed = TRUE;
+ break;
+ }
+ } else {
+ hash_rec->action = MAILDIR_FILE_ACTION_NONE;
+ }
}
if (closedir(dirp) < 0)
index_file_set_syscall_error(index, dir, "closedir()");
- /* Do we want to check changes in file contents? This slows down
- things as we need to do extra stat() for all files. */
- check_content_changes = getenv("MAILDIR_CHECK_CONTENT_CHANGES") != NULL;
-
- /* now walk through the index syncing and expunging existing mails */
- failed = !maildir_index_sync_files(index, dir, files,
- check_content_changes);
-
if (!failed) {
- /* then add the new mails */
- failed = !maildir_index_append_files(index, dir, files);
+ failed = !maildir_sync_uidlist(index, dir, uidlist,
+ files, pool, new_count);
}
-
hash_destroy(files);
pool_unref(pool);
return !failed;
}
-int maildir_index_sync(struct mail_index *index,
- enum mail_lock_type data_lock_type __attr_unused__,
- int *changes)
+static int maildir_new_scan_first_file(struct mail_index *index,
+ const char *dir, DIR **dirp,
+ struct dirent **d)
{
- struct stat sti, std;
+ *dirp = opendir(dir);
+ if (*dirp == NULL)
+ return index_file_set_syscall_error(index, dir, "opendir()");
+
+ /* find first file */
+ while ((*d = readdir(*dirp)) != NULL) {
+ if ((*d)->d_name[0] != '.')
+ break;
+ }
+
+ if (*d == NULL) {
+ if (closedir(*dirp) < 0)
+ index_file_set_syscall_error(index, dir, "closedir()");
+ *dirp = NULL;
+ }
+
+ return TRUE;
+}
+
+static int maildir_index_lock_and_sync(struct mail_index *index, int *changes,
+ DIR *new_dirp, struct dirent *new_dent,
+ struct maildir_uidlist **uidlist_r)
+{
+ struct stat st, std;
struct utimbuf ut;
- const char *cur_dir, *new_dir;
+ struct maildir_uidlist *uidlist;
+ const char *uidlist_path, *cur_dir, *new_dir;
time_t index_mtime;
+ int cur_changed;
- i_assert(index->lock_type != MAIL_LOCK_SHARED);
-
- if (changes != NULL)
- *changes = FALSE;
+ *uidlist_r = uidlist = NULL;
if (index->fd == -1) {
/* anon-mmaped */
index_mtime = index->file_sync_stamp;
} else {
- if (fstat(index->fd, &sti) < 0)
+ if (fstat(index->fd, &st) < 0)
return index_set_syscall_error(index, "fstat()");
- index_mtime = sti.st_mtime;
+ index_mtime = st.st_mtime;
}
- /* cur/ and new/ directories can have new mail - sync the cur/ first
- so it'll be a bit bit faster since we haven't yet added the new
- mail. */
cur_dir = t_strconcat(index->mailbox_path, "/cur", NULL);
if (stat(cur_dir, &std) < 0)
return index_file_set_syscall_error(index, cur_dir, "stat()");
- if (std.st_mtime != index_mtime) {
- if (changes != NULL) *changes = TRUE;
- if (!maildir_index_sync_dir(index, cur_dir))
- return FALSE;
+ uidlist_path = t_strconcat(index->mailbox_path,
+ "/" MAILDIR_UIDLIST_NAME, NULL);
+ if (stat(uidlist_path, &st) < 0) {
+ if (errno != ENOENT) {
+ return index_file_set_syscall_error(index, uidlist_path,
+ "stat()");
+ }
+
+ memset(&st, 0, sizeof(st));
+ cur_changed = TRUE;
+ } else {
+ /* FIXME: save device and inode into index header, so we don't
+ have to read it every time mailbox is opened */
+ cur_changed = index_mtime != std.st_mtime ||
+ st.st_ino != index->uidlist_ino ||
+ !CMP_DEV_T(st.st_dev, index->uidlist_dev);
}
- /* move mail from new/ to cur/ */
- new_dir = t_strconcat(index->mailbox_path, "/new", NULL);
- if (stat(new_dir, &std) < 0)
- return index_file_set_syscall_error(index, new_dir, "stat()");
+ if (new_dirp != NULL || cur_changed) {
+ if (maildir_uidlist_try_lock(index) < 0)
+ return FALSE;
- if (std.st_mtime != index_mtime) {
- if (changes != NULL) *changes = TRUE;
+ /* we may or may not have succeeded. if we didn't,
+ just continue by syncing with existing uidlist file */
- if (!maildir_index_build_dir(index, new_dir, cur_dir))
+ if (!cur_changed && !INDEX_IS_UIDLIST_LOCKED(index)) {
+ /* just new mails in new/ dir, we can't sync them
+ if we can't get the lock. */
+ return TRUE;
+ }
+
+ if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
return FALSE;
- /* set cur/ and new/ directory's timestamp into past to
- make sure if someone adds new mail it the new/ dir's
- timestamp isn't set to same as cur/ directory's. */
+ *uidlist_r = uidlist = maildir_uidlist_open(index);
+ if (uidlist != NULL &&
+ uidlist->uid_validity != index->header->uid_validity) {
+ /* uidvalidity changed */
+ if (!index->rebuilding) {
+ index_set_corrupted(index,
+ "UIDVALIDITY changed in uidlist");
+ return FALSE;
+ }
+
+ index->header->uid_validity = uidlist->uid_validity;
+ }
+
+ if (uidlist != NULL &&
+ index->header->next_uid > uidlist->next_uid) {
+ index_set_corrupted(index, "index.next_uid (%u) > "
+ "uidlist.next_uid (%u)",
+ index->header->next_uid,
+ uidlist->next_uid);
+ return FALSE;
+ }
+
+ if (changes != NULL)
+ *changes = TRUE;
+ }
+
+ /* move mail from new/ to cur/ */
+ if (new_dirp != NULL && INDEX_IS_UIDLIST_LOCKED(index)) {
+ new_dir = t_strconcat(index->mailbox_path, "/new", NULL);
+ if (!maildir_index_build_dir(index, new_dir, cur_dir,
+ new_dirp, new_dent))
+ return FALSE;
+
+ if (uidlist != NULL)
+ uidlist->rewrite = TRUE;
+
+ /* set cur/ directory's timestamp into past to make sure we
+ notice if new mail is moved there */
ut.actime = ut.modtime = ioloop_time-60;
if (utime(cur_dir, &ut) < 0) {
- return index_file_set_syscall_error(index, cur_dir,
- "utime()");
+ index_file_set_syscall_error(index, cur_dir, "utime()");
+ return FALSE;
}
- if (utime(new_dir, &ut) < 0) {
- return index_file_set_syscall_error(index, new_dir,
- "utime()");
+
+ /* We have to always scan the cur/ directory to make
+ sure we don't miss any mails some other non-Dovecot
+ client may have moved there. FIXME: make it
+ optional, it's unnecessary with Dovecot-only setup */
+ cur_changed = TRUE;
+
+ /* set the cur/ directory's timestamp */
+ std.st_mtime = ut.modtime;
+ }
+
+ if (cur_changed) {
+ if (!maildir_index_sync_dir(index, cur_dir, uidlist))
+ return FALSE;
+ }
+
+ if (uidlist != NULL && uidlist->next_uid > index->header->next_uid)
+ index->header->next_uid = uidlist->next_uid;
+
+ if ((new_dirp != NULL || cur_changed) &&
+ (uidlist == NULL || uidlist->rewrite)) {
+ if (!INDEX_IS_UIDLIST_LOCKED(index)) {
+ /* there's more new mails, but we need .lock file to
+ be able to sync them. */
+ return TRUE;
}
- /* it's possible that new mail came in just after we
- scanned the directory. scan the directory again, this will
- update the directory's timestamps so at next sync we'll
- always check the new/ dir once more, but at least we can be
- sure that no mail got lost. */
- if (!maildir_index_build_dir(index, new_dir, cur_dir))
+ if (fstat(index->maildir_lock_fd, &st) < 0) {
+ return index_file_set_syscall_error(index, uidlist_path,
+ "fstat()");
+ }
+
+ if (!maildir_uidlist_rewrite(index))
return FALSE;
}
+ /* uidlist file synced */
+ index->uidlist_ino = st.st_ino;
+ index->uidlist_dev = st.st_dev;
+
/* update sync stamp */
- if (stat(cur_dir, &std) < 0)
- return index_file_set_syscall_error(index, cur_dir, "stat()");
index->file_sync_stamp = std.st_mtime;
- if (index->fd != -1 && index->lock_type == MAIL_LOCK_UNLOCK) {
- /* no changes, we need to update index's timestamp
+ if (index->lock_type == MAIL_LOCK_UNLOCK && !index->anon_mmap) {
+ /* no changes to index, we need to update it's timestamp
ourself to get it changed */
ut.actime = ioloop_time;
ut.modtime = index->file_sync_stamp;
@@ -333,4 +573,40 @@
}
return TRUE;
+}
+
+int maildir_index_sync(struct mail_index *index,
+ enum mail_lock_type data_lock_type __attr_unused__,
+ int *changes)
+{
+ struct maildir_uidlist *uidlist;
+ DIR *new_dirp;
+ struct dirent *new_dent;
+ const char *new_dir;
+ int ret;
+
+ i_assert(index->lock_type != MAIL_LOCK_SHARED);
+
+ if (changes != NULL)
+ *changes = FALSE;
+
+ new_dir = t_strconcat(index->mailbox_path, "/new", NULL);
+ if (!maildir_new_scan_first_file(index, new_dir, &new_dirp, &new_dent))
+ return FALSE;
+
+ ret = maildir_index_lock_and_sync(index, changes, new_dirp, new_dent,
+ &uidlist);
+
+ if (uidlist != NULL)
+ maildir_uidlist_close(uidlist);
+
+ if (new_dirp != NULL) {
+ if (closedir(new_dirp) < 0) {
+ index_file_set_syscall_error(index, new_dir,
+ "closedir()");
+ }
+ }
+
+ maildir_uidlist_unlock(index);
+ return ret;
}
More information about the dovecot-cvs
mailing list