[dovecot-cvs] dovecot/src/lib-storage/index/maildir maildir-copy.c,
1.42, 1.43 maildir-save.c, 1.66, 1.67 maildir-storage.c, 1.111,
1.112 maildir-storage.h, 1.45, 1.46 maildir-transaction.c,
1.10, 1.11
tss-movial at dovecot.org
tss-movial at dovecot.org
Mon Mar 6 20:16:15 EET 2006
Update of /var/lib/cvs/dovecot/src/lib-storage/index/maildir
In directory talvi:/tmp/cvs-serv1965
Modified Files:
maildir-copy.c maildir-save.c maildir-storage.c
maildir-storage.h maildir-transaction.c
Log Message:
Merged save-copying and hardlink-copying code so that hardlink-copying updates indexes immediately.
Index: maildir-copy.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/maildir/maildir-copy.c,v
retrieving revision 1.42
retrieving revision 1.43
diff -u -d -r1.42 -r1.43
--- maildir-copy.c 13 Jan 2006 20:26:35 -0000 1.42
+++ maildir-copy.c 6 Mar 2006 18:16:12 -0000 1.43
@@ -12,25 +12,9 @@
#include <stdlib.h>
#include <unistd.h>
-struct maildir_copy_context {
- struct maildir_mailbox *mbox;
- bool hardlink;
-
- struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
- struct maildir_keywords_sync_ctx *keywords_sync_ctx;
-
- pool_t pool;
- struct rollback *rollbacks;
-};
-
struct hardlink_ctx {
const char *dest_path;
- bool found;
-};
-
-struct rollback {
- struct rollback *next;
- const char *fname;
+ bool success;
};
static int do_hardlink(struct maildir_mailbox *mbox, const char *path,
@@ -56,118 +40,88 @@
return -1;
}
- ctx->found = TRUE;
+ ctx->success = TRUE;
return 1;
}
static int
-maildir_copy_hardlink(struct mail *mail,
+maildir_copy_hardlink(struct maildir_transaction_context *t, struct mail *mail,
enum mail_flags flags, struct mail_keywords *keywords,
- struct maildir_copy_context *ctx)
+ struct mail *dest_mail)
{
- struct index_mail *imail = (struct index_mail *)mail;
- struct maildir_mailbox *dest_mbox = ctx->mbox;
+ struct maildir_mailbox *dest_mbox =
+ (struct maildir_mailbox *)t->ictx.ibox;
struct maildir_mailbox *src_mbox =
- (struct maildir_mailbox *)imail->ibox;
+ (struct maildir_mailbox *)mail->box;
+ struct maildir_save_context *ctx;
struct hardlink_ctx do_ctx;
- struct rollback *rb;
const char *dest_fname;
- unsigned int keywords_count;
- array_t ARRAY_DEFINE(keywords_arr, unsigned int);
-
- dest_fname = maildir_generate_tmp_filename(&ioloop_timeval);
-
- keywords_count = keywords == NULL ? 0 : keywords->count;
- if (keywords_count > 0) {
- ARRAY_CREATE(&keywords_arr, pool_datastack_create(),
- unsigned int, keywords->count);
- array_append(&keywords_arr, keywords->idx, keywords->count);
+ uint32_t seq;
- if (ctx->keywords_sync_ctx == NULL) {
- /* uidlist must be locked while accessing
- keywords files */
- if (maildir_uidlist_sync_init(dest_mbox->uidlist, TRUE,
- &ctx->uidlist_sync_ctx) <= 0) {
- /* error or timeout */
- return -1;
- }
+ i_assert((t->ictx.flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);
- ctx->keywords_sync_ctx =
- maildir_keywords_sync_init(dest_mbox->keywords,
- dest_mbox->ibox.index);
- }
- }
+ if (t->save_ctx == NULL)
+ t->save_ctx = maildir_save_transaction_init(t);
+ ctx = t->save_ctx;
+ /* don't allow caller to specify recent flag */
flags &= ~MAIL_RECENT;
if (dest_mbox->ibox.keep_recent)
flags |= MAIL_RECENT;
- dest_fname = maildir_filename_set_flags(ctx->keywords_sync_ctx,
- dest_fname, flags,
- keywords_count != 0 ?
- &keywords_arr : NULL);
+ memset(&do_ctx, 0, sizeof(do_ctx));
- if (keywords_count == 0 && flags == MAIL_RECENT)
- dest_fname = t_strconcat("new/", dest_fname, NULL);
- else
- dest_fname = t_strconcat("cur/", dest_fname, NULL);
+ /* the generated filename is _always_ unique, so we don't bother
+ trying to check if it already exists */
+ dest_fname = maildir_generate_tmp_filename(&ioloop_timeval);
+ if (keywords == NULL || keywords->count == 0) {
+ /* no keywords, hardlink directly to destination */
+ if (flags == MAIL_RECENT) {
+ do_ctx.dest_path =
+ t_strconcat(dest_mbox->path, "/new/",
+ dest_fname, NULL);
+ } else {
+ const char *fname;
- memset(&do_ctx, 0, sizeof(do_ctx));
- do_ctx.dest_path =
- t_strconcat(dest_mbox->path, "/", dest_fname, NULL);
+ fname = maildir_filename_set_flags(NULL, dest_fname,
+ flags, NULL);
- if (maildir_file_do(src_mbox, imail->mail.mail.uid,
- do_hardlink, &do_ctx) < 0)
+ do_ctx.dest_path =
+ t_strconcat(dest_mbox->path, "/cur/",
+ fname, NULL);
+ }
+ } else {
+ /* keywords, hardlink to tmp/ with basename and later when we
+ have uidlist locked, move it to new/cur. */
+ do_ctx.dest_path =
+ t_strconcat(dest_mbox->path, "/tmp/", dest_fname, NULL);
+ }
+ if (maildir_file_do(src_mbox, mail->uid, do_hardlink, &do_ctx) < 0)
return -1;
- if (!do_ctx.found)
+ if (!do_ctx.success) {
+ /* couldn't copy with hardlinking, fallback to copying */
return 0;
+ }
- rb = p_new(ctx->pool, struct rollback, 1);
- rb->fname = p_strdup(ctx->pool, dest_fname);
-
- rb->next = ctx->rollbacks;
- ctx->rollbacks = rb;
- return 1;
-}
-
-static struct maildir_copy_context *
-maildir_copy_init(struct maildir_mailbox *mbox)
-{
- struct maildir_copy_context *ctx;
- pool_t pool;
-
- pool = pool_alloconly_create("maildir_copy_context", 2048);
-
- ctx = p_new(pool, struct maildir_copy_context, 1);
- ctx->pool = pool;
- ctx->hardlink = getenv("MAILDIR_COPY_WITH_HARDLINKS") != NULL;
- ctx->mbox = mbox;
- return ctx;
-}
-
-int maildir_transaction_copy_commit(struct maildir_copy_context *ctx)
-{
- if (ctx->keywords_sync_ctx != NULL) {
- maildir_keywords_sync_deinit(ctx->keywords_sync_ctx);
- maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
+ if (keywords == NULL || keywords->count == 0) {
+ /* hardlinked to destination, set hardlinked-flag */
+ seq = maildir_save_add(t, dest_fname,
+ flags | MAILDIR_SAVE_FLAG_HARDLINK, NULL,
+ dest_mail != NULL);
+ } else {
+ /* hardlinked to tmp/, treat as normal copied mail */
+ seq = maildir_save_add(t, dest_fname, flags, keywords,
+ dest_mail != NULL);
}
- pool_unref(ctx->pool);
- return 0;
-}
-void maildir_transaction_copy_rollback(struct maildir_copy_context *ctx)
-{
- struct rollback *rb;
+ if (dest_mail != NULL) {
+ i_assert(seq != 0);
- for (rb = ctx->rollbacks; rb != NULL; rb = rb->next) {
- t_push();
- (void)unlink(t_strconcat(ctx->mbox->path, "/",
- rb->fname, NULL));
- t_pop();
+ if (mail_set_seq(dest_mail, seq) < 0)
+ return -1;
}
-
- pool_unref(ctx->pool);
+ return 1;
}
int maildir_copy(struct mailbox_transaction_context *_t, struct mail *mail,
@@ -177,18 +131,13 @@
struct maildir_transaction_context *t =
(struct maildir_transaction_context *)_t;
struct maildir_mailbox *mbox = (struct maildir_mailbox *)t->ictx.ibox;
- struct maildir_copy_context *ctx;
int ret;
- if (t->copy_ctx == NULL)
- t->copy_ctx = maildir_copy_init(mbox);
- ctx = t->copy_ctx;
-
- if (ctx->hardlink &&
- mail->box->storage == STORAGE(ctx->mbox->storage)) {
- // FIXME: handle dest_mail
+ if (mbox->storage->copy_with_hardlinks &&
+ mail->box->storage == mbox->ibox.box.storage) {
t_push();
- ret = maildir_copy_hardlink(mail, flags, keywords, ctx);
+ ret = maildir_copy_hardlink(t, mail, flags,
+ keywords, dest_mail);
t_pop();
if (ret > 0)
Index: maildir-save.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/maildir/maildir-save.c,v
retrieving revision 1.66
retrieving revision 1.67
diff -u -d -r1.66 -r1.67
--- maildir-save.c 26 Feb 2006 10:05:18 -0000 1.66
+++ maildir-save.c 6 Mar 2006 18:16:12 -0000 1.67
@@ -90,7 +90,7 @@
return ret;
}
-static struct maildir_save_context *
+struct maildir_save_context *
maildir_save_transaction_init(struct maildir_transaction_context *t)
{
struct maildir_mailbox *mbox = (struct maildir_mailbox *)t->ictx.ibox;
@@ -113,9 +113,62 @@
ctx->keywords_buffer = buffer_create_const_data(pool, NULL, 0);
array_create_from_buffer(&ctx->keywords_array, ctx->keywords_buffer,
sizeof(unsigned int));
+ ctx->finished = TRUE;
return ctx;
}
+uint32_t maildir_save_add(struct maildir_transaction_context *t,
+ const char *base_fname, enum mail_flags flags,
+ struct mail_keywords *keywords, bool want_mail)
+{
+ struct maildir_save_context *ctx = t->save_ctx;
+ struct maildir_mailbox *mbox = (struct maildir_mailbox *)t->ictx.ibox;
+ struct maildir_filename *mf;
+
+ /* now, we want to be able to rollback the whole append session,
+ so we'll just store the name of this temp file and move it later
+ into new/ or cur/. */
+ /* @UNSAFE */
+ mf = p_malloc(ctx->pool, sizeof(*mf) +
+ sizeof(unsigned int) * (keywords == NULL ? 0 :
+ keywords->count));
+ mf->next = ctx->files;
+ mf->basename = p_strdup(ctx->pool, base_fname);
+ mf->flags = flags;
+ ctx->files = mf;
+
+ if (keywords != NULL) {
+ i_assert(sizeof(keywords->idx[0]) == sizeof(unsigned int));
+
+ /* @UNSAFE */
+ mf->keywords_count = keywords->count;
+ memcpy(mf + 1, keywords->idx,
+ sizeof(unsigned int) * keywords->count);
+ }
+
+ if (!ctx->synced && want_mail) {
+ if (maildir_storage_sync_force(mbox) < 0)
+ ctx->failed = TRUE;
+ else
+ ctx->synced = TRUE;
+ }
+
+ if (ctx->synced) {
+ /* insert into index */
+ mail_index_append(ctx->trans, 0, &ctx->seq);
+ mail_index_update_flags(ctx->trans, ctx->seq,
+ MODIFY_REPLACE, flags);
+ if (keywords != NULL) {
+ mail_index_update_keywords(ctx->trans, ctx->seq,
+ MODIFY_REPLACE, keywords);
+ }
+ } else {
+ ctx->seq = 0;
+ }
+
+ return ctx->seq;
+}
+
int maildir_save_init(struct mailbox_transaction_context *_t,
enum mail_flags flags, struct mail_keywords *keywords,
time_t received_date, int timezone_offset __attr_unused__,
@@ -127,7 +180,6 @@
(struct maildir_transaction_context *)_t;
struct maildir_save_context *ctx;
struct maildir_mailbox *mbox = (struct maildir_mailbox *)t->ictx.ibox;
- struct maildir_filename *mf;
struct ostream *output;
const char *fname, *path;
@@ -166,46 +218,9 @@
if (mbox->ibox.keep_recent)
flags |= MAIL_RECENT;
- /* now, we want to be able to rollback the whole append session,
- so we'll just store the name of this temp file and move it later
- into new/ or cur/. */
- /* @UNSAFE */
- mf = p_malloc(ctx->pool, sizeof(*mf) +
- sizeof(unsigned int) * (keywords == NULL ? 0 :
- keywords->count));
- mf->next = ctx->files;
- mf->basename = p_strdup(ctx->pool, fname);
- mf->flags = flags;
- ctx->files = mf;
-
- if (keywords != NULL) {
- i_assert(sizeof(keywords->idx[0]) == sizeof(unsigned int));
-
- /* @UNSAFE */
- mf->keywords_count = keywords->count;
- memcpy(mf + 1, keywords->idx,
- sizeof(unsigned int) * keywords->count);
- }
-
- if (!ctx->synced && want_mail) {
- if (maildir_storage_sync_force(mbox) < 0)
- ctx->failed = TRUE;
- else
- ctx->synced = TRUE;
- }
+ maildir_save_add(t, fname, flags, keywords, want_mail);
- if (ctx->synced) {
- /* insert into index */
- mail_index_append(ctx->trans, 0, &ctx->seq);
- mail_index_update_flags(ctx->trans, ctx->seq,
- MODIFY_REPLACE, flags);
- if (keywords != NULL) {
- mail_index_update_keywords(ctx->trans, ctx->seq,
- MODIFY_REPLACE, keywords);
- }
- }
t_pop();
-
*ctx_r = &ctx->ctx;
return ctx->failed ? -1 : 0;
}
@@ -293,6 +308,7 @@
t_pop();
return -1;
}
+ t_pop();
if (dest_mail != NULL) {
i_assert(ctx->seq != 0);
@@ -301,7 +317,6 @@
return -1;
}
- t_pop();
return 0;
}
@@ -317,25 +332,34 @@
maildir_get_updated_filename(struct maildir_save_context *ctx,
struct maildir_filename *mf)
{
- if (mf->flags == MAIL_RECENT && mf->keywords_count == 0)
- return NULL;
+ if (mf->keywords_count == 0) {
+ if ((mf->flags & MAIL_FLAGS_MASK) == MAIL_RECENT)
+ return NULL;
+ return maildir_filename_set_flags(NULL, mf->basename,
+ mf->flags & MAIL_FLAGS_MASK,
+ NULL);
+ }
buffer_update_const_data(ctx->keywords_buffer, mf + 1,
mf->keywords_count * sizeof(unsigned int));
return maildir_filename_set_flags(
maildir_sync_get_keywords_sync_ctx(ctx->sync_ctx),
- mf->basename, mf->flags, &ctx->keywords_array);
+ mf->basename, mf->flags & MAIL_FLAGS_MASK,
+ &ctx->keywords_array);
}
static void
-maildir_save_commit_abort(struct maildir_save_context *ctx,
- struct maildir_filename *pos)
+maildir_transaction_unlink_copied_files(struct maildir_save_context *ctx,
+ struct maildir_filename *pos)
{
struct maildir_filename *mf;
const char *path, *dest;
/* try to unlink the mails already moved */
for (mf = ctx->files; mf != pos; mf = mf->next) {
+ if ((mf->flags & MAILDIR_SAVE_FLAG_DELETED) != 0)
+ continue;
+
t_push();
dest = maildir_get_updated_filename(ctx, mf);
if (dest != NULL)
@@ -345,10 +369,9 @@
ctx->newdir, mf->basename);
}
(void)unlink(path);
+ t_pop();
}
ctx->files = pos;
-
- maildir_transaction_save_rollback(ctx);
}
int maildir_transaction_save_commit_pre(struct maildir_save_context *ctx)
@@ -365,7 +388,7 @@
/* Start syncing so that keywords_sync_ctx gets set.. */
ctx->sync_ctx = maildir_sync_index_begin(ctx->mbox);
if (ctx->sync_ctx == NULL) {
- maildir_save_commit_abort(ctx, ctx->files);
+ maildir_transaction_save_rollback(ctx);
return -1;
}
@@ -373,7 +396,7 @@
&ctx->uidlist_sync_ctx) <= 0) {
/* error or timeout - our transaction is broken */
maildir_sync_index_abort(ctx->sync_ctx);
- maildir_save_commit_abort(ctx, ctx->files);
+ maildir_transaction_save_rollback(ctx);
return -1;
}
@@ -385,28 +408,41 @@
flags = MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
MAILDIR_UIDLIST_REC_FLAG_RECENT;
- /* move them into new/ */
+ /* move them into new/ and/or cur/ */
ret = 0;
- for (mf = ctx->files; mf != NULL; mf = mf->next) {
+ for (mf = ctx->files; mf != NULL && ret == 0; mf = mf->next) {
t_push();
dest = maildir_get_updated_filename(ctx, mf);
fname = dest != NULL ? dest : mf->basename;
- if (maildir_file_move(ctx, mf->basename, dest) < 0 ||
- maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
- fname, flags) < 0) {
- maildir_save_commit_abort(ctx, mf);
- t_pop();
- ret = -1;
- break;
+ /* if hardlink-flag is set, the file is already in destination.
+ if the hardlinked mail contained keywords, it was linked
+ into tmp/ and it doesn't have the hardlink-flag set, so it's
+ treated as any other saved mail. */
+ if ((mf->flags & MAILDIR_SAVE_FLAG_HARDLINK) == 0)
+ ret = maildir_file_move(ctx, mf->basename, dest);
+ if (ret == 0) {
+ ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
+ fname, flags);
}
t_pop();
}
+ if (ret < 0) {
+ /* unlink the files we just moved in an attempt to rollback
+ the transaction. uidlist is still locked, so at least other
+ Dovecot instances haven't yet seen the files. */
+ maildir_transaction_unlink_copied_files(ctx, mf);
+ }
+
if (maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx) < 0)
ret = -1;
ctx->uidlist_sync_ctx = NULL;
+ if (ret < 0) {
+ /* returning failure finishes the save_context */
+ maildir_transaction_save_rollback(ctx);
+ }
return ret;
}
@@ -429,6 +465,7 @@
struct maildir_filename *mf;
string_t *str;
size_t dir_len;
+ bool hardlinks = FALSE;
i_assert(ctx->output == NULL);
@@ -444,10 +481,19 @@
/* clean up the temp files */
for (mf = ctx->files; mf != NULL; mf = mf->next) {
- str_truncate(str, dir_len);
- str_append(str, mf->basename);
- (void)unlink(str_c(str));
+ if ((mf->flags & MAILDIR_SAVE_FLAG_HARDLINK) == 0) {
+ mf->flags |= MAILDIR_SAVE_FLAG_DELETED;
+ str_truncate(str, dir_len);
+ str_append(str, mf->basename);
+ (void)unlink(str_c(str));
+ } else {
+ hardlinks = TRUE;
+ }
}
+
+ if (hardlinks)
+ maildir_transaction_unlink_copied_files(ctx, NULL);
+
t_pop();
pool_unref(ctx->pool);
Index: maildir-storage.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/maildir/maildir-storage.c,v
retrieving revision 1.111
retrieving revision 1.112
diff -u -d -r1.111 -r1.112
--- maildir-storage.c 22 Feb 2006 14:52:14 -0000 1.111
+++ maildir-storage.c 6 Mar 2006 18:16:12 -0000 1.112
@@ -126,6 +126,8 @@
pool = pool_alloconly_create("storage", 512);
storage = p_new(pool, struct maildir_storage, 1);
storage->control_dir = p_strdup(pool, home_expand(control_dir));
+ storage->copy_with_hardlinks =
+ getenv("MAILDIR_COPY_WITH_HARDLINKS") != NULL;
istorage = INDEX_STORAGE(storage);
istorage->storage = maildir_storage;
Index: maildir-storage.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/maildir/maildir-storage.h,v
retrieving revision 1.45
retrieving revision 1.46
diff -u -d -r1.45 -r1.46
--- maildir-storage.h 22 Feb 2006 16:02:27 -0000 1.45
+++ maildir-storage.h 6 Mar 2006 18:16:12 -0000 1.46
@@ -31,6 +31,9 @@
calculating file's virtual size (added missing CRs). */
#define MAILDIR_EXTRA_VIRTUAL_SIZE "W"
+#define MAILDIR_SAVE_FLAG_HARDLINK 0x10000000
+#define MAILDIR_SAVE_FLAG_DELETED 0x20000000
+
#include "index-storage.h"
#define STORAGE(maildir_storage) \
@@ -47,6 +50,7 @@
struct index_storage storage;
const char *control_dir;
+ unsigned int copy_with_hardlinks:1;
};
struct maildir_mailbox {
@@ -70,7 +74,6 @@
struct maildir_transaction_context {
struct index_transaction_context ictx;
struct maildir_save_context *save_ctx;
- struct maildir_copy_context *copy_ctx;
};
extern struct mail_vfuncs maildir_mail_vfuncs;
@@ -121,6 +124,12 @@
int maildir_save_finish(struct mail_save_context *ctx, struct mail *dest_mail);
void maildir_save_cancel(struct mail_save_context *ctx);
+struct maildir_save_context *
+maildir_save_transaction_init(struct maildir_transaction_context *t);
+uint32_t maildir_save_add(struct maildir_transaction_context *t,
+ const char *base_fname, enum mail_flags flags,
+ struct mail_keywords *keywords, bool want_mail);
+
int maildir_transaction_save_commit_pre(struct maildir_save_context *ctx);
void maildir_transaction_save_commit_post(struct maildir_save_context *ctx);
void maildir_transaction_save_rollback(struct maildir_save_context *ctx);
Index: maildir-transaction.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/maildir/maildir-transaction.c,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- maildir-transaction.c 2 Dec 2005 10:03:41 -0000 1.10
+++ maildir-transaction.c 6 Mar 2006 18:16:12 -0000 1.11
@@ -30,10 +30,6 @@
ret = -1;
}
}
- if (t->copy_ctx != NULL) {
- if (maildir_transaction_copy_commit(t->copy_ctx) < 0)
- ret = -1;
- }
save_ctx = t->save_ctx;
@@ -56,7 +52,5 @@
if (t->save_ctx != NULL)
maildir_transaction_save_rollback(t->save_ctx);
- if (t->copy_ctx != NULL)
- maildir_transaction_copy_rollback(t->copy_ctx);
index_transaction_rollback(_t);
}
More information about the dovecot-cvs
mailing list