dovecot: When saving multiple mails, close some of the files tem...
dovecot at dovecot.org
dovecot at dovecot.org
Sun Dec 9 15:50:29 EET 2007
details: http://hg.dovecot.org/dovecot/rev/038467bffcbd
changeset: 6977:038467bffcbd
user: Timo Sirainen <tss at iki.fi>
date: Sun Dec 09 15:48:38 2007 +0200
description:
When saving multiple mails, close some of the files temporarily so we don't
use all available fds.
diffstat:
3 files changed, 83 insertions(+), 48 deletions(-)
src/lib-storage/index/dbox/dbox-file.c | 42 ++++++++++++---
src/lib-storage/index/dbox/dbox-file.h | 4 +
src/lib-storage/index/dbox/dbox-save.c | 85 +++++++++++++++++---------------
diffs (199 lines):
diff -r 7cedc391e6c5 -r 038467bffcbd src/lib-storage/index/dbox/dbox-file.c
--- a/src/lib-storage/index/dbox/dbox-file.c Sun Dec 09 15:47:03 2007 +0200
+++ b/src/lib-storage/index/dbox/dbox-file.c Sun Dec 09 15:48:38 2007 +0200
@@ -85,15 +85,7 @@ static void dbox_file_free(struct dbox_f
if (file->metadata_pool != NULL)
pool_unref(&file->metadata_pool);
- if (file->input != NULL)
- i_stream_unref(&file->input);
- if (file->output != NULL)
- o_stream_unref(&file->output);
- if (file->fd != -1) {
- if (close(file->fd) < 0)
- dbox_file_set_syscall_error(file, "close");
- file->fd = -1;
- }
+ dbox_file_close(file);
i_free(file->current_path);
i_free(file->fname);
i_free(file);
@@ -506,6 +498,38 @@ int dbox_file_open_or_create(struct dbox
return 1;
else
return dbox_file_open(file, read_header, deleted_r);
+}
+
+int dbox_file_open_if_needed(struct dbox_file *file)
+{
+ const char *path;
+ int ret;
+
+ if (file->fd != -1)
+ return 0;
+
+ T_FRAME(
+ ret = dbox_file_open_fd(file);
+ );
+ if (ret == 0) {
+ path = t_strdup_printf("%s/%s", file->mbox->path, file->fname);
+ mail_storage_set_critical(file->mbox->ibox.box.storage,
+ "open(%s) failed: %m", path);
+ }
+ return ret <= 0 ? -1 : 0;
+}
+
+void dbox_file_close(struct dbox_file *file)
+{
+ if (file->input != NULL)
+ i_stream_unref(&file->input);
+ if (file->output != NULL)
+ o_stream_unref(&file->output);
+ if (file->fd != -1) {
+ if (close(file->fd) < 0)
+ dbox_file_set_syscall_error(file, "close");
+ file->fd = -1;
+ }
}
const char *dbox_file_get_path(struct dbox_file *file)
diff -r 7cedc391e6c5 -r 038467bffcbd src/lib-storage/index/dbox/dbox-file.h
--- a/src/lib-storage/index/dbox/dbox-file.h Sun Dec 09 15:47:03 2007 +0200
+++ b/src/lib-storage/index/dbox/dbox-file.h Sun Dec 09 15:48:38 2007 +0200
@@ -153,6 +153,10 @@ int dbox_file_assign_id(struct dbox_file
deleted, deleted_r=TRUE and 1 is returned. */
int dbox_file_open_or_create(struct dbox_file *file, bool read_header,
bool *deleted_r);
+/* Open the file's fd if it's currently closed. Assumes that the file exists. */
+int dbox_file_open_if_needed(struct dbox_file *file);
+/* Close the file handle from the file, but don't free it. */
+void dbox_file_close(struct dbox_file *file);
/* Returns the current fulle path for an opened/created file. It's an error to
call this function for a non-opened file. */
diff -r 7cedc391e6c5 -r 038467bffcbd src/lib-storage/index/dbox/dbox-save.c
--- a/src/lib-storage/index/dbox/dbox-save.c Sun Dec 09 15:47:03 2007 +0200
+++ b/src/lib-storage/index/dbox/dbox-save.c Sun Dec 09 15:48:38 2007 +0200
@@ -18,7 +18,7 @@
struct dbox_save_mail {
struct dbox_file *file;
- uint32_t seq;
+ uint32_t seq, uid;
uint32_t append_offset;
uoff_t message_size;
};
@@ -250,6 +250,12 @@ int dbox_save_finish(struct mail_save_co
i_stream_unref(&ctx->input);
count = array_count(&ctx->mails);
+ if (count >= ctx->mbox->max_open_files) {
+ /* too many open files, close one of them */
+ save_mail = array_idx_modifiable(&ctx->mails, count -
+ ctx->mbox->max_open_files);
+ dbox_file_close(save_mail->file);
+ }
save_mail = array_idx_modifiable(&ctx->mails, count - 1);
if (ctx->failed) {
dbox_file_cancel_append(save_mail->file,
@@ -273,28 +279,21 @@ void dbox_save_cancel(struct mail_save_c
(void)dbox_save_finish(_ctx);
}
-static int
-dbox_save_mail_write_header(struct dbox_save_mail *mail, uint32_t uid)
+static int dbox_save_mail_write_header(struct dbox_save_mail *mail)
{
struct dbox_message_header dbox_msg_hdr;
- struct ostream *output = mail->file->output;
- uoff_t orig_offset;
- int ret = 0;
i_assert(mail->file->msg_header_size == sizeof(dbox_msg_hdr));
- mail->file->last_append_uid = uid;
- dbox_msg_header_fill(&dbox_msg_hdr, uid, mail->message_size);
-
- orig_offset = output->offset;
- o_stream_seek(output, mail->append_offset);
- if (o_stream_send(output, &dbox_msg_hdr, sizeof(dbox_msg_hdr)) < 0 ||
- o_stream_flush(output) < 0) {
+ mail->file->last_append_uid = mail->uid;
+ dbox_msg_header_fill(&dbox_msg_hdr, mail->uid, mail->message_size);
+
+ if (pwrite_full(mail->file->fd, &dbox_msg_hdr,
+ sizeof(dbox_msg_hdr), mail->append_offset) < 0) {
dbox_file_set_syscall_error(mail->file, "write");
- ret = -1;
- }
- o_stream_seek(output, orig_offset);
- return ret;
+ return -1;
+ }
+ return 0;
}
static int
@@ -350,32 +349,40 @@ static int dbox_save_commit(struct dbox_
struct dbox_mail_index_record rec;
struct dbox_save_mail *mails;
unsigned int i, count;
-
- /* first write updated mail headers and collect all files we wrote to */
+ int ret = 0;
+
+ /* assign UIDs to mails */
mails = array_get_modifiable(&ctx->mails, &count);
- for (i = 0; i < count; i++) {
- if (dbox_save_mail_write_header(&mails[i], first_uid++) < 0)
- return -1;
- }
-
- /* update append offsets in file headers */
+ for (i = 0; i < count; i++)
+ mails[i].uid = first_uid++;
+
+ /* update headers */
qsort(mails, count, sizeof(*mails), dbox_save_mail_file_cmp);
for (i = 0; i < count; i++) {
- if (i > 0 && mails[i].file == mails[i-1].file) {
- /* already written */
- continue;
- }
-
- if (dbox_save_file_commit_header(&mails[i]) < 0) {
- /* have to uncommit all changes so far */
- for (; i > 0; i--) {
- if (i > 1 &&
- mails[i-2].file == mails[i-1].file)
- continue;
- dbox_save_file_uncommit_header(&mails[i-1]);
+ if (dbox_file_open_if_needed(mails[i].file) < 0 ||
+ dbox_save_mail_write_header(&mails[i]) < 0) {
+ ret = -1;
+ break;
+ }
+
+ /* write file header only once after all mails headers
+ have been written */
+ if (i+1 == count || mails[i].file != mails[i+1].file) {
+ if (dbox_save_file_commit_header(&mails[i]) < 0) {
+ ret = -1;
+ break;
}
- return -1;
- }
+ dbox_file_close(mails[i].file);
+ }
+ }
+ if (ret < 0) {
+ /* have to uncommit all written changes */
+ for (; i > 0; i--) {
+ if (i > 1 && mails[i-2].file == mails[i-1].file)
+ continue;
+ dbox_save_file_uncommit_header(&mails[i-1]);
+ }
+ return -1;
}
/* set file_id / offsets to records */
More information about the dovecot-cvs
mailing list