dovecot-2.2: virtual: Never keep more than specified number of p...
dovecot at dovecot.org
dovecot at dovecot.org
Thu Jul 3 14:46:06 UTC 2014
details: http://hg.dovecot.org/dovecot-2.2/rev/251747c3afe5
changeset: 17580:251747c3afe5
user: Timo Sirainen <tss at iki.fi>
date: Thu Jul 03 17:44:32 2014 +0300
description:
virtual: Never keep more than specified number of physical mailboxes open.
This should make virtual mailboxes work for users who have a a ton of
mailboxes with a ton of mails. Earlier code would likely have failed either
with "Too many open files" or crashed with "Out of memory".
You can change the max number of open mailboxes with:
plugin {
virtual_max_open_mailboxes = 64
}
The default is 64.
diffstat:
src/plugins/virtual/virtual-mail.c | 4 +-
src/plugins/virtual/virtual-save.c | 4 +-
src/plugins/virtual/virtual-storage.c | 160 +++++++++++++++++++++++++++++++--
src/plugins/virtual/virtual-storage.h | 22 ++++
src/plugins/virtual/virtual-sync.c | 78 +++++++++------
5 files changed, 221 insertions(+), 47 deletions(-)
diffs (truncated from 534 to 300 lines):
diff -r 092137bd092b -r 251747c3afe5 src/plugins/virtual/virtual-mail.c
--- a/src/plugins/virtual/virtual-mail.c Thu Jul 03 17:40:37 2014 +0300
+++ b/src/plugins/virtual/virtual-mail.c Thu Jul 03 17:44:32 2014 +0300
@@ -108,11 +108,13 @@
bbox = virtual_backend_box_lookup(mbox, vmail->cur_vrec.mailbox_id);
vmail->cur_backend_mail = backend_mail_find(vmail, bbox->box);
if (vmail->cur_backend_mail == NULL) {
- if (mailbox_open(bbox->box) < 0) {
+ if (virtual_backend_box_open(mbox, bbox) < 0) {
virtual_box_copy_error(mail->box, bbox->box);
return -1;
}
(void)virtual_mail_set_backend_mail(mail, bbox);
+ } else {
+ virtual_backend_box_accessed(mbox, bbox);
}
vmail->cur_lost = !mail_set_uid(vmail->cur_backend_mail,
vmail->cur_vrec.real_uid);
diff -r 092137bd092b -r 251747c3afe5 src/plugins/virtual/virtual-save.c
--- a/src/plugins/virtual/virtual-save.c Thu Jul 03 17:40:37 2014 +0300
+++ b/src/plugins/virtual/virtual-save.c Thu Jul 03 17:44:32 2014 +0300
@@ -36,7 +36,7 @@
i_assert(ctx->backend_save_ctx == NULL);
i_assert(ctx->open_errstr == NULL);
- if (mailbox_open(mbox->save_bbox->box) < 0) {
+ if (virtual_backend_box_open(mbox, mbox->save_bbox) < 0) {
errstr = mailbox_get_last_error(mbox->save_bbox->box,
&ctx->open_error);
ctx->open_errstr = i_strdup(errstr);
@@ -45,6 +45,8 @@
virtual_transaction_get(_t, mbox->save_bbox->box);
ctx->backend_save_ctx = mailbox_save_alloc(backend_trans);
}
+ } else {
+ virtual_backend_box_accessed(mbox, mbox->save_bbox);
}
return _t->save_ctx;
}
diff -r 092137bd092b -r 251747c3afe5 src/plugins/virtual/virtual-storage.c
--- a/src/plugins/virtual/virtual-storage.c Thu Jul 03 17:40:37 2014 +0300
+++ b/src/plugins/virtual/virtual-storage.c Thu Jul 03 17:44:32 2014 +0300
@@ -4,6 +4,7 @@
#include "array.h"
#include "ioloop.h"
#include "str.h"
+#include "llist.h"
#include "mkdir-parents.h"
#include "unlink-directory.h"
#include "index-mail.h"
@@ -20,6 +21,8 @@
#include <dirent.h>
#include <sys/stat.h>
+#define VIRTUAL_DEFAULT_MAX_OPEN_MAILBOXES 64
+
extern struct mail_storage virtual_storage;
extern struct mailbox virtual_mailbox;
extern struct virtual_mailbox_vfuncs virtual_mailbox_vfuncs;
@@ -70,6 +73,24 @@
return &storage->storage;
}
+static int
+virtual_storage_create(struct mail_storage *_storage,
+ struct mail_namespace *ns ATTR_UNUSED,
+ const char **error_r)
+{
+ struct virtual_storage *storage = (struct virtual_storage *)_storage;
+ const char *value;
+
+ value = mail_user_plugin_getenv(_storage->user, "virtual_max_open_mailboxes");
+ if (value == NULL)
+ storage->max_open_mailboxes = VIRTUAL_DEFAULT_MAX_OPEN_MAILBOXES;
+ else if (str_to_uint(value, &storage->max_open_mailboxes) < 0) {
+ *error_r = "Invalid virtual_max_open_mailboxes setting";
+ return -1;
+ }
+ return 0;
+}
+
static void
virtual_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED,
struct mailbox_list_settings *set)
@@ -145,9 +166,9 @@
return -1;
}
-static int virtual_backend_box_open(struct virtual_mailbox *mbox,
- struct virtual_backend_box *bbox,
- enum mailbox_flags flags)
+static int virtual_backend_box_alloc(struct virtual_mailbox *mbox,
+ struct virtual_backend_box *bbox,
+ enum mailbox_flags flags)
{
struct mail_user *user = mbox->storage->storage.user;
struct mail_namespace *ns;
@@ -190,7 +211,7 @@
bboxes = array_get(&mbox->backend_boxes, &count);
for (i = 0; i < count; ) {
- ret = virtual_backend_box_open(mbox, bboxes[i], flags);
+ ret = virtual_backend_box_alloc(mbox, bboxes[i], flags);
if (ret <= 0) {
if (ret < 0)
break;
@@ -236,6 +257,119 @@
return &mbox->box;
}
+void virtual_backend_box_sync_mail_unset(struct virtual_backend_box *bbox)
+{
+ struct mailbox_transaction_context *trans;
+
+ if (bbox->sync_mail != NULL) {
+ trans = bbox->sync_mail->transaction;
+ mail_free(&bbox->sync_mail);
+ (void)mailbox_transaction_commit(&trans);
+ }
+}
+
+static bool
+virtual_backend_box_close_any_except(struct virtual_mailbox *mbox,
+ struct virtual_backend_box *except_bbox)
+{
+ struct virtual_backend_box *bbox;
+
+ /* first try to close a mailbox without any transactions.
+ we'll also skip any mailbox that has notifications enabled (ideally
+ these would be handled by mailbox list index) */
+ for (bbox = mbox->open_backend_boxes_head; bbox != NULL; bbox = bbox->next_open) {
+ i_assert(bbox->box->opened);
+
+ if (bbox != except_bbox &&
+ bbox->box->transaction_count == 0 &&
+ bbox->box->notify_callback == NULL) {
+ i_assert(bbox->sync_mail == NULL);
+ virtual_backend_box_close(mbox, bbox);
+ return TRUE;
+ }
+ }
+
+ /* next try to close a mailbox that has sync_mail, but no
+ other transactions */
+ for (bbox = mbox->open_backend_boxes_head; bbox != NULL; bbox = bbox->next_open) {
+ if (bbox != except_bbox &&
+ bbox->sync_mail != NULL &&
+ bbox->box->transaction_count == 1 &&
+ bbox->box->notify_callback == NULL) {
+ virtual_backend_box_sync_mail_unset(bbox);
+ i_assert(bbox->box->transaction_count == 0);
+ virtual_backend_box_close(mbox, bbox);
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+void virtual_backend_box_opened(struct virtual_mailbox *mbox,
+ struct virtual_backend_box *bbox)
+{
+ /* the backend mailbox was already opened. if we didn't get here
+ from virtual_backend_box_open() we may need to close a mailbox */
+ while (mbox->backends_open_count > mbox->storage->max_open_mailboxes &&
+ virtual_backend_box_close_any_except(mbox, bbox))
+ ;
+
+ mbox->backends_open_count++;
+ DLLIST2_APPEND_FULL(&mbox->open_backend_boxes_head,
+ &mbox->open_backend_boxes_tail, bbox,
+ prev_open, next_open);
+}
+
+int virtual_backend_box_open(struct virtual_mailbox *mbox,
+ struct virtual_backend_box *bbox)
+{
+ i_assert(!bbox->box->opened);
+
+ /* try to keep the number of open mailboxes below the threshold
+ before opening the mailbox */
+ while (mbox->backends_open_count >= mbox->storage->max_open_mailboxes &&
+ virtual_backend_box_close_any_except(mbox, bbox))
+ ;
+
+ if (mailbox_open(bbox->box) < 0)
+ return -1;
+ virtual_backend_box_opened(mbox, bbox);
+ return 0;
+}
+
+void virtual_backend_box_close(struct virtual_mailbox *mbox,
+ struct virtual_backend_box *bbox)
+{
+ i_assert(bbox->box->opened);
+
+ if (bbox->search_result != NULL)
+ mailbox_search_result_free(&bbox->search_result);
+
+ if (bbox->search_args != NULL &&
+ bbox->search_args_initialized) {
+ mail_search_args_deinit(bbox->search_args);
+ bbox->search_args_initialized = FALSE;
+ }
+ i_assert(mbox->backends_open_count > 0);
+ mbox->backends_open_count--;
+
+ DLLIST2_REMOVE_FULL(&mbox->open_backend_boxes_head,
+ &mbox->open_backend_boxes_tail, bbox,
+ prev_open, next_open);
+ mailbox_close(bbox->box);
+}
+
+void virtual_backend_box_accessed(struct virtual_mailbox *mbox,
+ struct virtual_backend_box *bbox)
+{
+ DLLIST2_REMOVE_FULL(&mbox->open_backend_boxes_head,
+ &mbox->open_backend_boxes_tail, bbox,
+ prev_open, next_open);
+ DLLIST2_APPEND_FULL(&mbox->open_backend_boxes_head,
+ &mbox->open_backend_boxes_tail, bbox,
+ prev_open, next_open);
+}
+
static void virtual_mailbox_close_internal(struct virtual_mailbox *mbox)
{
struct virtual_backend_box **bboxes;
@@ -243,23 +377,18 @@
bboxes = array_get_modifiable(&mbox->backend_boxes, &count);
for (i = 0; i < count; i++) {
- if (bboxes[i]->search_result != NULL)
- mailbox_search_result_free(&bboxes[i]->search_result);
-
if (bboxes[i]->box == NULL)
continue;
- if (bboxes[i]->search_args != NULL &&
- bboxes[i]->search_args_initialized) {
- mail_search_args_deinit(bboxes[i]->search_args);
- bboxes[i]->search_args_initialized = FALSE;
- }
+ if (bboxes[i]->box->opened)
+ virtual_backend_box_close(mbox, bboxes[i]);
mailbox_free(&bboxes[i]->box);
if (array_is_created(&bboxes[i]->sync_outside_expunges))
array_free(&bboxes[i]->sync_outside_expunges);
array_free(&bboxes[i]->sync_pending_removes);
array_free(&bboxes[i]->uids);
}
+ i_assert(mbox->backends_open_count == 0);
}
static int
@@ -343,16 +472,19 @@
struct virtual_backend_box *const *bboxes;
unsigned int i, count;
struct mailbox_status status;
+ bool opened;
mbox->have_guids = TRUE;
mbox->have_save_guids = TRUE;
bboxes = array_get(&mbox->backend_boxes, &count);
for (i = 0; i < count; i++) {
+ opened = bboxes[i]->box->opened;
if (mailbox_get_status(bboxes[i]->box, 0, &status) < 0) {
virtual_box_copy_error(&mbox->box, bboxes[i]->box);
return -1;
}
+ i_assert(bboxes[i]->box->opened == opened);
if (!status.have_guids)
mbox->have_guids = FALSE;
if (!status.have_save_guids)
@@ -432,7 +564,7 @@
to wait for changes and avoid opening all mailboxes here. */
array_foreach(&mbox->backend_boxes, bboxp) {
- if (mailbox_open((*bboxp)->box) < 0) {
+ if (virtual_backend_box_open(mbox, *bboxp) < 0) {
/* we can't report error in here, so do it later */
(*bboxp)->open_failed = TRUE;
continue;
@@ -548,7 +680,7 @@
.v = {
NULL,
virtual_storage_alloc,
- NULL,
+ virtual_storage_create,
index_storage_destroy,
NULL,
virtual_storage_get_list_settings,
diff -r 092137bd092b -r 251747c3afe5 src/plugins/virtual/virtual-storage.h
--- a/src/plugins/virtual/virtual-storage.h Thu Jul 03 17:40:37 2014 +0300
+++ b/src/plugins/virtual/virtual-storage.h Thu Jul 03 17:44:32 2014 +0300
@@ -51,6 +51,8 @@
/* List of mailboxes while a virtual mailbox is being opened.
Used to track loops. */
ARRAY_TYPE(const_string) open_stack;
+
+ unsigned int max_open_mailboxes;
More information about the dovecot-cvs
mailing list