dovecot-2.1: lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to d...
dovecot at dovecot.org
dovecot at dovecot.org
Thu Oct 18 05:23:39 EEST 2012
details: http://hg.dovecot.org/dovecot-2.1/rev/51f2be9aa8ad
changeset: 14767:51f2be9aa8ad
user: Timo Sirainen <tss at iki.fi>
date: Thu Oct 18 05:22:36 2012 +0300
description:
lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file.
diffstat:
src/lib-index/mail-cache-compress.c | 4 +
src/lib-index/mail-cache-private.h | 8 ++-
src/lib-index/mail-cache.c | 123 +++++++++++++++++++++++------------
src/lib-index/mail-index.h | 5 +-
4 files changed, 96 insertions(+), 44 deletions(-)
diffs (232 lines):
diff -r 0d1de37ad9d8 -r 51f2be9aa8ad src/lib-index/mail-cache-compress.c
--- a/src/lib-index/mail-cache-compress.c Thu Oct 18 05:16:54 2012 +0300
+++ b/src/lib-index/mail-cache-compress.c Thu Oct 18 05:22:36 2012 +0300
@@ -461,6 +461,9 @@
if (MAIL_INDEX_IS_IN_MEMORY(cache->index) || cache->index->readonly)
return 0;
+ /* compression isn't very efficient with small read()s */
+ cache->map_with_read = FALSE;
+
if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
/* we're using dotlocking, cache file creation itself creates
the dotlock file we need. */
@@ -495,5 +498,6 @@
bool mail_cache_need_compress(struct mail_cache *cache)
{
return cache->need_compress_file_seq != 0 &&
+ (cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) == 0 &&
!cache->index->readonly;
}
diff -r 0d1de37ad9d8 -r 51f2be9aa8ad src/lib-index/mail-cache-private.h
--- a/src/lib-index/mail-cache-private.h Thu Oct 18 05:16:54 2012 +0300
+++ b/src/lib-index/mail-cache-private.h Thu Oct 18 05:22:36 2012 +0300
@@ -130,9 +130,14 @@
ino_t st_ino;
dev_t st_dev;
+ size_t mmap_length;
+ /* a) mmaping the whole file */
void *mmap_base;
- size_t mmap_length;
+ /* b) using file cache */
struct file_cache *file_cache;
+ /* c) using small read() calls with MAIL_INDEX_OPEN_FLAG_SAVEONLY */
+ uoff_t read_offset;
+ buffer_t *read_buf;
/* mail_cache_map() increases this always. */
unsigned int remap_counter;
@@ -169,6 +174,7 @@
unsigned int hdr_modified:1;
unsigned int field_header_write_pending:1;
unsigned int compressing:1;
+ unsigned int map_with_read:1;
};
struct mail_cache_loop_track {
diff -r 0d1de37ad9d8 -r 51f2be9aa8ad src/lib-index/mail-cache.c
--- a/src/lib-index/mail-cache.c Thu Oct 18 05:16:54 2012 +0300
+++ b/src/lib-index/mail-cache.c Thu Oct 18 05:22:36 2012 +0300
@@ -7,6 +7,7 @@
#include "nfs-workarounds.h"
#include "file-cache.h"
#include "mmap-util.h"
+#include "read-full.h"
#include "write-full.h"
#include "mail-cache-private.h"
@@ -264,17 +265,87 @@
return TRUE;
}
+static int
+mail_cache_map_finish(struct mail_cache *cache, uoff_t offset, size_t size,
+ const void *data, bool copy_hdr)
+{
+ if (offset == 0) {
+ const struct mail_cache_header *hdr = data;
+
+ if (!mail_cache_verify_header(cache, hdr)) {
+ cache->need_compress_file_seq =
+ !MAIL_CACHE_IS_UNUSABLE(cache) &&
+ cache->hdr->file_seq != 0 ?
+ cache->hdr->file_seq : 0;
+ return -1;
+ }
+ if (!copy_hdr)
+ cache->hdr = data;
+ else {
+ memcpy(&cache->hdr_ro_copy, data,
+ sizeof(cache->hdr_ro_copy));
+ cache->hdr = &cache->hdr_ro_copy;
+ }
+ mail_cache_update_need_compress(cache);
+ }
+
+ if (offset + size > cache->mmap_length)
+ return 0;
+ return 1;
+}
+
+static int
+mail_cache_map_with_read(struct mail_cache *cache, size_t offset, size_t size,
+ const void **data_r)
+{
+ void *data;
+ ssize_t ret;
+
+ if (cache->read_buf == NULL) {
+ cache->read_buf =
+ buffer_create_dynamic(default_pool, size);
+ } else if (cache->read_offset <= offset &&
+ cache->read_offset + cache->read_buf->used >= offset+size) {
+ /* already mapped */
+ *data_r = CONST_PTR_OFFSET(cache->mmap_base,
+ offset - cache->read_offset);
+ return mail_cache_map_finish(cache, offset, size, *data_r, TRUE);
+ } else {
+ buffer_set_used_size(cache->read_buf, 0);
+ }
+ data = buffer_append_space_unsafe(cache->read_buf, size);
+ ret = pread(cache->fd, data, size, offset);
+ if (ret < 0) {
+ if (errno != ESTALE)
+ mail_cache_set_syscall_error(cache, "read()");
+
+ buffer_set_used_size(cache->read_buf, 0);
+ cache->hdr = NULL;
+ cache->mmap_length = 0;
+ return -1;
+ }
+ buffer_set_used_size(cache->read_buf, ret);
+
+ cache->read_offset = offset;
+ cache->mmap_length = offset + size;
+
+ *data_r = data;
+ return mail_cache_map_finish(cache, offset, size, data, TRUE);
+}
+
int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size,
const void **data_r)
{
const void *data;
ssize_t ret;
- cache->remap_counter++;
-
if (size == 0)
size = sizeof(struct mail_cache_header);
+ cache->remap_counter++;
+ if (cache->map_with_read)
+ return mail_cache_map_with_read(cache, offset, size, data_r);
+
if (cache->file_cache != NULL) {
cache->hdr = NULL;
@@ -294,28 +365,9 @@
data = file_cache_get_map(cache->file_cache,
&cache->mmap_length);
-
- if (offset == 0) {
- if (!mail_cache_verify_header(cache, data)) {
- cache->need_compress_file_seq =
- !MAIL_CACHE_IS_UNUSABLE(cache) &&
- cache->hdr->file_seq != 0 ?
- cache->hdr->file_seq : 0;
- return -1;
- }
- memcpy(&cache->hdr_ro_copy, data,
- sizeof(cache->hdr_ro_copy));
- }
- cache->hdr = &cache->hdr_ro_copy;
- if (offset == 0)
- mail_cache_update_need_compress(cache);
-
- if (offset > cache->mmap_length) {
- *data_r = NULL;
- return 0;
- }
- *data_r = CONST_PTR_OFFSET(data, offset);
- return offset + size > cache->mmap_length ? 0 : 1;
+ *data_r = offset > cache->mmap_length ? NULL :
+ CONST_PTR_OFFSET(data, offset);
+ return mail_cache_map_finish(cache, offset, size, *data_r, TRUE);
}
if (offset < cache->mmap_length &&
@@ -348,24 +400,9 @@
mail_cache_set_syscall_error(cache, "mmap()");
return -1;
}
-
- if (!mail_cache_verify_header(cache, cache->mmap_base)) {
- cache->need_compress_file_seq =
- !MAIL_CACHE_IS_UNUSABLE(cache) &&
- cache->hdr->file_seq != 0 ?
- cache->hdr->file_seq : 0;
- return -1;
- }
-
- cache->hdr = cache->mmap_base;
- if (offset == 0)
- mail_cache_update_need_compress(cache);
- if (offset > cache->mmap_length) {
- *data_r = NULL;
- return 0;
- }
- *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset);
- return offset + size > cache->mmap_length ? 0 : 1;
+ *data_r = offset > cache->mmap_length ? NULL :
+ CONST_PTR_OFFSET(cache->mmap_base, offset);
+ return mail_cache_map_finish(cache, offset, size, *data_r, FALSE);
}
static int mail_cache_try_open(struct mail_cache *cache)
@@ -437,6 +474,8 @@
if (!MAIL_INDEX_IS_IN_MEMORY(index) &&
(index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0)
cache->file_cache = file_cache_new(-1);
+ cache->map_with_read =
+ (cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) != 0;
cache->ext_id =
mail_index_ext_register(index, "cache", 0,
diff -r 0d1de37ad9d8 -r 51f2be9aa8ad src/lib-index/mail-index.h
--- a/src/lib-index/mail-index.h Thu Oct 18 05:16:54 2012 +0300
+++ b/src/lib-index/mail-index.h Thu Oct 18 05:22:36 2012 +0300
@@ -27,7 +27,10 @@
MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS = 0x100,
/* If we run out of disk space, fail modifications instead of moving
indexes to memory. */
- MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY = 0x200
+ MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY = 0x200,
+ /* We're only going to save new messages to the index.
+ Avoid unnecessary reads. */
+ MAIL_INDEX_OPEN_FLAG_SAVEONLY = 0x400
};
enum mail_index_header_compat_flags {
More information about the dovecot-cvs
mailing list