dovecot-1.1: When rebuilding uidlist, read the existing one enti...
dovecot at dovecot.org
dovecot at dovecot.org
Fri Mar 7 14:44:31 EET 2008
details: http://hg.dovecot.org/dovecot-1.1/rev/4445415da4ff
changeset: 7373:4445415da4ff
user: Timo Sirainen <tss at iki.fi>
date: Fri Mar 07 14:25:06 2008 +0200
description:
When rebuilding uidlist, read the existing one entirely into memory at the
beginning. This this is a lot faster than reading it with random access in
smaller pieces.
diffstat:
1 file changed, 45 insertions(+), 1 deletion(-)
src/plugins/fts-squat/squat-uidlist.c | 46 ++++++++++++++++++++++++++++++++-
diffs (82 lines):
diff -r 2201fd4548e3 -r 4445415da4ff src/plugins/fts-squat/squat-uidlist.c
--- a/src/plugins/fts-squat/squat-uidlist.c Fri Mar 07 08:20:36 2008 +0200
+++ b/src/plugins/fts-squat/squat-uidlist.c Fri Mar 07 14:25:06 2008 +0200
@@ -8,12 +8,12 @@
#include "read-full.h"
#include "write-full.h"
#include "ostream.h"
+#include "mmap-util.h"
#include "squat-trie-private.h"
#include "squat-uidlist.h"
#include <stdio.h>
#include <sys/stat.h>
-#include <sys/mman.h>
#define UIDLIST_LIST_SIZE 31
#define UIDLIST_BLOCK_LIST_COUNT 100
@@ -473,6 +473,40 @@ static int squat_uidlist_map(struct squa
return squat_uidlist_map_header(uidlist);
}
+static int squat_uidlist_read_to_memory(struct squat_uidlist *uidlist)
+{
+ size_t i, page_size = mmap_get_page_size();
+ char x;
+
+ if (uidlist->file_cache != NULL) {
+ return uidlist_file_cache_read(uidlist, 0,
+ uidlist->hdr.used_file_size);
+ }
+ /* Tell the kernel we're going to use the uidlist data, so it loads
+ it into memory and keeps it there. */
+ (void)madvise(uidlist->mmap_base, uidlist->mmap_size, MADV_WILLNEED);
+ /* It also speeds up a bit for us to sequentially load everything
+ into memory, although at least Linux catches up quite fast even
+ without this code. Compiler can quite easily optimize away this
+ entire for loop, but volatile seems to help with gcc 4.2. */
+ for (i = 0; i < uidlist->mmap_size; i += page_size)
+ x = ((const volatile char *)uidlist->data)[i];
+ return 0;
+}
+
+static void squat_uidlist_free_from_memory(struct squat_uidlist *uidlist)
+{
+ size_t page_size = mmap_get_page_size();
+
+ if (uidlist->file_cache != NULL) {
+ file_cache_invalidate(uidlist->file_cache,
+ page_size, (uoff_t)-1);
+ } else {
+ (void)madvise(uidlist->mmap_base, uidlist->mmap_size,
+ MADV_DONTNEED);
+ }
+}
+
struct squat_uidlist *squat_uidlist_init(struct squat_trie *trie)
{
struct squat_uidlist *uidlist;
@@ -880,6 +914,12 @@ int squat_uidlist_rebuild_init(struct sq
build_ctx->build_hdr.count*2/3)
return 0;
}
+
+ /* make sure the entire uidlist is in memory before beginning,
+ otherwise the pages are faulted to memory in random order which
+ takes forever. */
+ if (squat_uidlist_read_to_memory(build_ctx->uidlist) < 0)
+ return -1;
temp_path = t_strconcat(build_ctx->uidlist->path, ".tmp", NULL);
fd = open(temp_path, O_RDWR | O_TRUNC | O_CREAT, 0600);
@@ -1041,6 +1081,10 @@ int squat_uidlist_rebuild_finish(struct
}
ctx->build_ctx->need_reopen = TRUE;
}
+
+ /* we no longer require the entire uidlist to be in memory,
+ let it be used for something more useful. */
+ squat_uidlist_free_from_memory(ctx->uidlist);
o_stream_unref(&ctx->output);
if (close(ctx->fd) < 0)
More information about the dovecot-cvs
mailing list