[dovecot-cvs] dovecot/src/lib-index mailbox-list-index-private.h, 1.1, 1.2 mailbox-list-index-sync.c, 1.1, 1.2 mailbox-list-index.c, 1.1, 1.2
tss at dovecot.org
tss at dovecot.org
Sun Dec 10 16:04:50 UTC 2006
Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv5671/lib-index
Modified Files:
mailbox-list-index-private.h mailbox-list-index-sync.c
mailbox-list-index.c
Log Message:
Mailbox list index works now with mmap_disable=yes.
Index: mailbox-list-index-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mailbox-list-index-private.h,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mailbox-list-index-private.h 25 Nov 2006 22:17:40 -0000 1.1
+++ mailbox-list-index-private.h 10 Dec 2006 16:04:46 -0000 1.2
@@ -51,15 +51,21 @@
char *filepath;
char separator;
struct mail_index *mail_index;
+ struct file_cache *file_cache;
int fd;
void *mmap_base;
+ const void *const_mmap_base;
size_t mmap_size;
const struct mailbox_list_index_header *hdr;
+
+ unsigned int mmap_disable:1;
};
#define MAILBOX_LIST_RECORDS(dir) \
+ ((const struct mailbox_list_record *)(dir + 1))
+#define MAILBOX_LIST_RECORDS_MODIFIABLE(dir) \
((struct mailbox_list_record *)(dir + 1))
#define MAILBOX_LIST_RECORD_IDX(dir, rec) \
((rec) - MAILBOX_LIST_RECORDS(dir))
Index: mailbox-list-index-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mailbox-list-index-sync.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mailbox-list-index-sync.c 25 Nov 2006 22:17:40 -0000 1.1
+++ mailbox-list-index-sync.c 10 Dec 2006 16:04:46 -0000 1.2
@@ -4,8 +4,10 @@
#include "array.h"
#include "bsearch-insert-pos.h"
#include "crc32.h"
+#include "file-cache.h"
#include "file-set-size.h"
#include "mmap-util.h"
+#include "ostream.h"
#include "mail-index-private.h"
#include "mailbox-list-index-private.h"
@@ -53,6 +55,9 @@
struct mailbox_list_index_header hdr;
struct mailbox_list_sync_dir *root, *sync_root;
+ struct ostream *output;
+ buffer_t *output_buf;
+
unsigned int failed:1;
unsigned int partial:1;
unsigned int seen_sync_root:1;
@@ -97,6 +102,9 @@
recs = MAILBOX_LIST_RECORDS(dir);
for (i = 0; i < dir->count; i++) {
+ if (recs[i].deleted)
+ continue;
+
sync_rec = array_append_space(&sync_dir->records);
sync_rec->name_hash = recs[i].name_hash;
sync_rec->uid = recs[i].uid;
@@ -104,7 +112,7 @@
mail_index_offset_to_uint32(recs[i].dir_offset);
max_len = ctx->index->mmap_size - recs[i].name_offset;
- name = CONST_PTR_OFFSET(ctx->index->mmap_base,
+ name = CONST_PTR_OFFSET(ctx->index->const_mmap_base,
recs[i].name_offset);
sync_rec->name = p_strndup(ctx->pool, name, max_len);
@@ -430,15 +438,30 @@
/* all allocations must be 32bit aligned */
pos = (pos + 3) & ~3;
- if (pos + size > ctx->index->mmap_size) {
- if (mailbox_list_index_sync_grow(ctx, size + 3) < 0)
- return -1;
+ if (ctx->index->mmap_disable) {
+ /* write the data into temporary buffer first */
+ buffer_set_used_size(ctx->output_buf, 0);
- i_assert(pos + size < ctx->index->mmap_size);
+ if (ctx->output->offset != pos) {
+ i_assert(pos > ctx->output->offset &&
+ pos - ctx->output->offset <= 3);
+
+ buffer_append_zero(ctx->output_buf,
+ pos - ctx->output->offset);
+ }
+
+ *base_r = buffer_append_space_unsafe(ctx->output_buf, size);
+ } else {
+ if (pos + size > ctx->index->mmap_size) {
+ if (mailbox_list_index_sync_grow(ctx, size + 3) < 0)
+ return -1;
+
+ i_assert(pos + size < ctx->index->mmap_size);
+ }
+ *base_r = PTR_OFFSET(ctx->index->mmap_base, pos);
}
*base_offset_r = pos;
- *base_r = PTR_OFFSET(ctx->index->mmap_base, *base_offset_r);
ctx->hdr.used_space = pos + size;
return 0;
}
@@ -448,8 +471,11 @@
struct mailbox_list_sync_dir *sync_dir,
uint32_t offset_pos, bool partial)
{
- struct mailbox_list_dir_record *dir, *new_dir;
- struct mailbox_list_record *recs, *new_recs;
+ struct mailbox_list_index *index = ctx->index;
+ const struct mailbox_list_dir_record *dir;
+ const struct mailbox_list_record *recs;
+ struct mailbox_list_dir_record *new_dir;
+ struct mailbox_list_record *new_recs;
struct mailbox_list_sync_record *sync_recs;
unsigned int src, dest, orig, count, nondeleted_count;
unsigned int name_space_needed, deleted_space;
@@ -457,7 +483,7 @@
void *base;
i_assert((offset_pos % sizeof(uint32_t)) == 0);
- i_assert(offset_pos < ctx->index->mmap_size);
+ i_assert(offset_pos < index->mmap_size);
/* count how much space we need and how much we wasted for deleted
records */
@@ -492,15 +518,16 @@
} else {
/* the offset should have been verified already to be valid */
i_assert(sync_dir->offset == offset_pos);
- i_assert(sync_dir->offset < ctx->index->mmap_size);
- dir = PTR_OFFSET(ctx->index->mmap_base, sync_dir->offset);
+ i_assert(sync_dir->offset < index->mmap_size);
+ dir = CONST_PTR_OFFSET(index->const_mmap_base,
+ sync_dir->offset);
recs = MAILBOX_LIST_RECORDS(dir);
}
new_dir = base;
new_dir->count = nondeleted_count;
- new_recs = MAILBOX_LIST_RECORDS(new_dir);
+ new_recs = MAILBOX_LIST_RECORDS_MODIFIABLE(new_dir);
name_pos = (const char *)(new_recs + nondeleted_count) -
(const char *)base;
for (src = dest = 0; src < count; src++) {
@@ -547,16 +574,42 @@
i_assert(dest == nondeleted_count);
i_assert(name_pos == name_space_needed);
+ if (index->mmap_disable) {
+ file_cache_write(index->file_cache, ctx->output_buf->data,
+ ctx->output_buf->used, ctx->output->offset);
+ o_stream_send(ctx->output, ctx->output_buf->data,
+ ctx->output_buf->used);
+ }
+
if (offset_pos == 0) {
/* we're writing the root directory */
- i_assert(base_offset == sizeof(*ctx->index->hdr));
+ i_assert(base_offset == sizeof(*index->hdr));
} else {
/* add a link to this newly created directory. */
- uint32_t *pos;
+ uint32_t data = mail_index_uint32_to_offset(base_offset);
- pos = PTR_OFFSET(ctx->index->mmap_base, offset_pos);
- i_assert(mail_index_offset_to_uint32(*pos) == 0);
- *pos = mail_index_uint32_to_offset(base_offset);
+ if (!index->mmap_disable) {
+ uint32_t *pos;
+
+ pos = PTR_OFFSET(index->mmap_base, offset_pos);
+ i_assert(mail_index_offset_to_uint32(*pos) == 0);
+ *pos = data;
+ } else {
+ uoff_t old_offset = ctx->output->offset;
+
+ file_cache_write(index->file_cache,
+ &data, sizeof(data), offset_pos);
+
+ o_stream_seek(ctx->output, offset_pos);
+ o_stream_send(ctx->output, &data, sizeof(data));
+ o_stream_seek(ctx->output, old_offset);
+ }
+ }
+
+ if (index->mmap_disable) {
+ /* file_cache_write() calls may have moved mmaping */
+ index->const_mmap_base = file_cache_get_map(index->file_cache,
+ &index->mmap_size);
}
sync_dir->offset = base_offset;
@@ -581,11 +634,34 @@
i_assert(dir->count == count);
i_assert(sync_dir->seen_records_count < count);
- recs = MAILBOX_LIST_RECORDS(dir);
+ if (!ctx->index->mmap_disable)
+ recs = MAILBOX_LIST_RECORDS_MODIFIABLE(dir);
+ else {
+ /* @UNSAFE: copy the records into a temporary buffer that
+ we modify and then write back to disk */
+ t_push();
+ recs = t_new(struct mailbox_list_record, dir->count);
+ memcpy(recs, MAILBOX_LIST_RECORDS(dir),
+ sizeof(struct mailbox_list_record) * dir->count);
+ }
for (i = 0; i < dir->count; i++) {
if (!sync_recs[i].seen)
recs[i].deleted = TRUE;
}
+ if (ctx->index->mmap_disable) {
+ uoff_t offset, old_offset;
+ size_t size = sizeof(struct mailbox_list_record) * dir->count;
+
+ offset = sync_dir->offset +
+ sizeof(struct mailbox_list_dir_record);
+ file_cache_write(ctx->index->file_cache, recs, size, offset);
+
+ old_offset = ctx->output->offset;
+ o_stream_seek(ctx->output, offset);
+ o_stream_send(ctx->output, recs, size);
+ o_stream_seek(ctx->output, old_offset);
+ t_pop();
+ }
return 0;
}
@@ -643,7 +719,8 @@
continue;
/* these may change after each sync_write_dir() call */
- dir = CONST_PTR_OFFSET(ctx->index->mmap_base, sync_dir->offset);
+ dir = CONST_PTR_OFFSET(ctx->index->const_mmap_base,
+ sync_dir->offset);
recs = MAILBOX_LIST_RECORDS(dir);
/* child_offset_pos needs to point to record's dir_offset */
@@ -654,7 +731,7 @@
i_assert(j < dir->count);
child_offset_pos = (const char *)&recs[j].dir_offset -
- (const char *)ctx->index->mmap_base;
+ (const char *)ctx->index->const_mmap_base;
if (mailbox_list_index_sync_write_dir(ctx, sync_recs[i].dir,
child_offset_pos,
partial) < 0)
@@ -669,6 +746,13 @@
struct mailbox_list_index_header *hdr;
bool partial;
+ if (ctx->index->mmap_disable) {
+ ctx->output = o_stream_create_file(ctx->index->fd, default_pool,
+ 0, FALSE);
+ ctx->output_buf = buffer_create_dynamic(default_pool, 4096);
+ o_stream_seek(ctx->output, ctx->hdr.used_space);
+ }
+
if (ctx->sync_root == ctx->root) {
ctx->seen_sync_root = TRUE;
partial = (ctx->flags & MAILBOX_LIST_SYNC_FLAG_PARTIAL) != 0;
@@ -681,15 +765,25 @@
if (mailbox_list_index_sync_write_dir(ctx, ctx->root, 0, partial) < 0)
return -1;
- /* update header */
- hdr = ctx->index->mmap_base;
- hdr->next_uid = ctx->hdr.next_uid;
- hdr->used_space = ctx->hdr.used_space;
- hdr->deleted_space = ctx->hdr.deleted_space;
+ if (!ctx->index->mmap_disable) {
+ /* update header */
+ hdr = ctx->index->mmap_base;
+ hdr->next_uid = ctx->hdr.next_uid;
+ hdr->used_space = ctx->hdr.used_space;
+ hdr->deleted_space = ctx->hdr.deleted_space;
- if (msync(ctx->index->mmap_base, hdr->used_space, MS_SYNC) < 0) {
- mailbox_list_index_set_syscall_error(ctx->index, "msync()");
- return -1;
+ if (msync(ctx->index->mmap_base,
+ hdr->used_space, MS_SYNC) < 0) {
+ mailbox_list_index_set_syscall_error(ctx->index,
+ "msync()");
+ return -1;
+ }
+ } else {
+ o_stream_seek(ctx->output, 0);
+ o_stream_send(ctx->output, &ctx->hdr, sizeof(ctx->hdr));
+
+ o_stream_destroy(&ctx->output);
+ buffer_free(ctx->output_buf);
}
return 0;
}
Index: mailbox-list-index.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mailbox-list-index.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mailbox-list-index.c 25 Nov 2006 22:17:40 -0000 1.1
+++ mailbox-list-index.c 10 Dec 2006 16:04:46 -0000 1.2
@@ -5,6 +5,7 @@
#include "crc32.h"
#include "ioloop.h"
#include "str.h"
+#include "file-cache.h"
#include "file-dotlock.h"
#include "mmap-util.h"
#include "write-full.h"
@@ -56,12 +57,16 @@
static void mailbox_list_index_unmap(struct mailbox_list_index *index)
{
+ if (index->file_cache != NULL)
+ file_cache_invalidate(index->file_cache, 0, (uoff_t)-1);
+
if (index->mmap_base != NULL) {
if (munmap(index->mmap_base, index->mmap_size) < 0)
mailbox_list_index_set_syscall_error(index, "munmap()");
index->mmap_base = NULL;
- index->mmap_size = 0;
}
+ index->const_mmap_base = NULL;
+ index->mmap_size = 0;
index->hdr = NULL;
}
@@ -70,6 +75,8 @@
{
mailbox_list_index_unmap(index);
+ if (index->file_cache != NULL)
+ file_cache_free(&index->file_cache);
if (index->fd != -1) {
if (close(index->fd) < 0)
mailbox_list_index_set_syscall_error(index, "close()");
@@ -123,24 +130,44 @@
int mailbox_list_index_map(struct mailbox_list_index *index)
{
const struct mailbox_list_index_header *hdr;
+ ssize_t ret;
mailbox_list_index_unmap(index);
- // FIXME: handle non-mmaps
- index->mmap_base = mmap_rw_file(index->fd, &index->mmap_size);
- if (index->mmap_base == MAP_FAILED) {
- index->mmap_base = NULL;
- return mailbox_list_index_set_syscall_error(index, "mmap()");
+ if (!index->mmap_disable) {
+ index->mmap_base = mmap_rw_file(index->fd, &index->mmap_size);
+ if (index->mmap_base == MAP_FAILED) {
+ index->mmap_base = NULL;
+ mailbox_list_index_set_syscall_error(index, "mmap()");
+ return -1;
+ }
+
+ index->const_mmap_base = index->mmap_base;
+ } else {
+ if (index->file_cache == NULL)
+ index->file_cache = file_cache_new(index->fd);
+
+ ret = file_cache_read(index->file_cache, 0, SSIZE_T_MAX);
+ if (ret < 0) {
+ mailbox_list_index_set_syscall_error(index,
+ "file_cache_read()");
+ return -1;
+ }
+ index->const_mmap_base = file_cache_get_map(index->file_cache,
+ &index->mmap_size);
}
if (index->mmap_size < sizeof(*hdr)) {
mailbox_list_index_set_corrupted(index, "File too small");
+ index->const_mmap_base = NULL;
return 0;
}
- hdr = index->mmap_base;
- if (mailbox_list_index_check_header(index, hdr) < 0)
+ hdr = index->const_mmap_base;
+ if (mailbox_list_index_check_header(index, hdr) < 0) {
+ index->const_mmap_base = NULL;
return 0;
+ }
index->hdr = hdr;
return 1;
@@ -278,6 +305,7 @@
index->separator = separator;
index->mail_index = mail_index;
index->fd = -1;
+ index->mmap_disable = mail_index->mmap_disable;
return index;
}
@@ -287,6 +315,7 @@
*_index = NULL;
+ mailbox_list_index_file_close(index);
i_free(index->filepath);
i_free(index);
}
@@ -314,7 +343,7 @@
return -1;
}
max_len = index->mmap_size - rec->name_offset;
- name = CONST_PTR_OFFSET(index->mmap_base, rec->name_offset);
+ name = CONST_PTR_OFFSET(index->const_mmap_base, rec->name_offset);
/* get name length. don't bother checking if it's not NUL-terminated,
because practically it always is even if the file is corrupted.
just make sure we don't crash if it happens. */
@@ -365,7 +394,7 @@
"dir_offset not 32bit aligned");
}
- dir = CONST_PTR_OFFSET(index->mmap_base, cur_offset);
+ dir = CONST_PTR_OFFSET(index->const_mmap_base, cur_offset);
next_offset = mail_index_offset_to_uint32(dir->next_offset);
if (next_offset != 0 && next_offset <= cur_offset) {
return mailbox_list_index_set_corrupted(index,
@@ -374,7 +403,7 @@
cur_offset = next_offset;
} while (cur_offset != 0);
- cur_offset = (const char *)dir - (const char *)index->mmap_base;
+ cur_offset = (const char *)dir - (const char *)index->const_mmap_base;
if (dir->count > INT_MAX/sizeof(struct mailbox_list_record) ||
dir->count * sizeof(struct mailbox_list_record) >
index->mmap_size - cur_offset) {
More information about the dovecot-cvs
mailing list