dovecot: Updating index file by overwriting changed parts didn't...
dovecot at dovecot.org
dovecot at dovecot.org
Wed Jun 20 01:56:47 EEST 2007
details: http://hg.dovecot.org/dovecot/rev/9493c7f1ebca
changeset: 5784:9493c7f1ebca
user: Timo Sirainen <tss at iki.fi>
date: Wed Jun 20 01:34:18 2007 +0300
description:
Updating index file by overwriting changed parts didn't work correctly.
diffstat:
3 files changed, 94 insertions(+), 28 deletions(-)
src/lib-index/mail-index-map.c | 5 +
src/lib-index/mail-index-private.h | 11 +++
src/lib-index/mail-index-write.c | 106 +++++++++++++++++++++++++++---------
diffs (214 lines):
diff -r 92bc0a7580f6 -r 9493c7f1ebca src/lib-index/mail-index-map.c
--- a/src/lib-index/mail-index-map.c Wed Jun 20 01:08:13 2007 +0300
+++ b/src/lib-index/mail-index-map.c Wed Jun 20 01:34:18 2007 +0300
@@ -697,8 +697,13 @@ static int mail_index_map_latest_file(st
return ret;
}
+ index->last_read_log_file_seq = new_map->hdr.log_file_seq;
+ index->last_read_log_file_head_offset =
+ new_map->hdr.log_file_head_offset;
index->last_read_log_file_tail_offset =
new_map->hdr.log_file_tail_offset;
+ index->last_read_stat = st;
+
mail_index_unmap(index, map);
*map = new_map;
return 1;
diff -r 92bc0a7580f6 -r 9493c7f1ebca src/lib-index/mail-index-private.h
--- a/src/lib-index/mail-index-private.h Wed Jun 20 01:08:13 2007 +0300
+++ b/src/lib-index/mail-index-private.h Wed Jun 20 01:34:18 2007 +0300
@@ -5,6 +5,8 @@
#include "mail-index.h"
#include "mail-index-view-private.h"
#include "mail-index-transaction-private.h"
+
+#include <sys/stat.h>
struct mail_transaction_header;
struct mail_transaction_log_view;
@@ -158,9 +160,14 @@ struct mail_index {
struct mail_index_map *map;
uint32_t indexid;
- /* last known log_file_tail_offset in main index file. used for
- optimizing main index updates. */
+ /* last_read_log_file_* contains the seq/offsets we last read from
+ the main index file's headers. these are used ro figure out when
+ the main index file should be updated, and if we can update it
+ by writing on top of it or if we need to recreate it. */
+ uint32_t last_read_log_file_seq;
+ uint32_t last_read_log_file_head_offset;
uint32_t last_read_log_file_tail_offset;
+ struct stat last_read_stat;
int lock_type, shared_lock_count, excl_lock_count;
unsigned int lock_id;
diff -r 92bc0a7580f6 -r 9493c7f1ebca src/lib-index/mail-index-write.c
--- a/src/lib-index/mail-index-write.c Wed Jun 20 01:08:13 2007 +0300
+++ b/src/lib-index/mail-index-write.c Wed Jun 20 01:34:18 2007 +0300
@@ -1,11 +1,16 @@
/* Copyright (C) 2003-2007 Timo Sirainen */
#include "lib.h"
+#include "read-full.h"
#include "write-full.h"
#include "mail-index-private.h"
#include "mail-transaction-log-private.h"
#include <stdio.h>
+
+#define MAIL_INDEX_MIN_UPDATE_SIZE 1024
+/* if we're updating >= count-n messages, recreate the index */
+#define MAIL_INDEX_MAX_OVERWRITE_NEG_SEQ_COUNT 10
static int mail_index_recreate(struct mail_index *index)
{
@@ -67,27 +72,6 @@ static int mail_index_write_map_over(str
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
-
- /* write records. */
- if (map->write_seq_first != 0) {
- size_t rec_offset =
- (map->write_seq_first-1) * map->hdr.record_size;
-
- if (pwrite_full(index->fd,
- CONST_PTR_OFFSET(map->records, rec_offset),
- (map->write_seq_last -
- map->write_seq_first + 1) *
- map->hdr.record_size,
- map->hdr.header_size + rec_offset) < 0)
- return -1;
- }
-
- /* write base header. it has changed practically always, so
- map->write_base_header might not be TRUE here in all situations.
- It's used only to figure out if we want to write the map at all. */
- base_size = I_MIN(map->hdr.base_header_size, sizeof(map->hdr));
- if (pwrite_full(index->fd, &map->hdr, base_size, 0) < 0)
- return -1;
/* write extended headers */
if (map->write_ext_header) {
@@ -98,7 +82,49 @@ static int mail_index_write_map_over(str
base_size) < 0)
return -1;
}
+
+ /* write records. */
+ if (map->write_seq_first != 0) {
+ size_t rec_offset =
+ (map->write_seq_first-1) * map->hdr.record_size;
+ size_t recs_size = map->hdr.record_size *
+ (map->write_seq_last - map->write_seq_first + 1);
+
+ if (pwrite_full(index->fd,
+ CONST_PTR_OFFSET(map->records, rec_offset),
+ recs_size,
+ map->hdr.header_size + rec_offset) < 0)
+ return -1;
+ }
+
+ /* Write base header last. If we happen to crash in above pwrites, it
+ doesn't matter because we haven't yet written log file offsets, so
+ all the changes will be re-applied and the header/data state will
+ stay valid.
+
+ The base header changes practically always, so
+ map->write_base_header might not be TRUE here in all situations.
+ It's used only to figure out if we want to write the map at all. */
+ base_size = I_MIN(map->hdr.base_header_size, sizeof(map->hdr));
+ if (pwrite_full(index->fd, &map->hdr, base_size, 0) < 0)
+ return -1;
return 0;
+}
+
+static bool mail_index_has_last_changed(struct mail_index *index)
+{
+ struct mail_index_header hdr;
+ int ret;
+
+ if ((ret = pread_full(index->fd, &hdr, sizeof(hdr), 0)) <= 0) {
+ if (ret < 0 && errno != ESTALE)
+ mail_index_set_syscall_error(index, "pread_full()");
+ return TRUE;
+ }
+
+ return hdr.log_file_head_offset !=
+ index->last_read_log_file_head_offset ||
+ hdr.log_file_seq != index->last_read_log_file_seq;
}
#define mail_index_map_has_changed(map) \
@@ -109,6 +135,7 @@ void mail_index_write(struct mail_index
{
struct mail_index_map *map = index->map;
const struct mail_index_header *hdr = &map->hdr;
+ struct stat st;
unsigned int lock_id;
if (!mail_index_map_has_changed(map))
@@ -118,15 +145,41 @@ void mail_index_write(struct mail_index
/* header size growed. we can't update this file anymore. */
map->write_atomic = TRUE;
}
- if (index->fd == -1) {
+ if (index->fd == -1 || index->last_read_log_file_seq == 0) {
/* index file doesn't exist, it's corrupted or we haven't
opened it for some reason */
map->write_atomic = TRUE;
}
+
+ if (index->last_read_stat.st_size < MAIL_INDEX_MIN_UPDATE_SIZE ||
+ (map->write_seq_last - map->write_seq_first + 1) +
+ MAIL_INDEX_MAX_OVERWRITE_NEG_SEQ_COUNT >= map->records_count) {
+ /* the file is so small that we don't even bother trying to
+ update it / changes are so large we might as well recreate */
+ map->write_atomic = TRUE;
+ }
+
+ if (!map->write_atomic) {
+ /* we can't update the file unless it's the same as it was
+ when we last read it. this is the first quick check before
+ locking. */
+ if (stat(index->filepath, &st) < 0) {
+ if (errno != ENOENT)
+ mail_index_set_syscall_error(index, "stat()");
+ map->write_atomic = TRUE;
+ } else if (st.st_ino != index->last_read_stat.st_ino ||
+ !CMP_ST_CTIME(&st, &index->last_read_stat))
+ map->write_atomic = TRUE;
+ }
+
if (!map->write_atomic) {
if (mail_index_try_lock_exclusive(index, &lock_id) <= 0) {
- /* locking failed, rewrite */
- map->write_atomic = TRUE;
+ /* locking failed, recreate */
+ map->write_atomic = TRUE;
+ } else if (mail_index_has_last_changed(index)) {
+ /* changed, we can't trust updating it anymore */
+ map->write_atomic = TRUE;
+ mail_index_unlock(index, lock_id);
}
}
@@ -139,13 +192,14 @@ void mail_index_write(struct mail_index
}
} else {
if (mail_index_write_map_over(index) < 0) {
- mail_index_set_error(index,
- "pwrite_full(%s) failed: %m", index->filepath);
+ mail_index_set_syscall_error(index, "pwrite_full()");
mail_index_set_inconsistent(index);
}
mail_index_unlock(index, lock_id);
}
+ index->last_read_log_file_seq = hdr->log_file_seq;
+ index->last_read_log_file_head_offset = hdr->log_file_head_offset;
index->last_read_log_file_tail_offset = hdr->log_file_tail_offset;
map->write_atomic = FALSE;
More information about the dovecot-cvs
mailing list