[dovecot-cvs] dovecot/src/lib-index mail-index-fsck.c, 1.28,
1.29 mail-index-sync.c, 1.64, 1.65 mail-index-view-sync.c,
1.46, 1.47 mail-index.c, 1.215,
1.216 mail-transaction-log-view.c, 1.42,
1.43 mail-transaction-log.c, 1.102,
1.103 mail-transaction-log.h, 1.27, 1.28
cras at dovecot.org
cras at dovecot.org
Sat Jan 7 00:31:44 EET 2006
- Previous message: [dovecot-cvs] dovecot/src/lib-index mail-index.c, 1.214,
1.215 mail-index-lock.c, 1.47, 1.48
- Next message: [dovecot-cvs] dovecot/src/lib-index mail-cache.c, 1.74,
1.75 mail-index-lock.c, 1.48, 1.49 mail-index-private.h, 1.60,
1.61 mail-index.c, 1.216, 1.217 mail-transaction-log-append.c,
1.15, 1.16 mail-transaction-log.c, 1.103,
1.104 mail-transaction-log.h, 1.28, 1.29
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv21517
Modified Files:
mail-index-fsck.c mail-index-sync.c mail-index-view-sync.c
mail-index.c mail-transaction-log-view.c
mail-transaction-log.c mail-transaction-log.h
Log Message:
Handle missing/broken transaction logs better. Handle broken sync position
in index header better.
Index: mail-index-fsck.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-fsck.c,v
retrieving revision 1.28
retrieving revision 1.29
diff -u -d -r1.28 -r1.29
--- mail-index-fsck.c 27 Aug 2005 12:31:02 -0000 1.28
+++ mail-index-fsck.c 6 Jan 2006 22:31:40 -0000 1.29
@@ -28,7 +28,8 @@
{
struct mail_index_header hdr;
const struct mail_index_record *rec;
- uint32_t i, last_uid;
+ uint32_t i, last_uid, log_seq;
+ uoff_t log_offset;
*error_r = NULL;
@@ -40,6 +41,26 @@
return 0;
}
+ mail_transaction_log_get_head(index->log, &log_seq, &log_offset);
+ if (hdr.log_file_int_offset > hdr.log_file_ext_offset) {
+ mail_index_fsck_error(index,
+ "log_file_int_offset > log_file_ext_offset");
+ hdr.log_file_int_offset = hdr.log_file_ext_offset;
+ }
+ if ((hdr.log_file_seq == log_seq &&
+ hdr.log_file_ext_offset > log_offset) ||
+ (hdr.log_file_seq != log_seq &&
+ !mail_transaction_log_is_head_prev(index->log,
+ hdr.log_file_seq,
+ hdr.log_file_ext_offset))) {
+ mail_index_fsck_error(index,
+ "log file sync pos %u,%u -> %u, %"PRIuUOFF_T,
+ hdr.log_file_seq, hdr.log_file_ext_offset,
+ log_seq, log_offset);
+ hdr.log_file_seq = log_seq;
+ hdr.log_file_int_offset = hdr.log_file_ext_offset = log_offset;
+ }
+
hdr.flags &= ~MAIL_INDEX_HDR_FLAG_FSCK;
hdr.messages_count = 0;
Index: mail-index-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync.c,v
retrieving revision 1.64
retrieving revision 1.65
diff -u -d -r1.64 -r1.65
--- mail-index-sync.c 6 Jan 2006 17:37:30 -0000 1.64
+++ mail-index-sync.c 6 Jan 2006 22:31:40 -0000 1.65
@@ -274,11 +274,37 @@
return 1;
}
-static int mail_index_sync_commit_external(struct mail_index_sync_ctx *ctx,
- uint32_t seq, uoff_t offset)
+static int
+mail_index_sync_set_log_view(struct mail_index_view *view,
+ uint32_t start_file_seq, uoff_t start_file_offset)
+{
+ uint32_t log_seq;
+ uoff_t log_offset;
+ int ret;
+
+ mail_transaction_log_get_head(view->index->log, &log_seq, &log_offset);
+
+ ret = mail_transaction_log_view_set(view->log_view,
+ start_file_seq, start_file_offset,
+ log_seq, log_offset,
+ MAIL_TRANSACTION_TYPE_MASK);
+ if (ret <= 0) {
+ /* either corrupted or the file was deleted for
+ some reason. either way, we can't go forward */
+ mail_index_set_error(view->index,
+ "Unexpected transaction log desync with index %s",
+ view->index->filepath);
+ mail_index_set_inconsistent(view->index);
+ return -1;
+ }
+ return 0;
+}
+
+static int mail_index_sync_commit_external(struct mail_index_sync_ctx *ctx)
{
int ret;
+ /* find the first external transaction, if there are any */
while ((ret = mail_transaction_log_view_next(ctx->view->log_view,
&ctx->hdr, &ctx->data,
NULL)) > 0) {
@@ -289,10 +315,14 @@
return -1;
if (ret > 0) {
- if (mail_transaction_log_view_set(ctx->view->log_view,
- ctx->index->hdr->log_file_seq,
- ctx->index->hdr->log_file_ext_offset,
- seq, offset, MAIL_TRANSACTION_TYPE_MASK) < 0)
+ uint32_t seq;
+ uoff_t offset;
+
+ /* found it. update log view's range to begin from it and
+ write all external transactions to index. */
+ mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
+ &seq, &offset);
+ if (mail_index_sync_set_log_view(ctx->view, seq, offset) < 0)
return -1;
if (mail_index_sync_update_index(ctx, TRUE) < 0)
return -1;
@@ -355,6 +385,21 @@
return 0;
}
+ if (index->hdr->log_file_int_offset > index->hdr->log_file_ext_offset ||
+ (index->hdr->log_file_seq == seq &&
+ index->hdr->log_file_ext_offset > offset) ||
+ (index->hdr->log_file_seq != seq &&
+ !mail_transaction_log_is_head_prev(index->log,
+ index->hdr->log_file_seq,
+ index->hdr->log_file_ext_offset))) {
+ /* broken sync positions. fix them. */
+ if (mail_index_fsck(index) <= 0) {
+ mail_index_unlock(index, lock_id);
+ mail_transaction_log_sync_unlock(index->log);
+ return -1;
+ }
+ }
+
ctx = i_new(struct mail_index_sync_ctx, 1);
ctx->index = index;
ctx->lock_id = lock_id;
@@ -367,22 +412,9 @@
ctx->trans = mail_index_transaction_begin(dummy_view, FALSE, TRUE);
mail_index_view_close(dummy_view);
- if (index->hdr->log_file_seq == seq &&
- index->hdr->log_file_ext_offset > offset) {
- /* synced offset is greater than what we have available.
- the log sequences have gotten messed up. */
- mail_transaction_log_file_set_corrupted(index->log->head,
- "log_file_ext_offset (%u) > log size (%"PRIuUOFF_T")",
- index->hdr->log_file_ext_offset, offset);
- mail_index_sync_rollback(ctx);
- return -1;
- }
-
- if (mail_transaction_log_view_set(ctx->view->log_view,
- index->hdr->log_file_seq,
- index->hdr->log_file_int_offset,
- seq, offset,
- MAIL_TRANSACTION_TYPE_MASK) < 0) {
+ if (mail_index_sync_set_log_view(ctx->view,
+ index->hdr->log_file_seq,
+ index->hdr->log_file_int_offset) < 0) {
mail_index_sync_rollback(ctx);
return -1;
}
@@ -400,7 +432,7 @@
yet. They need to be synced with the real mailbox first. */
if (seq != index->hdr->log_file_seq ||
offset != index->hdr->log_file_ext_offset) {
- if (mail_index_sync_commit_external(ctx, seq, offset) < 0) {
+ if (mail_index_sync_commit_external(ctx) < 0) {
mail_index_sync_rollback(ctx);
return -1;
}
@@ -408,14 +440,10 @@
mail_index_view_close(ctx->view);
ctx->view = mail_index_view_open(index);
- if (mail_transaction_log_view_set(ctx->view->log_view,
+ if (mail_index_sync_set_log_view(ctx->view,
index->hdr->log_file_seq,
- index->hdr->log_file_int_offset,
- seq, offset,
- MAIL_TRANSACTION_TYPE_MASK) < 0) {
- mail_index_sync_rollback(ctx);
+ index->hdr->log_file_int_offset) < 0)
return -1;
- }
}
/* we need to have all the transactions sorted to optimize
@@ -606,8 +634,8 @@
{
struct mail_index *index = ctx->index;
const struct mail_index_header *hdr;
- uint32_t seq, seq2;
- uoff_t offset, offset2;
+ uint32_t seq;
+ uoff_t offset;
int ret = 0;
if (mail_transaction_log_view_is_corrupted(ctx->view->log_view))
@@ -622,9 +650,10 @@
if (ret == 0 && (hdr->log_file_seq != seq ||
hdr->log_file_int_offset != offset ||
hdr->log_file_ext_offset != offset)) {
- if (mail_transaction_log_view_set(ctx->view->log_view,
- hdr->log_file_seq, hdr->log_file_int_offset,
- seq, offset, MAIL_TRANSACTION_TYPE_MASK) < 0)
+ /* write all pending changes to index. */
+ if (mail_index_sync_set_log_view(ctx->view,
+ hdr->log_file_seq,
+ hdr->log_file_int_offset) < 0)
ret = -1;
else if (mail_index_sync_update_index(ctx, FALSE) < 0)
ret = -1;
@@ -635,11 +664,8 @@
ret = -1;
else {
/* cache_offsets have changed, sync them */
- mail_transaction_log_get_head(index->log,
- &seq2, &offset2);
- if (mail_transaction_log_view_set(ctx->view->log_view,
- seq, offset, seq2, offset2,
- MAIL_TRANSACTION_TYPE_MASK) < 0)
+ if (mail_index_sync_set_log_view(ctx->view,
+ seq, offset) < 0)
ret = -1;
else if (mail_index_sync_update_index(ctx, FALSE) < 0)
ret = -1;
Index: mail-index-view-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-view-sync.c,v
retrieving revision 1.46
retrieving revision 1.47
diff -u -d -r1.46 -r1.47
--- mail-index-view-sync.c 6 Jan 2006 17:37:30 -0000 1.46
+++ mail-index-view-sync.c 6 Jan 2006 22:31:40 -0000 1.47
@@ -93,6 +93,31 @@
}
}
+static int view_sync_set_log_view_range(struct mail_index_view *view,
+ enum mail_transaction_type type_mask)
+{
+ const struct mail_index_header *hdr = view->index->hdr;
+ int ret;
+
+ ret = mail_transaction_log_view_set(view->log_view,
+ view->log_file_seq,
+ view->log_file_offset,
+ hdr->log_file_seq,
+ hdr->log_file_int_offset,
+ type_mask);
+ if (ret <= 0) {
+ if (ret == 0) {
+ /* FIXME: use the new index to get needed changes */
+ mail_index_set_error(view->index,
+ "Transaction log got desynced for index %s",
+ view->index->filepath);
+ mail_index_set_inconsistent(view->index);
+ }
+ return -1;
+ }
+ return 0;
+}
+
static int
view_sync_get_expunges(struct mail_index_view *view, array_t *expunges_r)
{
@@ -103,12 +128,7 @@
unsigned int count;
int ret;
- if (mail_transaction_log_view_set(view->log_view,
- view->log_file_seq,
- view->log_file_offset,
- view->index->hdr->log_file_seq,
- view->index->hdr->log_file_int_offset,
- MAIL_TRANSACTION_EXPUNGE) < 0)
+ if (view_sync_set_log_view_range(view, MAIL_TRANSACTION_EXPUNGE) < 0)
return -1;
ARRAY_CREATE(expunges_r, default_pool,
@@ -193,13 +213,7 @@
tell us if any visible changes were skipped. */
log_get_mask = visible_mask | (MAIL_TRANSACTION_TYPE_MASK ^
MAIL_TRANSACTION_VISIBLE_SYNC_MASK);
-
- if (mail_transaction_log_view_set(view->log_view,
- view->log_file_seq,
- view->log_file_offset,
- hdr->log_file_seq,
- hdr->log_file_int_offset,
- log_get_mask) < 0) {
+ if (view_sync_set_log_view_range(view, log_get_mask) < 0) {
if (array_is_created(&expunges))
array_free(&expunges);
return -1;
@@ -530,6 +544,7 @@
}
view->hdr = view->map->hdr;
+ /* set log view to empty range so unneeded memory gets freed */
(void)mail_transaction_log_view_set(view->log_view,
view->log_file_seq,
view->log_file_offset,
Index: mail-index.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.215
retrieving revision 1.216
diff -u -d -r1.215 -r1.216
--- mail-index.c 6 Jan 2006 21:16:43 -0000 1.215
+++ mail-index.c 6 Jan 2006 22:31:40 -0000 1.216
@@ -816,7 +816,8 @@
map_hdr->log_file_seq,
map_hdr->log_file_int_offset,
max_seq, max_offset,
- MAIL_TRANSACTION_TYPE_MASK) < 0) {
+ MAIL_TRANSACTION_TYPE_MASK) <= 0) {
+ /* can't use it. sync by re-reading index. */
mail_transaction_log_view_close(log_view);
return 0;
}
Index: mail-transaction-log-view.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log-view.c,v
retrieving revision 1.42
retrieving revision 1.43
diff -u -d -r1.42 -r1.43
--- mail-transaction-log-view.c 12 Oct 2005 17:14:32 -0000 1.42
+++ mail-transaction-log-view.c 6 Jan 2006 22:31:40 -0000 1.43
@@ -76,17 +76,19 @@
uint32_t max_file_seq, uoff_t max_file_offset,
enum mail_transaction_type type_mask)
{
- /* FIXME: error handling for "not found" case is bad.. should the
- caller after all check it and handle as it sees best..? */
struct mail_transaction_log_file *file, *first;
uint32_t seq;
uoff_t end_offset;
int ret;
+ i_assert(view->log != NULL);
i_assert(min_file_seq <= max_file_seq);
- if (view->log == NULL)
+ if (view->log == NULL) {
+ /* transaction log is closed already. this log view shouldn't
+ be used anymore. */
return -1;
+ }
if (min_file_seq == 0) {
/* new index, transaction file not synced yet */
@@ -114,15 +116,8 @@
/* find the oldest log file first. */
ret = mail_transaction_log_file_find(view->log, min_file_seq, &file);
- if (ret <= 0) {
- if (ret == 0) {
- mail_index_set_error(view->log->index,
- "Lost transaction log file %s seq %u",
- view->log->tail->filepath, min_file_seq);
- mail_index_set_inconsistent(view->log->index);
- }
- return -1;
- }
+ if (ret <= 0)
+ return ret;
if (min_file_offset == 0) {
/* this could happen if internal transactions haven't yet been
@@ -144,16 +139,8 @@
end_offset = min_file_seq == max_file_seq ?
max_file_offset : (uoff_t)-1;
ret = mail_transaction_log_file_map(file, min_file_offset, end_offset);
- if (ret <= 0) {
- if (ret == 0) {
- /* File is corrupted or stale NFS handle */
- mail_index_set_error(view->log->index,
- "Lost transaction log file %s seq %u",
- file->filepath, file->hdr.file_seq);
- mail_index_set_inconsistent(view->log->index);
- }
- return -1;
- }
+ if (ret <= 0)
+ return ret;
first = file;
for (seq = min_file_seq+1; seq <= max_file_seq; seq++) {
@@ -179,26 +166,15 @@
}
/* missing files in the middle */
- mail_index_set_error(view->log->index,
- "Lost transaction log file %s seq %u",
- view->log->tail->filepath, seq);
- mail_index_set_inconsistent(view->log->index);
- return -1;
+ return 0;
}
end_offset = file->hdr.file_seq == max_file_seq ?
max_file_offset : (uoff_t)-1;
ret = mail_transaction_log_file_map(file, file->hdr.hdr_size,
end_offset);
- if (ret == 0) {
- /* File is corrupted or stale NFS handle */
- mail_index_set_error(view->log->index,
- "Lost transaction log file %s seq %u",
- file->filepath, file->hdr.file_seq);
- mail_index_set_inconsistent(view->log->index);
- }
if (ret <= 0)
- return -1;
+ return ret;
}
i_assert(max_file_offset == (uoff_t)-1 ||
@@ -236,7 +212,7 @@
i_assert(view->cur_offset <= view->cur->sync_offset);
i_assert(view->cur->hdr.file_seq == min_file_seq);
- return 0;
+ return 1;
}
void
Index: mail-transaction-log.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.c,v
retrieving revision 1.102
retrieving revision 1.103
diff -u -d -r1.102 -r1.103
--- mail-transaction-log.c 27 Aug 2005 12:41:21 -0000 1.102
+++ mail-transaction-log.c 6 Jan 2006 22:31:40 -0000 1.103
@@ -1037,8 +1037,6 @@
if (errno == ESTALE) {
/* log file was deleted in NFS server, fail silently */
- buffer_set_used_size(file->buffer,
- offset - file->buffer_offset);
return 0;
}
@@ -1073,8 +1071,8 @@
}
/* with mmap_no_write we could alternatively just write to log with
- msync() rather than pwrite(). that'd cause slightly more disk I/O,
- so rather use more memory. */
+ msync() rather than pwrite(). but since there aren't many such OSes
+ left, it's easier to just use mmap_disable behavior with it */
use_mmap = !index->mmap_disable && !index->mmap_no_write;
if (file->buffer != NULL && file->buffer_offset <= start_offset) {
@@ -1084,24 +1082,29 @@
return 1;
}
- if (fstat(file->fd, &st) < 0) {
- mail_index_file_set_syscall_error(index, file->filepath,
- "fstat()");
- return -1;
- }
- if (start_offset > (uoff_t)st.st_size) {
- mail_transaction_log_file_set_corrupted(file,
- "start_offset (%"PRIuUOFF_T") > file size "
- "(%"PRIuUOFF_T")", start_offset, (uoff_t)st.st_size);
- return -1;
- }
-
- if (file->mmap_base != NULL && (uoff_t)st.st_size == file->mmap_size &&
- file->buffer_offset <= start_offset && end_offset == (uoff_t)-1) {
- /* it's all mmaped already */
- if (mail_transaction_log_file_sync(file) < 0)
+ if (use_mmap) {
+ if (fstat(file->fd, &st) < 0) {
+ mail_index_file_set_syscall_error(index, file->filepath,
+ "fstat()");
return -1;
- return 1;
+ }
+ if (start_offset > (uoff_t)st.st_size) {
+ mail_transaction_log_file_set_corrupted(file,
+ "start_offset (%"PRIuUOFF_T") > file size "
+ "(%"PRIuUOFF_T")", start_offset,
+ (uoff_t)st.st_size);
+ return -1;
+ }
+
+ if (file->mmap_base != NULL &&
+ (uoff_t)st.st_size == file->mmap_size &&
+ file->buffer_offset <= start_offset &&
+ end_offset == (uoff_t)-1) {
+ /* it's all mmaped already */
+ if (mail_transaction_log_file_sync(file) < 0)
+ return -1;
+ return 1;
+ }
}
if (file->buffer != NULL &&
@@ -1245,3 +1248,10 @@
*file_seq_r = log->head->hdr.file_seq;
*file_offset_r = log->head->sync_offset;
}
+
+int mail_transaction_log_is_head_prev(struct mail_transaction_log *log,
+ uint32_t file_seq, uoff_t file_offset)
+{
+ return log->head->hdr.prev_file_seq == file_seq &&
+ log->head->hdr.prev_file_offset == file_offset;
+}
Index: mail-transaction-log.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.h,v
retrieving revision 1.27
retrieving revision 1.28
diff -u -d -r1.27 -r1.28
--- mail-transaction-log.h 5 Mar 2005 19:23:28 -0000 1.27
+++ mail-transaction-log.h 6 Jan 2006 22:31:40 -0000 1.28
@@ -113,7 +113,7 @@
mail_transaction_log_view_open(struct mail_transaction_log *log);
void mail_transaction_log_view_close(struct mail_transaction_log_view *view);
-/* Set view boundaries. Returns -1 if error, 0 if ok. */
+/* Set view boundaries. Returns -1 if error, 0 if files are lost, 1 if ok. */
int
mail_transaction_log_view_set(struct mail_transaction_log_view *view,
uint32_t min_file_seq, uoff_t min_file_offset,
@@ -159,5 +159,8 @@
/* Returns the current head. Works only when log is locked. */
void mail_transaction_log_get_head(struct mail_transaction_log *log,
uint32_t *file_seq_r, uoff_t *file_offset_r);
+/* Returns TRUE if given seq/offset is current head log's rotate point. */
+int mail_transaction_log_is_head_prev(struct mail_transaction_log *log,
+ uint32_t file_seq, uoff_t file_offset);
#endif
- Previous message: [dovecot-cvs] dovecot/src/lib-index mail-index.c, 1.214,
1.215 mail-index-lock.c, 1.47, 1.48
- Next message: [dovecot-cvs] dovecot/src/lib-index mail-cache.c, 1.74,
1.75 mail-index-lock.c, 1.48, 1.49 mail-index-private.h, 1.60,
1.61 mail-index.c, 1.216, 1.217 mail-transaction-log-append.c,
1.15, 1.16 mail-transaction-log.c, 1.103,
1.104 mail-transaction-log.h, 1.28, 1.29
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
More information about the dovecot-cvs
mailing list