diff -r d5cae1f5fb6a src/lib-index/mail-transaction-log-append.c --- a/src/lib-index/mail-transaction-log-append.c Sun Mar 16 12:10:57 2008 +0200 +++ b/src/lib-index/mail-transaction-log-append.c Fri Mar 21 10:35:16 2008 +0200 @@ -3,6 +3,7 @@ #include "lib.h" #include "array.h" #include "buffer.h" +#include "crc32.h" #include "write-full.h" #include "mail-index-private.h" #include "mail-index-view-private.h" @@ -14,7 +15,6 @@ struct log_append_context { struct mail_index_transaction *trans; buffer_t *output; - uint32_t first_append_size; bool sync_includes_this; }; @@ -23,7 +23,6 @@ static void log_append_buffer(struct log enum mail_transaction_type type) { struct mail_transaction_header hdr; - uint32_t hdr_size; i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0); i_assert((buf->used % 4) == 0); @@ -39,17 +38,9 @@ static void log_append_buffer(struct log if ((ctx->trans->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0) hdr.type |= MAIL_TRANSACTION_EXTERNAL; - hdr_size = mail_index_uint32_to_offset(sizeof(hdr) + buf->used + - (hdr_buf == NULL ? 0 : - hdr_buf->used)); - if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(ctx->file) && - ctx->first_append_size == 0) { - /* size will be written later once everything - is in disk */ - ctx->first_append_size = hdr_size; - } else { - hdr.size = hdr_size; - } + hdr.size = sizeof(hdr) + buf->used + + (hdr_buf == NULL ? 0 : hdr_buf->used); + hdr.size = mail_index_uint32_to_offset(hdr.size); buffer_append(ctx->output, &hdr, sizeof(hdr)); if (hdr_buf != NULL) @@ -80,6 +71,33 @@ static int log_buffer_move_to_memory(str return 0; } +static void log_update_transaction_frame(struct log_append_context *ctx) +{ + struct mail_transaction_header hdr; + struct mail_transaction_frame frame; + const void *data; + unsigned int data_offset; + + /* update header */ + memset(&hdr, 0, sizeof(hdr)); + hdr.type = MAIL_TRANSACTION_FRAME; + if ((ctx->trans->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0) + hdr.type |= MAIL_TRANSACTION_EXTERNAL; + hdr.size = mail_index_uint32_to_offset(ctx->output->used); + buffer_write(ctx->output, 0, &hdr, sizeof(hdr)); + + /* calculate transaction's crc32 and update the frame */ + data_offset = sizeof(hdr) + sizeof(frame); + i_assert(data_offset < ctx->output->used); + data = CONST_PTR_OFFSET(ctx->output->data, data_offset); + + memset(&frame, 0, sizeof(frame)); + frame.crc32 = crc32_data(&hdr, sizeof(hdr)); + frame.crc32 = crc32_data_more(frame.crc32, data, + ctx->output->used - data_offset); + buffer_write(ctx->output, sizeof(hdr), &frame, sizeof(frame)); +} + static int log_buffer_write(struct log_append_context *ctx, bool want_fsync) { struct mail_transaction_log_file *file = ctx->file; @@ -90,7 +108,8 @@ static int log_buffer_write(struct log_a return 0; } - i_assert(ctx->first_append_size != 0); + log_update_transaction_frame(ctx); + if (pwrite_full(file->fd, ctx->output->data, ctx->output->used, file->sync_offset) < 0) { /* write failure, fallback to in-memory indexes. */ @@ -103,16 +122,6 @@ static int log_buffer_write(struct log_a i_assert(!ctx->sync_includes_this || file->sync_offset + ctx->output->used == file->max_tail_offset); - - /* now that the whole transaction has been written, rewrite the first - record's size so the transaction becomes visible */ - if (pwrite_full(file->fd, &ctx->first_append_size, - sizeof(uint32_t), file->sync_offset) < 0) { - mail_index_file_set_syscall_error(file->log->index, - file->filepath, - "pwrite_full()"); - return log_buffer_move_to_memory(ctx); - } if ((want_fsync && !file->log->index->fsync_disable) || file->log->index->nfs_flush) { @@ -508,6 +517,8 @@ mail_transaction_log_append_locked(struc ctx.file = file; ctx.trans = t; ctx.output = buffer_create_dynamic(default_pool, 1024); + buffer_append_zero(ctx.output, sizeof(struct mail_transaction_header) + + sizeof(struct mail_transaction_frame)); /* send all extension introductions and resizes before appends to avoid resize overhead as much as possible */ diff -r d5cae1f5fb6a src/lib-index/mail-transaction-log-file.c --- a/src/lib-index/mail-transaction-log-file.c Sun Mar 16 12:10:57 2008 +0200 +++ b/src/lib-index/mail-transaction-log-file.c Fri Mar 21 10:35:16 2008 +0200 @@ -3,6 +3,7 @@ #include "lib.h" #include "ioloop.h" #include "buffer.h" +#include "crc32.h" #include "file-dotlock.h" #include "nfs-workarounds.h" #include "read-full.h" @@ -725,11 +726,69 @@ log_file_track_mailbox_sync_offset(struc } static int +log_file_check_transaction(struct mail_transaction_log_file *file, + const struct mail_transaction_header *hdr, + uint32_t trans_size) +{ + const struct mail_transaction_frame *frame = (const void *)(hdr + 1); + const struct mail_transaction_header *subhdr; + uint32_t crc32, offset, sub_size; + + if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) != + MAIL_TRANSACTION_FRAME) { + /* for backwards compatibility */ + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) { + if (log_file_track_mailbox_sync_offset(file, hdr, + trans_size) < 0) + return -1; + } + return 0; + } + + if (trans_size <= sizeof(*hdr) + sizeof(*frame)) { + mail_transaction_log_file_set_corrupted(file, + "Transaction size too small"); + return -1; + } + + offset = sizeof(*hdr) + sizeof(*frame); + crc32 = crc32_data(hdr, sizeof(*hdr)); + crc32 = crc32_data_more(crc32, frame + 1, trans_size - offset); + if (frame->crc32 != crc32) { + mail_transaction_log_file_set_corrupted(file, + "Transaction CRC32 wrong"); + return -1; + } + + while (offset < trans_size) { + subhdr = CONST_PTR_OFFSET(hdr, offset); + sub_size = mail_index_offset_to_uint32(subhdr->size); + if (sub_size == 0) { + mail_transaction_log_file_set_corrupted(file, + "Broken transaction internal record size"); + return -1; + } + if (offset + sub_size > trans_size) { + mail_transaction_log_file_set_corrupted(file, + "Transaction internal record size points " + "outside transaction"); + return -1; + } + if ((subhdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) { + if (log_file_track_mailbox_sync_offset(file, subhdr, + sub_size) < 0) + return -1; + } + offset += sub_size; + } + return 0; +} + +static int mail_transaction_log_file_sync(struct mail_transaction_log_file *file) { const struct mail_transaction_header *hdr; const void *data; - struct stat st; size_t size, avail; uint32_t trans_size = 0; @@ -744,7 +803,7 @@ mail_transaction_log_file_sync(struct ma trans_size = mail_index_offset_to_uint32(hdr->size); if (trans_size == 0) { /* unfinished */ - return 1; + return 0; } if (trans_size < sizeof(*hdr)) { mail_transaction_log_file_set_corrupted(file, @@ -756,49 +815,18 @@ mail_transaction_log_file_sync(struct ma break; /* transaction has been fully written */ - if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) { - if (log_file_track_mailbox_sync_offset(file, hdr, - trans_size) < 0) - return -1; - } + if (log_file_check_transaction(file, hdr, trans_size) < 0) + return -1; file->sync_offset += trans_size; trans_size = 0; } - if (file->mmap_base != NULL && !file->locked) { - /* Now that all the mmaped pages have page faulted, check if - the file had changed while doing that. Only after the last - page has faulted, the size returned by fstat() can be - trusted. Otherwise it might point to a page boundary while - the next page is still being written. - - Without this check we might see partial transactions, - sometimes causing "Extension record updated without intro - prefix" errors. */ - if (fstat(file->fd, &st) < 0) { - mail_index_file_set_syscall_error(file->log->index, - file->filepath, - "fstat()"); - return -1; - } - if ((uoff_t)st.st_size != file->last_size) { - file->last_size = st.st_size; - return 0; - } - } - avail = file->sync_offset - file->buffer_offset; if (avail != size) { - /* There's more data than we could sync at the moment. If the - last record's size wasn't valid, we can't know if it will - be updated unless we've locked the log. */ - if (trans_size != 0) { - /* pread()s or the above fstat() check for mmaps should - have guaranteed that this doesn't happen */ - mail_transaction_log_file_set_corrupted(file, - "hdr.size too large (%u)", trans_size); - return -1; - } else if (file->locked) { + /* There's more data than we could sync at the moment. + We can't know if the rest will be added later, unless we've + locked the log. */ + if (file->locked) { mail_transaction_log_file_set_corrupted(file, "Unexpected garbage at EOF"); return -1; @@ -819,7 +847,7 @@ mail_transaction_log_file_sync(struct ma return -1; } - return 1; + return 0; } static int @@ -958,10 +986,8 @@ mail_transaction_log_file_read(struct ma return mail_transaction_log_file_read(file, start_offset, TRUE); } - if ((ret = mail_transaction_log_file_sync(file)) <= 0) { - i_assert(ret != 0); /* happens only with mmap */ + if (mail_transaction_log_file_sync(file) < 0) return -1; - } i_assert(file->sync_offset >= file->buffer_offset); buffer_set_used_size(file->buffer, @@ -1045,7 +1071,6 @@ mail_transaction_log_file_map_mmap(struc uoff_t start_offset) { struct stat st; - int ret; /* we are going to mmap() this file, but it's not necessarily mmaped currently. */ @@ -1067,30 +1092,20 @@ mail_transaction_log_file_map_mmap(struc if ((uoff_t)st.st_size == file->mmap_size) { /* we already have the whole file mmaped */ - if ((ret = mail_transaction_log_file_sync(file)) < 0) - return 0; - if (ret > 0) - return 1; - /* size changed, re-mmap */ + return mail_transaction_log_file_sync(file) < 0 ? -1 : 1; } - do { - mail_transaction_log_file_munmap(file); + mail_transaction_log_file_munmap(file); - if (file->last_size - start_offset < mmap_get_page_size()) { - /* just reading the file is probably faster */ - return mail_transaction_log_file_read(file, - start_offset, - FALSE); - } + if (file->last_size - start_offset < mmap_get_page_size()) { + /* just reading the file is probably faster */ + return mail_transaction_log_file_read(file, start_offset, + FALSE); + } - if (mail_transaction_log_file_mmap(file) < 0) - return -1; - if ((ret = mail_transaction_log_file_sync(file)) < 0) - return 0; - } while (ret == 0); - - return 1; + if (mail_transaction_log_file_mmap(file) < 0) + return -1; + return mail_transaction_log_file_sync(file) < 0 ? -1 : 1; } int mail_transaction_log_file_map(struct mail_transaction_log_file *file, diff -r d5cae1f5fb6a src/lib-index/mail-transaction-log.h --- a/src/lib-index/mail-transaction-log.h Sun Mar 16 12:10:57 2008 +0200 +++ b/src/lib-index/mail-transaction-log.h Fri Mar 21 10:35:16 2008 +0200 @@ -28,6 +28,7 @@ enum mail_transaction_type { MAIL_TRANSACTION_EXT_REC_UPDATE = 0x00000200, MAIL_TRANSACTION_KEYWORD_UPDATE = 0x00000400, MAIL_TRANSACTION_KEYWORD_RESET = 0x00000800, + MAIL_TRANSACTION_FRAME = 0x00001000, MAIL_TRANSACTION_TYPE_MASK = 0x0000ffff, @@ -48,6 +49,13 @@ struct mail_transaction_header { struct mail_transaction_header { uint32_t size; uint32_t type; /* enum mail_transaction_type */ +}; + +/* Transaction frame's size extends through all the records within this + transaction. crc32 is the CRC32 of the entire transaction, including the + frame's header (but excluding the crc32 field itself, of course). */ +struct mail_transaction_frame { + uint32_t crc32; }; struct mail_transaction_expunge {