[dovecot-cvs] dovecot/src/lib-index mail-index-lock.c, 1.31,
1.32 mail-index-private.h, 1.27, 1.28 mail-index-sync-update.c,
1.41, 1.42 mail-index-sync.c, 1.30,
1.31 mail-index-transaction-private.h, 1.10,
1.11 mail-index-transaction-view.c, 1.3,
1.4 mail-index-transaction.c, 1.20,
1.21 mail-index-view-private.h, 1.8,
1.9 mail-index-view-sync.c, 1.19, 1.20 mail-index-view.c, 1.19,
1.20 mail-index.c, 1.144, 1.145 mail-index.h, 1.126,
1.127 mail-transaction-log-view.c, 1.24,
1.25 mail-transaction-log.c, 1.53, 1.54 mail-transaction-log.h,
1.16, 1.17 mail-transaction-util.c, 1.16,
1.17 mail-transaction-util.h, 1.8, 1.9
cras at dovecot.org
cras at dovecot.org
Sun Sep 5 20:53:48 EEST 2004
Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv8010/lib-index
Modified Files:
mail-index-lock.c mail-index-private.h
mail-index-sync-update.c mail-index-sync.c
mail-index-transaction-private.h mail-index-transaction-view.c
mail-index-transaction.c mail-index-view-private.h
mail-index-view-sync.c mail-index-view.c mail-index.c
mail-index.h mail-transaction-log-view.c
mail-transaction-log.c mail-transaction-log.h
mail-transaction-util.c mail-transaction-util.h
Log Message:
Save extra record/header infos into index file permanently.
Index: mail-index-lock.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-lock.c,v
retrieving revision 1.31
retrieving revision 1.32
diff -u -d -r1.31 -r1.32
--- mail-index-lock.c 3 Sep 2004 20:59:26 -0000 1.31
+++ mail-index-lock.c 5 Sep 2004 17:53:45 -0000 1.32
@@ -19,6 +19,7 @@
*/
#include "lib.h"
+#include "buffer.h"
#include "mmap-util.h"
#include "file-lock.h"
#include "write-full.h"
@@ -193,7 +194,26 @@
}
}
- ret = write_full(fd, index->map->hdr, sizeof(*index->map->hdr));
+ if (index->map->hdr->base_header_size >= sizeof(*index->map->hdr)) {
+ /* header size is what's expected */
+ ret = write_full(fd, index->map->hdr,
+ index->map->hdr->header_size);
+ } else {
+ /* write base header */
+ ret = write_full(fd, index->map->hdr,
+ index->map->hdr->base_header_size);
+ if (ret == 0) {
+ /* write extended headers */
+ const struct mail_index_header *hdr;
+ const void *hdr_ext;
+
+ hdr = index->map->hdr_base;
+ hdr_ext = CONST_PTR_OFFSET(hdr, hdr->base_header_size);
+ ret = write_full(fd, hdr_ext, hdr->header_size -
+ hdr->base_header_size);
+ }
+ }
+
if (ret < 0 || write_full(fd, index->map->records,
index->map->records_count *
index->map->hdr->record_size) < 0) {
@@ -264,7 +284,7 @@
/* if header size is smaller than what we have, we'll have to recreate
the index to grow it. so don't even try regular locking. */
- if (index->map->hdr != &index->map->hdr_copy &&
+ if (index->map->hdr != index->map->hdr_copy_buf->data &&
index->map->base_header_size == sizeof(*index->hdr)) {
/* wait two seconds for exclusive lock */
ret = mail_index_lock(index, F_WRLCK, 2, TRUE, lock_id_r);
Index: mail-index-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-private.h,v
retrieving revision 1.27
retrieving revision 1.28
diff -u -d -r1.27 -r1.28
--- mail-index-private.h 31 Jul 2004 03:06:48 -0000 1.27
+++ mail-index-private.h 5 Sep 2004 17:53:45 -0000 1.28
@@ -6,11 +6,6 @@
struct mail_transaction_header;
-/* Maximum number of extra record data items we allowed. Raising this limit
- only means it takes a few bytes more memory, but 32 should be enough for a
- long time, right? :) */
-#define MAIL_INDEX_MAX_EXTRA_RECORDS 32
-
/* Index file is grown exponentially when we're adding less than this many
records. */
#define MAIL_INDEX_MAX_POWER_GROW (1024*1024 / sizeof(struct mail_index_record))
@@ -28,19 +23,29 @@
struct mail_index_extra_record_info {
const char *name;
- uint16_t offset;
- uint16_t size;
+ uint32_t hdr_offset;
+ uint32_t hdr_size;
+ uint32_t record_offset;
+ uint32_t record_size;
+};
+
+struct mail_index_extra_record_info_header {
+ uint32_t hdr_size;
+ uint32_t record_offset;
+ uint32_t record_size;
};
struct mail_index_map {
int refcount;
const struct mail_index_header *hdr;
+ const void *hdr_base;
void *records; /* struct mail_index_record[] */
unsigned int records_count;
- struct mail_index_extra_record_info *
- extra_record_map[MAIL_INDEX_MAX_EXTRA_RECORDS];
+ pool_t extra_records_pool;
+ buffer_t *extra_infos; /* struct mail_index_extra_record_info[] */
+ buffer_t *extra_infos_id_map; /* uint32_t[] (index -> file) */
void *mmap_base;
size_t mmap_size, mmap_used_size;
@@ -50,7 +55,7 @@
uint32_t log_file_seq;
uoff_t log_file_offset;
- struct mail_index_header hdr_copy;
+ buffer_t *hdr_copy_buf;
uint32_t base_header_size; /* so we don't need lock to access it */
unsigned int write_to_disk:1;
@@ -65,11 +70,8 @@
mode_t mode;
gid_t gid;
- pool_t extra_records_pool;
- buffer_t *extra_records_buf;
- const struct mail_index_extra_record_info *extra_records;
- unsigned int extra_records_count;
- unsigned int max_record_size;
+ pool_t extra_infos_pool;
+ buffer_t *extra_infos; /* struct mail_index_extra_record_info[] */
char *filepath;
int fd;
@@ -126,6 +128,14 @@
void mail_index_unmap(struct mail_index *index, struct mail_index_map *map);
struct mail_index_map *
mail_index_map_to_memory(struct mail_index_map *map, uint32_t new_record_size);
+uint32_t mail_index_map_register_extra_info(struct mail_index *index,
+ struct mail_index_map *map,
+ const char *name,
+ uint32_t hdr_offset,
+ uint32_t hdr_size,
+ uint32_t record_size);
+int mail_index_map_get_extra_info_idx(struct mail_index_map *map,
+ uint32_t data_id, uint32_t *idx_r);
int mail_index_lookup_full(struct mail_index_view *view, uint32_t seq,
struct mail_index_map **map_r,
Index: mail-index-sync-update.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync-update.c,v
retrieving revision 1.41
retrieving revision 1.42
diff -u -d -r1.41 -r1.42
--- mail-index-sync-update.c 3 Sep 2004 20:19:18 -0000 1.41
+++ mail-index-sync-update.c 5 Sep 2004 17:53:45 -0000 1.42
@@ -13,6 +13,8 @@
#include <time.h>
+static const unsigned char *null4[] = { 0, 0, 0, 0 };
+
static void
mail_index_header_update_counts(struct mail_index_header *hdr,
uint8_t old_flags, uint8_t new_flags)
@@ -77,7 +79,7 @@
struct mail_index_sync_map_ctx *ctx = context;
struct mail_index_view *view = ctx->view;
struct mail_index_map *map = view->map;
- struct mail_index_header *hdr = &map->hdr_copy;
+ struct mail_index_header *hdr;
struct mail_index_record *rec;
uint32_t count, seq, seq1, seq2;
@@ -90,6 +92,7 @@
if (seq1 == 0)
return 1;
+ hdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL);
for (seq = seq1; seq <= seq2; seq++) {
rec = MAIL_INDEX_MAP_IDX(map, seq-1);
mail_index_header_update_counts(hdr, rec->flags, 0);
@@ -115,20 +118,19 @@
return 1;
}
-static int sync_append(const struct mail_transaction_append_header *hdr,
- const struct mail_index_record *rec, void *context)
+static int sync_append(const struct mail_index_record *rec, void *context)
{
struct mail_index_sync_map_ctx *ctx = context;
struct mail_index_view *view = ctx->view;
struct mail_index_map *map = view->map;
+ struct mail_index_header *hdr;
void *dest;
- i_assert(hdr->record_size <= map->hdr->record_size);
-
- if (rec->uid < map->hdr_copy.next_uid) {
+ hdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL);
+ if (rec->uid < hdr->next_uid) {
mail_transaction_log_view_set_corrupted(view->log_view,
"Append with UID %u, but next_uid = %u",
- rec->uid, map->hdr_copy.next_uid);
+ rec->uid, hdr->next_uid);
return -1;
}
@@ -143,17 +145,17 @@
map->mmap_size);
dest = MAIL_INDEX_MAP_IDX(map, map->records_count);
}
- memcpy(dest, rec, hdr->record_size);
- memset(PTR_OFFSET(dest, hdr->record_size), 0,
- map->hdr->record_size - hdr->record_size);
+ memcpy(dest, rec, sizeof(*rec));
+ memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
+ map->hdr->record_size - sizeof(*rec));
- map->hdr_copy.messages_count++;
- map->hdr_copy.next_uid = rec->uid+1;
+ hdr->messages_count++;
+ hdr->next_uid = rec->uid+1;
view->messages_count++;
map->records_count++;
- mail_index_header_update_counts(&map->hdr_copy, 0, rec->flags);
- mail_index_header_update_lowwaters(&map->hdr_copy, rec);
+ mail_index_header_update_counts(hdr, 0, rec->flags);
+ mail_index_header_update_lowwaters(hdr, rec);
return 1;
}
@@ -162,8 +164,8 @@
{
struct mail_index_sync_map_ctx *ctx = context;
struct mail_index_view *view = ctx->view;
- struct mail_index_record *rec;
struct mail_index_header *hdr;
+ struct mail_index_record *rec;
uint8_t flag_mask, old_flags;
keywords_mask_t keyword_mask;
uint32_t i, idx, seq1, seq2;
@@ -176,7 +178,7 @@
if (seq1 == 0)
return 1;
- hdr = &view->map->hdr_copy;
+ hdr = buffer_get_modifyable_data(view->map->hdr_copy_buf, NULL);
if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
hdr->flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
@@ -213,9 +215,11 @@
{
struct mail_index_sync_map_ctx *ctx = context;
struct mail_index_view *view = ctx->view;
+ struct mail_index_header *hdr;
uint32_t i;
- view->map->hdr_copy.cache_file_seq = u->new_file_seq;
+ hdr = buffer_get_modifyable_data(view->map->hdr_copy_buf, NULL);
+ hdr->cache_file_seq = u->new_file_seq;
for (i = 0; i < view->messages_count; i++)
MAIL_INDEX_MAP_IDX(view->map, i)->cache_offset = 0;
@@ -259,11 +263,129 @@
static int sync_header_update(const struct mail_transaction_header_update *u,
void *context)
{
+ struct mail_index_sync_map_ctx *ctx = context;
+ const struct mail_index_header *hdr = ctx->view->map->hdr;
+
+ if (u->offset >= hdr->base_header_size ||
+ u->offset + u->size > hdr->base_header_size) {
+ mail_transaction_log_view_set_corrupted(ctx->view->log_view,
+ "Header update outside range: %u + %u > %u",
+ u->offset, u->size, hdr->base_header_size);
+ return -1;
+ }
+
+ buffer_write(ctx->view->map->hdr_copy_buf, u->offset, u + 1, u->size);
+ ctx->view->map->hdr = ctx->view->map->hdr_copy_buf->data;
+ return 1;
+}
+
+static int sync_extra_intro(const struct mail_transaction_extra_intro *u,
+ void *context)
+{
+ struct mail_index_sync_map_ctx *ctx = context;
+ struct mail_index_extra_record_info_header einfo_hdr;
+ const struct mail_index_extra_record_info *einfo;
+ struct mail_index_header *hdr;
+ const char *name;
+ buffer_t *hdr_buf;
+ uint32_t data_id;
+
+ t_push();
+ name = t_strndup(u + 1, u->name_size);
+
+ hdr_buf = ctx->view->map->hdr_copy_buf;
+ data_id = mail_index_map_register_extra_info(ctx->view->index,
+ ctx->view->map, name,
+ hdr_buf->used, u->hdr_size,
+ u->record_size);
+
+ einfo = ctx->view->index->extra_infos->data;
+ einfo += data_id;
+
+ /* name NUL [padding] einfo_hdr [header data] */
+ buffer_append(hdr_buf, name, strlen(name)+1);
+ if ((hdr_buf->used % 4) != 0)
+ buffer_append(hdr_buf, null4, 4 - (hdr_buf->used % 4));
+
+ memset(&einfo_hdr, 0, sizeof(einfo_hdr));
+ einfo_hdr.hdr_size = einfo->hdr_size;
+ einfo_hdr.record_offset = einfo->record_offset;
+ einfo_hdr.record_size = einfo->record_size;
+ buffer_append(hdr_buf, &einfo_hdr, sizeof(einfo_hdr));
+ buffer_set_used_size(hdr_buf, hdr_buf->used + einfo->hdr_size);
+
+ hdr = buffer_get_modifyable_data(hdr_buf, NULL);
+ hdr->header_size = hdr_buf->used;
+
+ ctx->view->map->hdr = hdr;
+
+ t_pop();
+
+ if (data_id != u->data_id) {
+ mail_transaction_log_view_set_corrupted(ctx->view->log_view,
+ "Introduced extra with invalid data id: %u != %u",
+ u->data_id, data_id);
+ return -1;
+ }
+ return 1;
+}
+
+static int sync_extra_reset(const struct mail_transaction_extra_rec_header *u,
+ void *context)
+{
struct mail_index_sync_map_ctx *ctx = context;
- void *data;
+ struct mail_index_view *view = ctx->view;
+ struct mail_index_map *map = view->map;
+ const struct mail_index_extra_record_info *einfo;
+ struct mail_index_record *rec;
+ uint32_t i;
- data = PTR_OFFSET(&ctx->view->map->hdr_copy, u->offset);
- memcpy(data, u + 1, u->size);
+ if (map->extra_infos == NULL ||
+ u->data_id >= map->extra_infos->used / sizeof(*einfo)) {
+ mail_transaction_log_view_set_corrupted(view->log_view,
+ "Extra reset for unknown data id %u",
+ u->data_id);
+ return -1;
+ }
+
+ einfo = map->extra_infos->data;
+ einfo += u->data_id;
+
+ memset(buffer_get_space_unsafe(map->hdr_copy_buf,
+ einfo->hdr_offset, einfo->hdr_size),
+ 0, einfo->hdr_size);
+ map->hdr = map->hdr_copy_buf->data;
+
+ for (i = 0; i < view->messages_count; i++) {
+ rec = MAIL_INDEX_MAP_IDX(view->map, i);
+ memset(PTR_OFFSET(rec, einfo->record_offset), 0,
+ einfo->record_size);
+ }
+ return 1;
+}
+
+static int
+sync_extra_hdr_update(const struct mail_transaction_extra_hdr_update *u,
+ void *context)
+{
+ struct mail_index_sync_map_ctx *ctx = context;
+ struct mail_index_map *map = ctx->view->map;
+ const struct mail_index_extra_record_info *einfo;
+
+ if (map->extra_infos == NULL ||
+ u->data_id >= map->extra_infos->used / sizeof(*einfo)) {
+ mail_transaction_log_view_set_corrupted(ctx->view->log_view,
+ "Extra header update for unknown data id %u",
+ u->data_id);
+ return -1;
+ }
+
+ einfo = map->extra_infos->data;
+ einfo += u->data_id;
+
+ buffer_write(map->hdr_copy_buf, einfo->hdr_offset + u->offset,
+ u + 1, u->size);
+ map->hdr = map->hdr_copy_buf->data;
return 1;
}
@@ -275,20 +397,27 @@
struct mail_index_sync_map_ctx *ctx = context;
struct mail_index_view *view = ctx->view;
struct mail_index_record *rec;
+ const struct mail_index_extra_record_info *einfo;
uint32_t seq;
- uint16_t offset, size;
- /* FIXME: do data_id mapping conversion */
+ if (view->map->extra_infos == NULL ||
+ hdr->data_id >= view->map->extra_infos->used / sizeof(*einfo)) {
+ mail_transaction_log_view_set_corrupted(view->log_view,
+ "Extra record update for unknown data id %u",
+ hdr->data_id);
+ return -1;
+ }
if (mail_index_lookup_uid_range(view, u->uid, u->uid, &seq, &seq) < 0)
return -1;
if (seq != 0) {
- offset = view->index->extra_records[hdr->data_id].offset;
- size = view->index->extra_records[hdr->data_id].size;
+ einfo = view->map->extra_infos->data;
+ einfo += hdr->data_id;
rec = MAIL_INDEX_MAP_IDX(view->map, seq-1);
- memcpy(PTR_OFFSET(rec, offset), u + 1, size);
+ memcpy(PTR_OFFSET(rec, einfo->record_offset),
+ u + 1, einfo->record_size);
}
return 1;
}
@@ -296,8 +425,8 @@
static int mail_index_grow(struct mail_index *index, struct mail_index_map *map,
unsigned int count)
{
- struct mail_index_header hdr;
- size_t size;
+ void *hdr_copy;
+ size_t size, hdr_copy_size;
if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
return 0;
@@ -324,17 +453,24 @@
/* we only wish to grow the file, but mail_index_map() updates the
headers as well and may break our modified hdr_copy. so, take
a backup of it and put it back afterwards */
- hdr = map->hdr_copy;
+ t_push();
+ hdr_copy_size = map->hdr_copy_buf->used;
+ hdr_copy = t_malloc(hdr_copy_size);
+ memcpy(hdr_copy, map->hdr_copy_buf->data, hdr_copy_size);
- if (mail_index_map(index, TRUE) <= 0)
+ if (mail_index_map(index, TRUE) <= 0) {
+ t_pop();
return -1;
+ }
map = index->map;
- map->hdr_copy = hdr;
- map->hdr = &map->hdr_copy;
+ buffer_reset(map->hdr_copy_buf);
+ buffer_append(map->hdr_copy_buf, hdr_copy, hdr_copy_size);
+ map->hdr = map->hdr_copy_buf->data;
map->records_count = map->hdr->messages_count;
i_assert(map->mmap_size >= size);
+ t_pop();
return 0;
}
@@ -393,7 +529,8 @@
struct mail_index_view *view = sync_ctx->view;
struct mail_index_map *map;
struct mail_index_sync_map_ctx sync_map_ctx;
- const struct mail_transaction_header *hdr;
+ const struct mail_transaction_header *thdr;
+ struct mail_index_header *tmphdr;
const void *data;
unsigned int count, old_lock_id;
uint32_t seq, i, first_append_uid;
@@ -402,7 +539,7 @@
memset(&sync_map_ctx, 0, sizeof(sync_map_ctx));
sync_map_ctx.view = view;
- sync_map_ctx.update_cache = TRUE;
+ sync_map_ctx.update_cache = TRUE;
/* we'll have to update view->lock_id to avoid mail_index_view_lock()
trying to update the file later. */
@@ -411,27 +548,32 @@
return -1;
mail_index_unlock(index, old_lock_id);
- /* NOTE: locking may change index->map so make sure assignment
+ /* NOTE: locking may change index->map so make sure the assignment is
after locking */
map = index->map;
if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
map->write_to_disk = TRUE;
- map->hdr_copy = *map->hdr;
- map->hdr = &map->hdr_copy;
+ if (map->hdr != map->hdr_copy_buf->data) {
+ buffer_reset(map->hdr_copy_buf);
+ buffer_append(map->hdr_copy_buf, map->hdr,
+ map->hdr->header_size);
+ map->hdr = map->hdr_copy_buf->data;
+ }
mail_index_unmap(index, view->map);
view->map = map;
view->map->refcount++;
- first_append_uid = 0;
- had_dirty = (map->hdr_copy.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
+ tmphdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL);
+ had_dirty = (tmphdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
if (had_dirty)
- map->hdr_copy.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
+ tmphdr->flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
- while ((ret = mail_transaction_log_view_next(view->log_view, &hdr,
+ first_append_uid = 0;
+ while ((ret = mail_transaction_log_view_next(view->log_view, &thdr,
&data, &skipped)) > 0) {
- if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0 &&
+ if ((thdr->type & MAIL_TRANSACTION_EXPUNGE) != 0 &&
!map->write_to_disk) {
/* expunges have to be atomic. so we'll have to copy
the mapping, do the changes there and then finally
@@ -443,23 +585,13 @@
mail_index_sync_replace_map(view, map);
}
- if ((hdr->type & MAIL_TRANSACTION_APPEND) != 0) {
- const struct mail_transaction_append_header *append_hdr;
- const struct mail_index_record *rec;
+ if ((thdr->type & MAIL_TRANSACTION_APPEND) != 0) {
+ const struct mail_index_record *rec = data;
- rec = CONST_PTR_OFFSET(data, sizeof(*append_hdr));
if (first_append_uid == 0)
first_append_uid = rec->uid;
- append_hdr = data;
- if (append_hdr->record_size > map->hdr->record_size) {
- /* we have to grow our record size */
- map = mail_index_map_to_memory(map,
- append_hdr->record_size);
- mail_index_sync_replace_map(view, map);
- }
- count = (hdr->size - sizeof(*append_hdr)) /
- append_hdr->record_size;
+ count = thdr->size / sizeof(*rec);
if (mail_index_grow(index, map, count) < 0) {
ret = -1;
break;
@@ -472,12 +604,24 @@
}
}
- if (mail_transaction_map(index, hdr, data,
+ if (mail_transaction_map(view->map, thdr, data,
&mail_index_map_sync_funcs,
&sync_map_ctx) < 0) {
ret = -1;
break;
}
+ if ((thdr->type & MAIL_TRANSACTION_EXTRA_INTRO) != 0) {
+ const struct mail_index_extra_record_info *einfo;
+ size_t size;
+
+ einfo = buffer_get_data(map->extra_infos, &size);
+ einfo += (size / sizeof(*einfo)) - 1;
+
+ map = mail_index_map_to_memory(map,
+ einfo->record_offset +
+ einfo->record_size);
+ mail_index_sync_replace_map(view, map);
+ }
}
if (sync_map_ctx.cache_locked) {
@@ -495,13 +639,15 @@
mail_transaction_log_get_head(index->log, &seq, &offset);
- map->hdr_copy.log_file_seq = seq;
- map->hdr_copy.log_file_offset = offset;
+ /* hdr pointer may have changed, update it */
+ tmphdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL);
+ tmphdr->log_file_seq = seq;
+ tmphdr->log_file_offset = offset;
if (first_append_uid != 0)
- mail_index_update_day_headers(&map->hdr_copy, first_append_uid);
+ mail_index_update_day_headers(tmphdr, first_append_uid);
- if ((map->hdr_copy.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) == 0 &&
+ if ((tmphdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) == 0 &&
had_dirty) {
/* do we have dirty flags anymore? */
const struct mail_index_record *rec;
@@ -509,8 +655,7 @@
for (i = 0; i < map->records_count; i++) {
rec = MAIL_INDEX_MAP_IDX(map, i);
if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
- map->hdr_copy.flags |=
- MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
+ tmphdr->flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
break;
}
}
@@ -520,7 +665,7 @@
map->mmap_used_size = index->hdr->header_size +
map->records_count * map->hdr->record_size;
- memcpy(map->mmap_base, &map->hdr_copy, sizeof(map->hdr_copy));
+ memcpy(map->mmap_base, tmphdr, tmphdr->header_size);
if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) {
mail_index_set_syscall_error(index, "msync()");
ret = -1;
@@ -535,5 +680,6 @@
struct mail_transaction_map_functions mail_index_map_sync_funcs = {
sync_expunge, sync_append, sync_flag_update,
sync_cache_reset, sync_cache_update, sync_header_update,
- sync_extra_rec_update
+ sync_extra_intro, sync_extra_reset,
+ sync_extra_hdr_update, sync_extra_rec_update
};
Index: mail-index-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync.c,v
retrieving revision 1.30
retrieving revision 1.31
diff -u -d -r1.30 -r1.31
--- mail-index-sync.c 24 Aug 2004 05:19:22 -0000 1.30
+++ mail-index-sync.c 5 Sep 2004 17:53:45 -0000 1.31
@@ -125,7 +125,6 @@
ctx->hdr->size);
break;
case MAIL_TRANSACTION_APPEND: {
- const struct mail_transaction_append_header *hdr = ctx->data;
const struct mail_index_record *rec = ctx->data;
if (ctx->append_uid_first == 0 ||
@@ -133,7 +132,7 @@
ctx->append_uid_first = rec->uid;
rec = CONST_PTR_OFFSET(ctx->data,
- ctx->hdr->size - hdr->record_size);
+ ctx->hdr->size - sizeof(*rec));
if (rec->uid > ctx->append_uid_last)
ctx->append_uid_last = rec->uid;
Index: mail-index-transaction-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-transaction-private.h,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- mail-index-transaction-private.h 4 Jul 2004 21:56:12 -0000 1.10
+++ mail-index-transaction-private.h 5 Sep 2004 17:53:45 -0000 1.11
@@ -9,7 +9,6 @@
buffer_t *appends;
uint32_t first_new_seq, last_new_seq;
- unsigned int append_record_size;
buffer_t *expunges;
@@ -20,7 +19,9 @@
unsigned char hdr_change[sizeof(struct mail_index_header)];
unsigned char hdr_mask[sizeof(struct mail_index_header)];
- buffer_t *extra_rec_updates[MAIL_INDEX_MAX_EXTRA_RECORDS];
+ buffer_t *extra_rec_updates; /* buffer[] */
+ buffer_t *extra_intros;
+ uint32_t extra_intros_max_id;
uint32_t new_cache_file_seq, last_cache_file_seq;
buffer_t *cache_updates;
@@ -36,4 +37,7 @@
void mail_index_transaction_ref(struct mail_index_transaction *t);
void mail_index_transaction_unref(struct mail_index_transaction *t);
+int mail_index_seq_buffer_lookup(buffer_t *buffer, uint32_t seq,
+ size_t record_size, size_t *pos_r);
+
#endif
Index: mail-index-transaction-view.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-transaction-view.c,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- mail-index-transaction-view.c 3 Jul 2004 10:57:52 -0000 1.3
+++ mail-index-transaction-view.c 5 Sep 2004 17:53:45 -0000 1.4
@@ -124,17 +124,48 @@
rec = buffer_get_data(tview->t->appends, NULL);
seq = tview->t->first_new_seq;
message_count = tview->t->last_new_seq;
- for (; seq <= message_count; seq++) {
+ for (; seq <= message_count; seq++, rec++) {
if ((rec->flags & flags_mask) == (uint8_t)flags) {
*seq_r = seq;
break;
}
- rec = CONST_PTR_OFFSET(rec, view->index->max_record_size);
}
return 0;
}
+static int _tview_lookup_extra(struct mail_index_view *view, uint32_t seq,
+ uint32_t data_id, const void **data_r)
+{
+ struct mail_index_view_transaction *tview =
+ (struct mail_index_view_transaction *)view;
+ const struct mail_index_extra_record_info *einfo;
+ buffer_t *const *extra_bufs;
+ size_t size, pos;
+
+ if (seq < tview->t->first_new_seq)
+ return tview->parent->lookup_extra(view, seq, data_id, data_r);
+
+ i_assert(seq <= tview->t->last_new_seq);
+ i_assert(data_id < view->index->extra_infos->used / sizeof(*einfo));
+
+ einfo = view->index->extra_infos->data;
+ einfo += data_id;
+
+ extra_bufs = buffer_get_data(tview->t->extra_rec_updates, &size);
+ size /= sizeof(*extra_bufs);
+
+ if (size <= data_id || extra_bufs[data_id] == NULL ||
+ !mail_index_seq_buffer_lookup(extra_bufs[data_id], seq,
+ einfo->record_size, &pos)) {
+ *data_r = NULL;
+ return 1;
+ }
+
+ *data_r = CONST_PTR_OFFSET(extra_bufs[data_id]->data, pos);
+ return 1;
+}
+
static struct mail_index_view_methods view_methods = {
_tview_close,
_tview_get_message_count,
@@ -142,7 +173,8 @@
_tview_lookup_full,
_tview_lookup_uid,
_tview_lookup_uid_range,
- _tview_lookup_first
+ _tview_lookup_first,
+ _tview_lookup_extra
};
struct mail_index_view *
Index: mail-index-transaction.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-transaction.c,v
retrieving revision 1.20
retrieving revision 1.21
diff -u -d -r1.20 -r1.21
--- mail-index-transaction.c 30 Jul 2004 09:30:51 -0000 1.20
+++ mail-index-transaction.c 5 Sep 2004 17:53:45 -0000 1.21
@@ -33,13 +33,19 @@
static void mail_index_transaction_free(struct mail_index_transaction *t)
{
- unsigned int i;
+ buffer_t **recs;
+ unsigned int i, size;
mail_index_view_transaction_unref(t->view);
- for (i = 0; i < t->view->index->extra_records_count; i++) {
- if (t->extra_rec_updates[i] != NULL)
- buffer_free(t->extra_rec_updates[i]);
+ if (t->extra_rec_updates != NULL) {
+ recs = buffer_get_modifyable_data(t->extra_rec_updates, &size);
+ size /= sizeof(*recs);
+
+ for (i = 0; i < size; i++) {
+ if (recs[i] != NULL)
+ buffer_free(recs[i]);
+ }
}
if (t->appends != NULL)
@@ -50,6 +56,8 @@
buffer_free(t->updates);
if (t->cache_updates != NULL)
buffer_free(t->cache_updates);
+ if (t->extra_intros != NULL)
+ buffer_free(t->extra_intros);
i_free(t);
}
@@ -65,12 +73,15 @@
}
static void
-mail_index_buffer_convert_to_uids(struct mail_index_view *view,
+mail_index_buffer_convert_to_uids(struct mail_index_transaction *t,
buffer_t *buf, size_t record_size, int range)
{
+ struct mail_index_view *view = t->view;
+ const struct mail_index_record *rec;
unsigned char *data;
size_t size, i;
uint32_t *seq;
+ int j;
if (buf == NULL)
return;
@@ -79,12 +90,16 @@
data = buffer_get_modifyable_data(buf, &size);
for (i = 0; i < size; i += record_size) {
seq = (uint32_t *)&data[i];
- i_assert(seq[0] <= view->map->records_count);
- seq[0] = MAIL_INDEX_MAP_IDX(view->map, seq[0]-1)->uid;
- if (range) {
- i_assert(seq[1] <= view->map->records_count);
- seq[1] = MAIL_INDEX_MAP_IDX(view->map, seq[1]-1)->uid;
+ for (j = 0; j <= range; j++, seq++) {
+ if (*seq >= t->first_new_seq) {
+ rec = mail_index_transaction_lookup(t, *seq);
+ *seq = rec->uid;
+ } else {
+ *seq = MAIL_INDEX_MAP_IDX(view->map,
+ *seq - 1)->uid;
+ }
+ i_assert(*seq != 0);
}
}
}
@@ -93,27 +108,34 @@
mail_index_transaction_convert_to_uids(struct mail_index_transaction *t)
{
struct mail_index *index = t->view->index;
- unsigned int i;
+ const struct mail_index_extra_record_info *einfos;
+ buffer_t **updates;
+ unsigned int i, size;
if (mail_index_view_lock(t->view) < 0)
return -1;
- for (i = 0; i < index->extra_records_count; i++) {
- if (t->extra_rec_updates[i] == NULL)
- continue;
+ if (t->extra_rec_updates != NULL) {
+ einfos = buffer_get_data(index->extra_infos, NULL);
+ updates = buffer_get_modifyable_data(t->extra_rec_updates,
+ &size);
+ size /= sizeof(*updates);
- mail_index_buffer_convert_to_uids(t->view,
- t->extra_rec_updates[i],
- sizeof(uint32_t) +
- index->extra_records[i].size,
- FALSE);
+ for (i = 0; i < size; i++) {
+ if (updates[i] == NULL)
+ continue;
+
+ mail_index_buffer_convert_to_uids(t, updates[i],
+ sizeof(uint32_t) + einfos[i].record_size,
+ FALSE);
+ }
}
- mail_index_buffer_convert_to_uids(t->view, t->expunges,
+ mail_index_buffer_convert_to_uids(t, t->expunges,
sizeof(struct mail_transaction_expunge), TRUE);
- mail_index_buffer_convert_to_uids(t->view, t->updates,
+ mail_index_buffer_convert_to_uids(t, t->updates,
sizeof(struct mail_transaction_flag_update), TRUE);
- mail_index_buffer_convert_to_uids(t->view, t->cache_updates,
+ mail_index_buffer_convert_to_uids(t, t->cache_updates,
sizeof(struct mail_transaction_cache_update), FALSE);
return 0;
}
@@ -157,42 +179,6 @@
mail_index_transaction_unref(t);
}
-static void
-mail_index_transaction_update_append_size(struct mail_index_transaction *t)
-{
- buffer_t *new_buf;
- unsigned int new_record_size;
- const void *src;
- void *dest;
- size_t i, size;
-
- new_record_size = t->view->index->max_record_size;
- if (t->append_record_size == new_record_size)
- return;
-
- i_assert(t->append_record_size < new_record_size);
-
- if (t->append_record_size != 0) {
- /* resize the records in buffer */
- src = buffer_get_data(t->appends, &size);
- size /= t->append_record_size;
-
- new_buf = buffer_create_dynamic(default_pool,
- (size + 10) * new_record_size,
- (size_t)-1);
- for (i = 0; i < size; i++) {
- dest = buffer_append_space_unsafe(new_buf,
- new_record_size);
- memcpy(dest, src, t->append_record_size);
- src = CONST_PTR_OFFSET(src, t->append_record_size);
- }
- buffer_free(t->appends);
- t->appends = new_buf;
- }
-
- t->append_record_size = new_record_size;
-}
-
struct mail_index_record *
mail_index_transaction_lookup(struct mail_index_transaction *t, uint32_t seq)
{
@@ -200,10 +186,9 @@
i_assert(seq >= t->first_new_seq && seq <= t->last_new_seq);
- mail_index_transaction_update_append_size(t);
-
- pos = (seq - t->first_new_seq) * t->append_record_size;
- return buffer_get_space_unsafe(t->appends, pos, t->append_record_size);
+ pos = (seq - t->first_new_seq) * sizeof(struct mail_index_record);
+ return buffer_get_space_unsafe(t->appends, pos,
+ sizeof(struct mail_index_record));
}
void mail_index_append(struct mail_index_transaction *t, uint32_t uid,
@@ -214,9 +199,7 @@
if (t->appends == NULL) {
t->appends = buffer_create_dynamic(default_pool,
4096, (size_t)-1);
- t->append_record_size = t->view->index->max_record_size;
}
- mail_index_transaction_update_append_size(t);
/* sequence number is visible only inside given view,
so let it generate it */
@@ -225,8 +208,8 @@
else
*seq_r = t->last_new_seq = t->first_new_seq;
- rec = buffer_append_space_unsafe(t->appends, t->append_record_size);
- memset(rec, 0, t->append_record_size);
+ rec = buffer_append_space_unsafe(t->appends, sizeof(*rec));
+ memset(rec, 0, sizeof(*rec));
rec->uid = uid;
}
@@ -243,16 +226,14 @@
end = PTR_OFFSET(rec, size);
/* find the first mail with uid = 0 */
- while (rec != end) {
+ for (; rec != end; rec++) {
if (rec->uid == 0)
break;
- rec = PTR_OFFSET(rec, t->append_record_size);
}
- while (rec != end) {
+ for (; rec != end; rec++) {
i_assert(rec->uid == 0);
rec->uid = first_uid++;
- rec = PTR_OFFSET(rec, t->append_record_size);
}
*next_uid_r = first_uid;
@@ -511,9 +492,8 @@
&update, sizeof(update));
}
-static int
-mail_index_seq_buffer_lookup(buffer_t *buffer, uint32_t seq,
- size_t record_size, size_t *pos_r)
+int mail_index_seq_buffer_lookup(buffer_t *buffer, uint32_t seq,
+ size_t record_size, size_t *pos_r)
{
unsigned int idx, left_idx, right_idx;
void *data;
@@ -657,20 +637,22 @@
const void *data)
{
struct mail_index *index = t->view->index;
- struct mail_index_record *rec;
+ buffer_t **buf;
+ const struct mail_index_extra_record_info *einfo;
- i_assert(data_id < index->extra_records_count);
+ i_assert(data_id < index->extra_infos->used / sizeof(*einfo));
- if (seq >= t->first_new_seq) {
- /* just appended message, modify it directly */
- /* FIXME: do data_id mapping conversion */
- rec = mail_index_transaction_lookup(t, seq);
- memcpy(PTR_OFFSET(rec, index->extra_records[data_id].offset),
- data, index->extra_records[data_id].size);
- } else {
- mail_index_update_seq_buffer(&t->extra_rec_updates[data_id],
- seq, data, index->extra_records[data_id].size, NULL);
+ einfo = index->extra_infos->data;
+ einfo += data_id;
+
+ if (t->extra_rec_updates == NULL) {
+ t->extra_rec_updates =
+ buffer_create_dynamic(default_pool, 128, (size_t)-1);
}
+ buf = buffer_get_space_unsafe(t->extra_rec_updates,
+ data_id * sizeof(buffer_t *),
+ sizeof(buffer_t *));
+ mail_index_update_seq_buffer(buf, seq, data, einfo->record_size, NULL);
}
void mail_index_update_header(struct mail_index_transaction *t,
Index: mail-index-view-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-view-private.h,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- mail-index-view-private.h 4 Sep 2004 08:51:08 -0000 1.8
+++ mail-index-view-private.h 5 Sep 2004 17:53:45 -0000 1.9
@@ -18,6 +18,8 @@
uint32_t *first_seq_r, uint32_t *last_seq_r);
int (*lookup_first)(struct mail_index_view *view, enum mail_flags flags,
uint8_t flags_mask, uint32_t *seq_r);
+ int (*lookup_extra)(struct mail_index_view *view, uint32_t seq,
+ uint32_t data_id, const void **data_r);
};
struct mail_index_view {
Index: mail-index-view-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-view-sync.c,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -d -r1.19 -r1.20
--- mail-index-view-sync.c 4 Sep 2004 08:51:08 -0000 1.19
+++ mail-index-view-sync.c 5 Sep 2004 17:53:45 -0000 1.20
@@ -206,7 +206,7 @@
memset(&sync_map_ctx, 0, sizeof(sync_map_ctx));
sync_map_ctx.view = view;
- if (mail_transaction_map(view->index, ctx->hdr, ctx->data,
+ if (mail_transaction_map(view->map, ctx->hdr, ctx->data,
&mail_index_map_sync_funcs,
&sync_map_ctx) < 0)
return -1;
Index: mail-index-view.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-view.c,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -d -r1.19 -r1.20
--- mail-index-view.c 4 Sep 2004 08:51:08 -0000 1.19
+++ mail-index-view.c 5 Sep 2004 17:53:45 -0000 1.20
@@ -407,6 +407,32 @@
return 0;
}
+static int _view_lookup_extra(struct mail_index_view *view, uint32_t seq,
+ uint32_t data_id, const void **data_r)
+{
+ const struct mail_index_extra_record_info *einfo;
+ const struct mail_index_record *rec;
+ struct mail_index_map *map;
+ uint32_t idx, offset;
+ int ret;
+
+ if ((ret = mail_index_lookup_full(view, seq, &map, &rec)) < 0)
+ return -1;
+
+ if (rec == NULL ||
+ !mail_index_map_get_extra_info_idx(view->map, data_id, &idx)) {
+ *data_r = NULL;
+ return ret;
+ }
+
+ einfo = view->map->extra_infos->data;
+ einfo += idx;
+
+ offset = einfo->record_offset;
+ *data_r = offset == 0 ? NULL : CONST_PTR_OFFSET(rec, offset);
+ return ret;
+}
+
void mail_index_view_close(struct mail_index_view *view)
{
view->methods.close(view);
@@ -431,33 +457,17 @@
return mail_index_lookup_full(view, seq, &map, rec_r);
}
-int mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq,
- uint32_t *uid_r)
+int mail_index_lookup_full(struct mail_index_view *view, uint32_t seq,
+ struct mail_index_map **map_r,
+ const struct mail_index_record **rec_r)
{
- return view->methods.lookup_uid(view, seq, uid_r);
+ return view->methods.lookup_full(view, seq, map_r, rec_r);
}
-int mail_index_lookup_extra(struct mail_index_view *view, uint32_t seq,
- uint32_t data_id, const void **data_r)
+int mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq,
+ uint32_t *uid_r)
{
- const struct mail_index_record *rec;
- struct mail_index_map *map;
- uint32_t offset;
- int ret;
-
- if ((ret = mail_index_lookup_full(view, seq, &map, &rec)) < 0)
- return -1;
-
- if (rec == NULL) {
- *data_r = NULL;
- return ret;
- }
-
- /* FIXME: do data_id mapping conversion */
-
- offset = view->index->extra_records[data_id].offset;
- *data_r = CONST_PTR_OFFSET(rec, offset);
- return ret;
+ return view->methods.lookup_uid(view, seq, uid_r);
}
int mail_index_lookup_uid_range(struct mail_index_view *view,
@@ -474,11 +484,10 @@
return view->methods.lookup_first(view, flags, flags_mask, seq_r);
}
-int mail_index_lookup_full(struct mail_index_view *view, uint32_t seq,
- struct mail_index_map **map_r,
- const struct mail_index_record **rec_r)
+int mail_index_lookup_extra(struct mail_index_view *view, uint32_t seq,
+ uint32_t data_id, const void **data_r)
{
- return view->methods.lookup_full(view, seq, map_r, rec_r);
+ return view->methods.lookup_extra(view, seq, data_id, data_r);
}
static struct mail_index_view_methods view_methods = {
@@ -488,7 +497,8 @@
_view_lookup_full,
_view_lookup_uid,
_view_lookup_uid_range,
- _view_lookup_first
+ _view_lookup_first,
+ _view_lookup_extra
};
struct mail_index_view *mail_index_view_open(struct mail_index *index)
Index: mail-index.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.144
retrieving revision 1.145
diff -u -d -r1.144 -r1.145
--- mail-index.c 4 Sep 2004 08:58:40 -0000 1.144
+++ mail-index.c 5 Sep 2004 17:53:45 -0000 1.145
@@ -26,12 +26,11 @@
index->prefix = i_strdup(prefix);
index->fd = -1;
- index->extra_records_pool =
- pool_alloconly_create("extra_record_pool", 256);
- index->extra_records_buf =
- buffer_create_dynamic(index->extra_records_pool,
+ index->extra_infos_pool =
+ pool_alloconly_create("extra_infos_pool", 256);
+ index->extra_infos =
+ buffer_create_dynamic(index->extra_infos_pool,
64, (size_t)-1);
- index->max_record_size = sizeof(struct mail_index_record);
index->mode = 0600;
index->gid = (gid_t)-1;
@@ -41,7 +40,7 @@
void mail_index_free(struct mail_index *index)
{
mail_index_close(index);
- pool_unref(index->extra_records_pool);
+ pool_unref(index->extra_infos_pool);
i_free(index->error);
i_free(index->dir);
@@ -50,42 +49,171 @@
}
uint32_t mail_index_register_record_extra(struct mail_index *index,
- const char *name, uint16_t size)
+ const char *name, uint32_t hdr_size,
+ uint16_t record_size)
{
+ const struct mail_index_extra_record_info *einfos;
struct mail_index_extra_record_info info;
- size_t buf_size;
+ size_t extra_count;
unsigned int i;
+ einfos = buffer_get_data(index->extra_infos, &extra_count);
+ extra_count /= sizeof(*einfos);
+
/* see if it's there already */
- for (i = 0; i < index->extra_records_count; i++) {
- if (strcmp(index->extra_records[i].name, name) == 0) {
- i_assert(index->extra_records[i].size == size);
+ for (i = 0; i < extra_count; i++) {
+ if (strcmp(einfos[i].name, name) == 0) {
+ i_assert(einfos[i].hdr_size == hdr_size);
+ i_assert(einfos[i].record_size == record_size);
return i;
}
}
- i_assert(size % 4 == 0);
- i_assert(index->max_record_size + size <= 65535);
+ i_assert(hdr_size % 4 == 0);
+ i_assert(record_size % 4 == 0);
- if (index->extra_records_count >= MAIL_INDEX_MAX_EXTRA_RECORDS) {
- i_panic("Maximum extra record count reached, "
- "you'll need to recompile with larger limit. "
- "MAIL_INDEX_MAX_EXTRA_RECORDS = %d",
- MAIL_INDEX_MAX_EXTRA_RECORDS);
+ memset(&info, 0, sizeof(info));
+ info.name = p_strdup(index->extra_infos_pool, name);
+ info.hdr_size = hdr_size;
+ info.record_size = record_size;
+
+ buffer_append(index->extra_infos, &info, sizeof(info));
+ return extra_count;
+}
+
+static void mail_index_map_create_extra_infos(struct mail_index_map *map,
+ unsigned int initial_count)
+{
+ size_t extra_infos_size, extra_infos_id_map_size, size;
+
+ extra_infos_size = initial_count *
+ sizeof(struct mail_index_extra_record_info);
+ extra_infos_id_map_size = initial_count * sizeof(uint32_t);
+ if (map->extra_records_pool == NULL) {
+ size = extra_infos_size + extra_infos_id_map_size +
+ initial_count * 20;
+ map->extra_records_pool =
+ pool_alloconly_create("extra_infos",
+ nearest_power(size));
}
- memset(&info, 0, sizeof(info));
- info.name = p_strdup(index->extra_records_pool, name);
- info.size = size;
- info.offset = index->max_record_size;
+ map->extra_infos = buffer_create_dynamic(map->extra_records_pool,
+ extra_infos_size, (size_t)-1);
+ map->extra_infos_id_map = buffer_create_dynamic(map->extra_records_pool,
+ extra_infos_id_map_size,
+ (size_t)-1);
+}
- buffer_append(index->extra_records_buf, &info, sizeof(info));
- index->extra_records =
- buffer_get_data(index->extra_records_buf, &buf_size);
- index->extra_records_count = buf_size / sizeof(info);
+uint32_t mail_index_map_register_extra_info(struct mail_index *index,
+ struct mail_index_map *map,
+ const char *name,
+ uint32_t hdr_offset,
+ uint32_t hdr_size,
+ uint32_t record_size)
+{
+ const struct mail_index_extra_record_info *last_einfo;
+ struct mail_index_extra_record_info *einfo;
+ size_t size;
+ uint32_t idx, data_id;
- index->max_record_size += size;
- return index->extra_records_count-1;
+ if (map->extra_infos == NULL) {
+ mail_index_map_create_extra_infos(map, 5);
+ last_einfo = NULL;
+ idx = 0;
+ } else {
+ last_einfo = buffer_get_data(map->extra_infos, &size);
+ idx = size / sizeof(*last_einfo);
+ if (idx == 0)
+ last_einfo = NULL;
+ else
+ last_einfo += idx - 1;
+ }
+
+ einfo = buffer_append_space_unsafe(map->extra_infos, sizeof(*einfo));
+ memset(einfo, 0, sizeof(*einfo));
+
+ einfo->name = p_strdup(map->extra_records_pool, name);
+ einfo->hdr_offset = hdr_offset;
+ einfo->hdr_size = hdr_size;
+ einfo->record_size = record_size;
+
+ if (last_einfo != NULL) {
+ einfo->record_offset = last_einfo->record_offset +
+ last_einfo->record_size;
+ } else {
+ einfo->record_offset = sizeof(struct mail_index_record);
+ }
+
+ data_id = mail_index_register_record_extra(index, name,
+ hdr_size, record_size);
+ buffer_write(map->extra_infos_id_map, data_id * sizeof(uint32_t),
+ &idx, sizeof(idx));
+ return idx;
+}
+
+static int mail_index_read_extra_infos(struct mail_index *index,
+ struct mail_index_map *map)
+{
+ const struct mail_index_extra_record_info_header *einfo_hdr;
+ unsigned int i, old_count;
+ const char *name;
+ uint32_t data_id, offset, name_offset;
+
+ offset = map->hdr->base_header_size;
+ if (offset == map->hdr->header_size &&
+ map->extra_records_pool == NULL) {
+ /* nothing to do, skip allocatations and all */
+ return 1;
+ }
+
+ old_count = index->extra_infos->used /
+ sizeof(struct mail_index_extra_record_info);
+
+ if (map->extra_records_pool != NULL)
+ p_clear(map->extra_records_pool);
+ mail_index_map_create_extra_infos(map, old_count + 5);
+ data_id = (uint32_t)-1;
+ for (i = 0; i < old_count; i++) {
+ buffer_append(map->extra_infos_id_map,
+ &data_id, sizeof(data_id));
+ }
+
+ name = map->hdr_base;
+ while (offset < map->hdr->header_size) {
+ name_offset = offset;
+
+ while (offset < map->hdr->header_size && name[offset] != '\0')
+ offset++;
+ if (offset == map->hdr->header_size) {
+ mail_index_set_error(index, "Corrupted index file %s: "
+ "Header extension name doesn't end with NUL",
+ index->filepath);
+ return -1;
+ }
+ offset++;
+ while (offset < map->hdr->header_size && (offset % 4) != 0)
+ offset++;
+
+ einfo_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
+
+ if (offset + sizeof(*einfo_hdr) > map->hdr->header_size ||
+ offset + sizeof(*einfo_hdr) + einfo_hdr->hdr_size >
+ map->hdr->header_size) {
+ mail_index_set_error(index, "Corrupted index file %s: "
+ "Header extension goes outside header",
+ index->filepath);
+ return -1;
+ }
+
+ mail_index_map_register_extra_info(index, map,
+ name + name_offset,
+ offset + sizeof(*einfo_hdr),
+ einfo_hdr->hdr_size,
+ einfo_hdr->record_size);
+
+ offset += sizeof(*einfo_hdr) + einfo_hdr->hdr_size;
+ }
+ return 1;
}
static int mail_index_check_header(struct mail_index *index,
@@ -151,7 +279,7 @@
hdr->first_deleted_uid_lowwater > hdr->next_uid)
return 0;
- return 1;
+ return mail_index_read_extra_infos(index, map);
}
static void mail_index_map_clear(struct mail_index *index,
@@ -184,6 +312,9 @@
i_assert(map->refcount == 0);
mail_index_map_clear(index, map);
+ if (map->extra_records_pool != NULL)
+ pool_unref(map->extra_records_pool);
+ buffer_free(map->hdr_copy_buf);
i_free(map);
}
@@ -197,6 +328,7 @@
static int mail_index_mmap(struct mail_index *index, struct mail_index_map *map)
{
const struct mail_index_header *hdr;
+ struct mail_index_header *mhdr;
unsigned int records_count;
i_assert(!map->write_to_disk);
@@ -241,25 +373,35 @@
if (map->hdr->base_header_size < sizeof(*map->hdr)) {
/* header smaller than ours, make a copy so our newer headers
won't have garbage in them */
- memcpy(&map->hdr_copy, map->hdr, map->hdr->base_header_size);
- map->hdr = &map->hdr_copy;
- }
+ buffer_reset(map->hdr_copy_buf);
+ buffer_append(map->hdr_copy_buf,
+ map->hdr, map->hdr->base_header_size);
+ buffer_set_used_size(map->hdr_copy_buf, sizeof(*map->hdr));
+ mhdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL);
+ mhdr->base_header_size = sizeof(*map->hdr);
+ mhdr->header_size = map->hdr_copy_buf->used;
+
+ map->hdr = mhdr;
+ }
+
+ map->hdr_base = map->mmap_base;
map->records = PTR_OFFSET(map->mmap_base, map->hdr->header_size);
map->records_count = map->hdr->messages_count;
return 1;
}
static int mail_index_read_map(struct mail_index *index,
- struct mail_index_map *map)
+ struct mail_index_map *map, int *retry_r)
{
- struct mail_index_header hdr;
+ struct mail_index_header hdr, *hdrp;
void *data = NULL;
ssize_t ret;
size_t pos, records_size;
i_assert(map->mmap_base == NULL);
+ *retry_r = FALSE;
memset(&hdr, 0, sizeof(hdr));
ret = 1;
@@ -269,7 +411,50 @@
if (ret > 0)
pos += ret;
}
- if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE) {
+
+ if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
+ (ret > 0 || pos >= hdr.base_header_size)) {
+ if (hdr.base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
+ hdr.header_size < hdr.base_header_size) {
+ mail_index_set_error(index, "Corrupted index file %s: "
+ "Corrupted header sizes (base %u, full %u)",
+ index->filepath, hdr.base_header_size,
+ hdr.header_size);
+ return 0;
+ }
+
+ buffer_reset(map->hdr_copy_buf);
+ if (hdr.base_header_size < sizeof(hdr)) {
+ buffer_append(map->hdr_copy_buf, &hdr,
+ hdr.base_header_size);
+ buffer_set_used_size(map->hdr_copy_buf, sizeof(hdr) +
+ hdr.header_size -
+ hdr.base_header_size);
+
+ /* @UNSAFE */
+ ret = pread_full(index->fd,
+ PTR_OFFSET(map->hdr_copy_buf->data,
+ sizeof(hdr)),
+ hdr.header_size - hdr.base_header_size,
+ hdr.base_header_size);
+
+ hdrp = buffer_get_modifyable_data(map->hdr_copy_buf,
+ NULL);
+ hdrp->base_header_size = sizeof(hdr);
+ hdrp->header_size = map->hdr_copy_buf->used;
+ } else {
+ buffer_append(map->hdr_copy_buf, &hdr, pos);
+ buffer_set_used_size(map->hdr_copy_buf,
+ hdr.header_size);
+ /* @UNSAFE */
+ ret = pread_full(index->fd,
+ PTR_OFFSET(map->hdr_copy_buf->data,
+ pos),
+ hdr.header_size - pos, pos);
+ }
+ }
+
+ if (ret > 0) {
records_size = hdr.messages_count * hdr.record_size;
if (map->buffer == NULL) {
@@ -287,33 +472,36 @@
}
if (ret < 0) {
- if (errno == ESTALE)
+ if (errno == ESTALE) {
+ *retry_r = TRUE;
return 0;
+ }
mail_index_set_syscall_error(index, "pread_full()");
return -1;
}
if (ret == 0) {
mail_index_set_error(index,
- "Unexpected EOF while reading index file");
+ "Corrupted index file %s: File too small",
+ index->filepath);
return -1;
}
map->records = data;
map->records_count = hdr.messages_count;
- map->hdr_copy = hdr;
- map->hdr = &map->hdr_copy;
+ map->hdr = map->hdr_copy_buf->data;
+ map->hdr_base = map->hdr_copy_buf->data;
return 1;
}
static int mail_index_read_map_with_retry(struct mail_index *index,
struct mail_index_map *map)
{
- int i, ret;
+ int i, ret, retry;
for (i = 0; i < MAIL_INDEX_ESTALE_RETRY_COUNT; i++) {
- ret = mail_index_read_map(index, map);
- if (ret != 0)
+ ret = mail_index_read_map(index, map, &retry);
+ if (ret != 0 || !retry)
return ret;
/* ESTALE - reopen index file */
@@ -381,6 +569,9 @@
if (map == NULL) {
map = i_new(struct mail_index_map, 1);
map->refcount = 1;
+ map->hdr_copy_buf =
+ buffer_create_dynamic(default_pool,
+ sizeof(*map->hdr), (size_t)-1);
} else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
if (map->write_to_disk) {
/* we have modified this mapping and it's waiting to
@@ -432,9 +623,11 @@
mail_index_map_to_memory(struct mail_index_map *map, uint32_t new_record_size)
{
struct mail_index_map *mem_map;
+ struct mail_index_header *hdr;
+ struct mail_index_extra_record_info *einfos;
void *src, *dest;
size_t size, copy_size;
- unsigned int i;
+ unsigned int i, count;
if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
map->refcount++;
@@ -462,12 +655,50 @@
mem_map->records = buffer_get_modifyable_data(mem_map->buffer, NULL);
mem_map->records_count = map->records_count;
- mem_map->hdr_copy = *map->hdr;
- mem_map->hdr_copy.record_size = new_record_size;
- mem_map->hdr = &mem_map->hdr_copy;
+ mem_map->hdr_copy_buf = buffer_create_dynamic(default_pool,
+ map->hdr->header_size,
+ (size_t)-1);
+ buffer_append(mem_map->hdr_copy_buf, map->hdr, map->hdr->header_size);
+
+ hdr = buffer_get_modifyable_data(mem_map->hdr_copy_buf, NULL);
+ hdr->record_size = new_record_size;
+ mem_map->hdr = hdr;
+
+ /* copy extra_infos */
+ if (map->extra_infos_id_map != NULL) {
+ count = map->extra_infos_id_map->used / sizeof(uint32_t);
+ mail_index_map_create_extra_infos(mem_map, count);
+
+ buffer_append_buf(mem_map->extra_infos, map->extra_infos,
+ 0, (size_t)-1);
+ buffer_append_buf(mem_map->extra_infos_id_map,
+ map->extra_infos_id_map, 0, (size_t)-1);
+
+ /* fix the name pointers to use our own pool */
+ einfos = buffer_get_modifyable_data(mem_map->extra_infos, NULL);
+ for (i = 0; i < count; i++) {
+ einfos[i].name = p_strdup(mem_map->extra_records_pool,
+ einfos[i].name);
+ }
+ }
+
return mem_map;
}
+int mail_index_map_get_extra_info_idx(struct mail_index_map *map,
+ uint32_t data_id, uint32_t *idx_r)
+{
+ const uint32_t *id_map;
+
+ if (map->extra_infos_id_map == NULL ||
+ map->extra_infos_id_map->used / sizeof(*id_map) <= data_id)
+ return 0;
+
+ id_map = map->extra_infos_id_map->data;
+ *idx_r = id_map[data_id];
+ return *idx_r != (uint32_t)-1;
+}
+
static int mail_index_try_open_only(struct mail_index *index)
{
int i;
@@ -550,8 +781,8 @@
return -1;
}
- index->map->hdr_copy = *hdr;
- index->hdr = &index->map->hdr_copy;
+ buffer_write(index->map->hdr_copy_buf, 0, hdr, hdr_size);
+ i_assert(index->hdr == index->map->hdr_copy_buf->data);
}
return 0;
@@ -631,8 +862,7 @@
return ret;
}
-static void mail_index_header_init(struct mail_index *index,
- struct mail_index_header *hdr)
+static void mail_index_header_init(struct mail_index_header *hdr)
{
time_t now = time(NULL);
@@ -642,7 +872,7 @@
hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
hdr->base_header_size = sizeof(*hdr);
hdr->header_size = sizeof(*hdr);
- hdr->record_size = index->max_record_size;
+ hdr->record_size = sizeof(struct mail_index_record);
hdr->keywords_mask_size = sizeof(keywords_mask_t);
#ifndef WORDS_BIGENDIAN
@@ -671,7 +901,7 @@
/* doesn't exist, or corrupted */
if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
return 0;
- mail_index_header_init(index, &hdr);
+ mail_index_header_init(&hdr);
index->hdr = &hdr;
} else if (ret < 0)
return -1;
Index: mail-index.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.h,v
retrieving revision 1.126
retrieving revision 1.127
diff -u -d -r1.126 -r1.127
--- mail-index.h 30 Jul 2004 05:08:35 -0000 1.126
+++ mail-index.h 5 Sep 2004 17:53:45 -0000 1.127
@@ -3,7 +3,7 @@
#include "mail-types.h"
-#define MAIL_INDEX_MAJOR_VERSION 4
+#define MAIL_INDEX_MAJOR_VERSION 5
#define MAIL_INDEX_MINOR_VERSION 0
#define MAIL_INDEX_HEADER_MIN_SIZE 120
@@ -97,7 +97,6 @@
uint64_t sync_size;
uint32_t cache_file_seq;
- uint32_t extra_records_hdr_offset;
/* daily first UIDs that have been added to index. */
uint32_t day_stamp;
@@ -142,7 +141,8 @@
identifier for the data. if same name is tried to be registered multiple
times, the rest are ignored. returns identifier for the name. */
uint32_t mail_index_register_record_extra(struct mail_index *index,
- const char *name, uint16_t size);
+ const char *name, uint32_t hdr_size,
+ uint16_t record_size);
int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags);
void mail_index_close(struct mail_index *index);
Index: mail-transaction-log-view.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log-view.c,v
retrieving revision 1.24
retrieving revision 1.25
diff -u -d -r1.24 -r1.25
--- mail-transaction-log-view.c 28 Aug 2004 12:45:42 -0000 1.24
+++ mail-transaction-log-view.c 5 Sep 2004 17:53:45 -0000 1.25
@@ -22,6 +22,8 @@
uint32_t prev_file_seq;
uoff_t prev_file_offset;
+ uint32_t max_extra_data_id;
+
unsigned int broken:1;
};
@@ -286,25 +288,35 @@
", size=%u, end=%"PRIuSIZE_T")",
hdr->type & MAIL_TRANSACTION_TYPE_MASK,
view->cur_offset, hdr_size, file_size);
- view->cur_offset = file_size;
return -1;
}
if (hdr_size < sizeof(*hdr)) {
+ type_rec = NULL;
+ record_size = 0;
+ } else {
+ type_rec = mail_transaction_type_lookup(hdr->type);
+ if (type_rec != NULL)
+ record_size = type_rec->record_size;
+ else {
+ mail_transaction_log_file_set_corrupted(file,
+ "unknown record type 0x%x",
+ hdr->type & MAIL_TRANSACTION_TYPE_MASK);
+ return -1;
+ }
+ }
+
+ if (hdr_size < sizeof(*hdr) + record_size) {
mail_transaction_log_file_set_corrupted(file,
"record size too small (type=0x%x, size=%u)",
hdr->type & MAIL_TRANSACTION_TYPE_MASK, hdr_size);
- view->cur_offset = file_size;
return -1;
}
- type_rec = mail_transaction_type_lookup(hdr->type);
- if (type_rec != NULL)
- record_size = type_rec->record_size;
- else {
+ if ((hdr_size - sizeof(*hdr)) % record_size != 0) {
mail_transaction_log_file_set_corrupted(file,
- "unknown record type 0x%x",
- hdr->type & MAIL_TRANSACTION_TYPE_MASK);
- view->cur_offset = file->sync_offset;
+ "record size wrong (type 0x%x, %u %% %u != 0)",
+ hdr->type & MAIL_TRANSACTION_TYPE_MASK,
+ (hdr_size - sizeof(*hdr)), record_size);
return -1;
}
@@ -320,25 +332,34 @@
"extra bits in header type: 0x%x",
hdr->type & MAIL_TRANSACTION_TYPE_MASK);
return -1;
- } else if (hdr->type == MAIL_TRANSACTION_EXTRA_REC_UPDATE) {
- const struct mail_transaction_extra_rec_header *ehdr = data;
+ } else if (hdr->type == MAIL_TRANSACTION_EXTRA_INTRO) {
+ const struct mail_transaction_extra_intro *intro = data;
- if (ehdr->data_id >= view->log->index->extra_records_count) {
+ if (intro->data_id > view->max_extra_data_id)
+ view->max_extra_data_id = intro->data_id;
+ if (intro->name_size >
+ hdr_size - sizeof(*hdr) - sizeof(*intro)) {
mail_transaction_log_file_set_corrupted(file,
- "extra record update out of range (%u > %u)",
- ehdr->data_id,
- view->log->index->extra_records_count);
+ "extra intro: name_size too large");
return -1;
}
- }
+ } else if (hdr->type == MAIL_TRANSACTION_EXTRA_REC_UPDATE ||
+ hdr->type == MAIL_TRANSACTION_EXTRA_HDR_UPDATE ||
+ hdr->type == MAIL_TRANSACTION_EXTRA_RESET) {
+ const uint32_t *data_id = data;
+ uint32_t max_data_id;
- if ((hdr_size - sizeof(*hdr)) % record_size != 0) {
- mail_transaction_log_file_set_corrupted(file,
- "record size wrong (type 0x%x, %u %% %u != 0)",
- hdr->type & MAIL_TRANSACTION_TYPE_MASK,
- (hdr_size - sizeof(*hdr)), record_size);
- view->cur_offset = file->sync_offset;
- return -1;
+ max_data_id = view->log->index->map->extra_infos->used /
+ sizeof(struct mail_index_extra_record_info);
+ if (view->max_extra_data_id > max_data_id)
+ max_data_id = view->max_extra_data_id;
+
+ if (*data_id >= max_data_id) {
+ mail_transaction_log_file_set_corrupted(file,
+ "extra record update out of range (%u > %u)",
+ *data_id, max_data_id);
+ return -1;
+ }
}
*hdr_r = hdr;
@@ -372,8 +393,12 @@
append isn't in mask */
}
- if (ret <= 0)
- return ret;
+ if (ret < 0) {
+ view->cur_offset = view->cur->sync_offset;
+ return -1;
+ }
+ if (ret == 0)
+ return 0;
view->tmp_hdr = *hdr;
view->tmp_hdr.size =
Index: mail-transaction-log.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.c,v
retrieving revision 1.53
retrieving revision 1.54
diff -u -d -r1.53 -r1.54
--- mail-transaction-log.c 22 Aug 2004 10:13:05 -0000 1.53
+++ mail-transaction-log.c 5 Sep 2004 17:53:45 -0000 1.54
@@ -925,10 +925,8 @@
const struct mail_transaction_header *hdr,
const void *data)
{
- const struct mail_transaction_append_header *append_hdr = data;
const struct mail_index_record *old, *old_end;
struct mail_index_record *appends, *end, *rec, *dest;
- uint32_t record_size = t->append_record_size;
size_t size;
int deleted = FALSE;
@@ -944,11 +942,10 @@
/* we'll just check that none of the appends are already in
transaction log. this could happen if we crashed before we had
a chance to update index file */
- old = CONST_PTR_OFFSET(data, sizeof(*append_hdr));
old_end = CONST_PTR_OFFSET(data, hdr->size);
- while (old != old_end) {
+ for (old = data; old != old_end; old++) {
/* appends are sorted */
- for (rec = appends; rec != end; ) {
+ for (rec = appends; rec != end; rec++) {
if (rec->uid >= old->uid) {
if (rec->uid == old->uid) {
rec->uid = 0;
@@ -956,25 +953,56 @@
}
break;
}
- rec = PTR_OFFSET(rec, record_size);
}
- old = CONST_PTR_OFFSET(old, append_hdr->record_size);
}
if (deleted) {
/* compress deleted appends away */
- for (rec = dest = appends; rec != end; ) {
+ for (rec = dest = appends; rec != end; rec++) {
if (rec->uid != 0)
dest++;
else if (rec != dest)
*rec = *dest;
- rec = PTR_OFFSET(rec, record_size);
}
buffer_set_used_size(t->appends,
(char *)dest - (char *)appends);
}
}
+static void
+transaction_save_extra_intro(struct mail_index_transaction *t,
+ const struct mail_transaction_extra_intro *intro)
+{
+ const char *name;
+ void *p;
+ uint32_t data_id;
+ size_t pos;
+
+ if (t->extra_intros == NULL) {
+ t->extra_intros =
+ buffer_create_dynamic(default_pool, 128, (size_t)-1);
+ }
+
+ t_push();
+ name = t_strndup((const char *)(intro+1), intro->name_size);
+ data_id = mail_index_register_record_extra(t->view->index, name,
+ intro->hdr_size,
+ intro->record_size);
+ pos = data_id * sizeof(intro->data_id);
+ if (pos > t->extra_intros->used) {
+ /* unused records are -1 */
+ p = buffer_append_space_unsafe(t->extra_intros,
+ pos - t->extra_intros->used);
+ memset(p, 0xff, pos - t->extra_intros->used);
+ }
+
+ buffer_write(t->extra_intros, pos,
+ &intro->data_id, sizeof(intro->data_id));
+ if (intro->data_id > t->extra_intros_max_id)
+ t->extra_intros_max_id = intro->data_id;
+ t_pop();
+}
+
static int mail_transaction_log_scan_pending(struct mail_transaction_log *log,
struct mail_index_transaction *t)
{
@@ -1001,6 +1029,12 @@
max_cache_file_seq = reset->new_file_seq;
break;
}
+ case MAIL_TRANSACTION_EXTRA_INTRO: {
+ const struct mail_transaction_extra_intro *intro = data;
+
+ transaction_save_extra_intro(t, intro);
+ break;
+ }
}
}
@@ -1114,6 +1148,49 @@
return buf;
}
+static int
+mail_transaction_log_register_extra(struct mail_transaction_log_file *file,
+ struct mail_index_transaction *t,
+ uint32_t data_id, uint32_t *idx_r)
+{
+ const struct mail_index_extra_record_info *einfo;
+ struct mail_transaction_extra_intro *intro;
+ const uint32_t *id_map;
+ buffer_t *buf;
+ size_t size;
+ int ret;
+
+ /* first check if it's already in nonsynced part of transaction log */
+ if (t->extra_intros != NULL) {
+ id_map = buffer_get_data(t->extra_intros, &size);
+ size /= sizeof(*id_map);
+
+ if (data_id < size && id_map[data_id] != (uint32_t)-1) {
+ *idx_r = id_map[data_id];
+ return 0;
+ }
+ }
+ *idx_r = t->extra_intros_max_id++;
+
+ einfo = t->view->index->extra_infos->data;
+ einfo += data_id;
+
+ /* nope, register */
+ t_push();
+ buf = buffer_create_dynamic(pool_datastack_create(), 128, (size_t)-1);
+ intro = buffer_append_space_unsafe(buf, sizeof(*intro));
+ intro->data_id = *idx_r;
+ intro->hdr_size = einfo->hdr_size;
+ intro->record_size = einfo->record_size;
+ intro->name_size = strlen(einfo->name);
+ buffer_append(buf, einfo->name, intro->name_size);
+
+ ret = log_append_buffer(file, buf, NULL, MAIL_TRANSACTION_EXTRA_INTRO,
+ t->view->external);
+ t_pop();
+ return ret;
+}
+
int mail_transaction_log_append(struct mail_index_transaction *t,
uint32_t *log_file_seq_r,
uoff_t *log_file_offset_r)
@@ -1125,8 +1202,10 @@
struct mail_transaction_log_file *file;
struct mail_index_header idx_hdr;
uoff_t append_offset;
- buffer_t *hdr_buf;
+ buffer_t *hdr_buf, **updates;
unsigned int i, lock_id;
+ uint32_t idx;
+ size_t size;
int ret;
index = mail_index_view_get_index(view);
@@ -1189,8 +1268,13 @@
t->cache_updates = NULL;
}
+ t->extra_intros_max_id = t->view->index->map->extra_infos == NULL ? 0 :
+ (t->view->index->map->extra_infos->used /
+ sizeof(struct mail_index_extra_record_info));
+
if (t->appends != NULL ||
- (t->cache_updates != NULL && t->new_cache_file_seq == 0)) {
+ (t->cache_updates != NULL && t->new_cache_file_seq == 0) ||
+ (t->extra_rec_updates != NULL && t->extra_rec_updates->used > 0)) {
if (mail_transaction_log_scan_pending(log, t) < 0) {
if (!log->index->log_locked)
mail_transaction_log_file_unlock(file);
@@ -1200,15 +1284,7 @@
ret = 0;
if (t->appends != NULL) {
- struct mail_transaction_append_header hdr;
-
- memset(&hdr, 0, sizeof(hdr));
- hdr.record_size = t->append_record_size;
-
- hdr_buf = buffer_create_data(pool_datastack_create(),
- &hdr, sizeof(hdr));
- buffer_set_used_size(hdr_buf, sizeof(hdr));
- ret = log_append_buffer(file, t->appends, hdr_buf,
+ ret = log_append_buffer(file, t->appends, NULL,
MAIL_TRANSACTION_APPEND,
view->external);
}
@@ -1228,16 +1304,32 @@
view->external);
}
+ if (t->extra_rec_updates == NULL) {
+ updates = NULL;
+ size = 0;
+ } else {
+ updates = buffer_get_modifyable_data(t->extra_rec_updates,
+ &size);
+ size /= sizeof(*updates);
+ }
+
hdr_buf = buffer_create_data(pool_datastack_create(),
&extra_rec_hdr, sizeof(extra_rec_hdr));
buffer_set_used_size(hdr_buf, sizeof(extra_rec_hdr));
- for (i = 0; i < view->index->extra_records_count; i++) {
- if (t->extra_rec_updates[i] == NULL || ret != 0)
+ for (i = 0; i < size && ret == 0; i++) {
+ if (updates[i] == NULL)
continue;
- /* FIXME: do data_id mapping conversion */
- extra_rec_hdr.data_id = i;
- ret = log_append_buffer(file, t->extra_rec_updates[i], hdr_buf,
+ if (!mail_index_map_get_extra_info_idx(index->map, i, &idx)) {
+ /* new one */
+ ret = mail_transaction_log_register_extra(file, t, i,
+ &idx);
+ if (ret < 0)
+ break;
+ }
+
+ extra_rec_hdr.data_id = idx;
+ ret = log_append_buffer(file, updates[i], hdr_buf,
MAIL_TRANSACTION_EXTRA_REC_UPDATE,
view->external);
}
Index: mail-transaction-log.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.h,v
retrieving revision 1.16
retrieving revision 1.17
diff -u -d -r1.16 -r1.17
--- mail-transaction-log.h 31 Jul 2004 04:35:06 -0000 1.16
+++ mail-transaction-log.h 5 Sep 2004 17:53:45 -0000 1.17
@@ -19,7 +19,10 @@
MAIL_TRANSACTION_CACHE_RESET = 0x00000008,
MAIL_TRANSACTION_CACHE_UPDATE = 0x00000010,
MAIL_TRANSACTION_HEADER_UPDATE = 0x00000020,
- MAIL_TRANSACTION_EXTRA_REC_UPDATE = 0x00000040,
+ MAIL_TRANSACTION_EXTRA_INTRO = 0x00000040,
+ MAIL_TRANSACTION_EXTRA_RESET = 0x00000080,
+ MAIL_TRANSACTION_EXTRA_HDR_UPDATE = 0x00000100,
+ MAIL_TRANSACTION_EXTRA_REC_UPDATE = 0x00000200,
MAIL_TRANSACTION_TYPE_MASK = 0x0000ffff,
@@ -65,8 +68,23 @@
/* unsigned char data[]; */
};
+struct mail_transaction_extra_intro {
+ uint32_t data_id; /* must be first */
+ uint32_t hdr_size;
+ uint16_t record_size;
+ uint16_t name_size;
+ /* unsigned char name[]; */
+};
+
+struct mail_transaction_extra_hdr_update {
+ uint32_t data_id; /* must be first */
+ uint16_t offset;
+ uint16_t size;
+ /* unsigned char data[]; */
+};
+
struct mail_transaction_extra_rec_header {
- uint32_t data_id;
+ uint32_t data_id; /* must be first */
};
struct mail_transaction_extra_rec_update {
@@ -74,10 +92,6 @@
/* unsigned char data[]; */
};
-struct mail_transaction_append_header {
- uint32_t record_size;
-};
-
struct mail_transaction_log *
mail_transaction_log_open_or_create(struct mail_index *index);
void mail_transaction_log_close(struct mail_transaction_log *log);
Index: mail-transaction-util.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-util.c,v
retrieving revision 1.16
retrieving revision 1.17
diff -u -d -r1.16 -r1.17
--- mail-transaction-util.c 31 Jul 2004 04:35:06 -0000 1.16
+++ mail-transaction-util.c 5 Sep 2004 17:53:45 -0000 1.17
@@ -25,6 +25,10 @@
{ MAIL_TRANSACTION_CACHE_UPDATE, 0,
sizeof(struct mail_transaction_cache_update) },
{ MAIL_TRANSACTION_HEADER_UPDATE, 0, 1 }, /* variable size, use 1 */
+ { MAIL_TRANSACTION_EXTRA_INTRO, 0, 1 },
+ { MAIL_TRANSACTION_EXTRA_RESET, 0,
+ sizeof(struct mail_transaction_extra_rec_header) },
+ { MAIL_TRANSACTION_EXTRA_HDR_UPDATE, 0, 1 },
{ MAIL_TRANSACTION_EXTRA_REC_UPDATE, 0, 1 },
{ 0, 0, 0 }
};
@@ -54,29 +58,26 @@
return type;
}
-int mail_transaction_map(struct mail_index *index,
+int mail_transaction_map(struct mail_index_map *map,
const struct mail_transaction_header *hdr,
const void *data,
- struct mail_transaction_map_functions *map,
+ struct mail_transaction_map_functions *func_map,
void *context)
{
int ret = 0;
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_APPEND: {
- const struct mail_transaction_append_header *append_hdr = data;
const struct mail_index_record *rec, *end;
- if (map->append == NULL)
+ if (func_map->append == NULL)
break;
- rec = CONST_PTR_OFFSET(data, sizeof(*append_hdr));
end = CONST_PTR_OFFSET(data, hdr->size);
- while (rec != end) {
- ret = map->append(append_hdr, rec, context);
+ for (rec = data; rec != end; rec++) {
+ ret = func_map->append(rec, context);
if (ret <= 0)
break;
- rec = CONST_PTR_OFFSET(rec, append_hdr->record_size);
}
break;
}
@@ -84,12 +85,12 @@
case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
const struct mail_transaction_expunge *rec, *end;
- if (map->expunge == NULL)
+ if (func_map->expunge == NULL)
break;
end = CONST_PTR_OFFSET(data, hdr->size);
for (rec = data; rec != end; rec++) {
- ret = map->expunge(rec, context);
+ ret = func_map->expunge(rec, context);
if (ret <= 0)
break;
}
@@ -98,33 +99,33 @@
case MAIL_TRANSACTION_FLAG_UPDATE: {
const struct mail_transaction_flag_update *rec, *end;
- if (map->flag_update == NULL)
+ if (func_map->flag_update == NULL)
break;
end = CONST_PTR_OFFSET(data, hdr->size);
for (rec = data; rec != end; rec++) {
- ret = map->flag_update(rec, context);
+ ret = func_map->flag_update(rec, context);
if (ret <= 0)
break;
}
break;
}
case MAIL_TRANSACTION_CACHE_RESET: {
- const struct mail_transaction_cache_reset *u = data;
+ const struct mail_transaction_cache_reset *rec = data;
- if (map->cache_reset != NULL)
- ret = map->cache_reset(u, context);
+ if (func_map->cache_reset != NULL)
+ ret = func_map->cache_reset(rec, context);
break;
}
case MAIL_TRANSACTION_CACHE_UPDATE: {
const struct mail_transaction_cache_update *rec, *end;
- if (map->cache_update == NULL)
+ if (func_map->cache_update == NULL)
break;
end = CONST_PTR_OFFSET(data, hdr->size);
for (rec = data; rec != end; rec++) {
- ret = map->cache_update(rec, context);
+ ret = func_map->cache_update(rec, context);
if (ret <= 0)
break;
}
@@ -134,12 +135,61 @@
const struct mail_transaction_header_update *rec;
unsigned int i;
- if (map->header_update == NULL)
+ if (func_map->header_update == NULL)
break;
for (i = 0; i < hdr->size; ) {
rec = CONST_PTR_OFFSET(data, i);
- ret = map->header_update(rec, context);
+ ret = func_map->header_update(rec, context);
+ if (ret <= 0)
+ break;
+
+ i += sizeof(*rec) + rec->size;
+ }
+ break;
+ }
+ case MAIL_TRANSACTION_EXTRA_INTRO: {
+ const struct mail_transaction_extra_intro *rec = data;
+ unsigned int i;
+
+ if (func_map->extra_intro == NULL)
+ break;
+
+ for (i = 0; i < hdr->size; ) {
+ rec = CONST_PTR_OFFSET(data, i);
+ ret = func_map->extra_intro(rec, context);
+ if (ret <= 0)
+ break;
+
+ i += sizeof(*rec) + rec->name_size;
+ }
+ break;
+ }
+ case MAIL_TRANSACTION_EXTRA_RESET: {
+ const struct mail_transaction_extra_rec_header *rec = data;
+ unsigned int i, size;
+
+ if (func_map->extra_reset == NULL)
+ break;
+
+ size = hdr->size / sizeof(*rec);
+ for (i = 0; i < size; i++) {
+ ret = func_map->extra_reset(&rec[i], context);
+ if (ret <= 0)
+ break;
+ }
+ break;
+ }
+ case MAIL_TRANSACTION_EXTRA_HDR_UPDATE: {
+ const struct mail_transaction_extra_hdr_update *rec = data;
+ unsigned int i;
+
+ if (func_map->extra_hdr_update == NULL)
+ break;
+
+ for (i = 0; i < hdr->size; ) {
+ rec = CONST_PTR_OFFSET(data, i);
+ ret = func_map->extra_hdr_update(rec, context);
if (ret <= 0)
break;
@@ -150,19 +200,31 @@
case MAIL_TRANSACTION_EXTRA_REC_UPDATE: {
const struct mail_transaction_extra_rec_header *ehdr = data;
const struct mail_transaction_extra_rec_update *rec, *end;
+ const struct mail_index_extra_record_info *einfo;
unsigned int record_size;
- if (map->extra_rec_update == NULL)
+ if (func_map->extra_rec_update == NULL)
break;
- i_assert(ehdr->data_id < index->extra_records_count);
- record_size = sizeof(*ehdr) +
- index->extra_records[ehdr->data_id].size;
-
rec = CONST_PTR_OFFSET(data, sizeof(*ehdr));
+
+ if (map->extra_infos == NULL ||
+ ehdr->data_id >= map->extra_infos->used / sizeof(*einfo)) {
+ /* broken. let the extra_rec_update handler do the
+ error handling. */
+ ret = func_map->extra_rec_update(ehdr, rec, context);
+ if (ret >= 0)
+ i_unreached();
+ break;
+ }
+
+ einfo = map->extra_infos->data;
+ einfo += ehdr->data_id;
+ record_size = sizeof(*ehdr) + einfo->record_size;
+
end = CONST_PTR_OFFSET(data, hdr->size);
while (rec != end) {
- ret = map->extra_rec_update(ehdr, rec, context);
+ ret = func_map->extra_rec_update(ehdr, rec, context);
if (ret <= 0)
break;
Index: mail-transaction-util.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-util.h,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- mail-transaction-util.h 24 Jun 2004 11:10:41 -0000 1.8
+++ mail-transaction-util.h 5 Sep 2004 17:53:45 -0000 1.9
@@ -10,8 +10,7 @@
struct mail_transaction_map_functions {
int (*expunge)(const struct mail_transaction_expunge *e, void *context);
- int (*append)(const struct mail_transaction_append_header *hdr,
- const struct mail_index_record *rec, void *context);
+ int (*append)(const struct mail_index_record *rec, void *context);
int (*flag_update)(const struct mail_transaction_flag_update *u,
void *context);
int (*cache_reset)(const struct mail_transaction_cache_reset *u,
@@ -20,6 +19,13 @@
void *context);
int (*header_update)(const struct mail_transaction_header_update *u,
void *context);
+ int (*extra_intro)(const struct mail_transaction_extra_intro *u,
+ void *context);
+ int (*extra_reset)(const struct mail_transaction_extra_rec_header *u,
+ void *context);
+ int (*extra_hdr_update)
+ (const struct mail_transaction_extra_hdr_update *u,
+ void *context);
int (*extra_rec_update)
(const struct mail_transaction_extra_rec_header *hdr,
const struct mail_transaction_extra_rec_update *u,
@@ -31,10 +37,10 @@
enum mail_transaction_type
mail_transaction_type_mask_get(enum mail_index_sync_type sync_type);
-int mail_transaction_map(struct mail_index *index,
+int mail_transaction_map(struct mail_index_map *map,
const struct mail_transaction_header *hdr,
const void *data,
- struct mail_transaction_map_functions *map,
+ struct mail_transaction_map_functions *func_map,
void *context);
void
More information about the dovecot-cvs
mailing list