[dovecot-cvs] dovecot/src/lib-index mail-index-private.h, 1.45,
1.46 mail-index-sync-keywords.c, 1.1,
1.2 mail-index-sync-private.h, 1.18,
1.19 mail-index-transaction-private.h, 1.19,
1.20 mail-index-transaction.c, 1.37, 1.38 mail-index.c, 1.184,
1.185 mail-index.h, 1.140, 1.141 mail-transaction-log.c, 1.80, 1.81
cras at dovecot.org
cras at dovecot.org
Mon Jan 10 19:37:25 EET 2005
Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv8728/lib-index
Modified Files:
mail-index-private.h mail-index-sync-keywords.c
mail-index-sync-private.h mail-index-transaction-private.h
mail-index-transaction.c mail-index.c mail-index.h
mail-transaction-log.c
Log Message:
Keyword fixes.
Index: mail-index-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-private.h,v
retrieving revision 1.45
retrieving revision 1.46
diff -u -d -r1.45 -r1.46
--- mail-index-private.h 26 Dec 2004 09:12:40 -0000 1.45
+++ mail-index-private.h 10 Jan 2005 17:37:22 -0000 1.46
@@ -61,9 +61,14 @@
};
struct mail_keywords {
+ /* linked list of keyword transactions which use this structure.
+ the list may contain pointers to different transactions. this
+ structure should be freed when there are no more transactions. */
+ struct mail_keyword_transaction *kt;
+
+ unsigned int start, end;
unsigned int count;
- const char **keywords;
- struct mail_index_keyword_transaction *transaction;
+ uint8_t bitmask[4]; /* variable size */
};
struct mail_index_keyword_header {
@@ -136,6 +141,10 @@
uint32_t sync_log_file_seq;
uoff_t sync_log_file_offset;
+ pool_t keywords_pool;
+ buffer_t *keywords_buf;
+ const char *const *keywords;
+
uint32_t keywords_ext_id;
unsigned int last_grow_count;
Index: mail-index-sync-keywords.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync-keywords.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mail-index-sync-keywords.c 26 Dec 2004 09:12:40 -0000 1.1
+++ mail-index-sync-keywords.c 10 Jan 2005 17:37:22 -0000 1.2
@@ -15,6 +15,11 @@
const char *name;
uint32_t i, diff;
+ if (size / sizeof(rec->name_size) < rec->keywords_count) {
+ /* keyword_count is badly broken */
+ return NULL;
+ }
+
buf = buffer_create_static_hard(pool_datastack_create(),
(rec->keywords_count + 1) *
sizeof(const char *));
@@ -44,17 +49,20 @@
return buf->data;
}
-static int keywords_get_missing(struct mail_index *index,
+static int keywords_get_missing(struct mail_index_sync_map_ctx *ctx,
const char *const *keywords,
const char *const **missing_r)
{
- struct mail_index_map *map = index->map;
+ struct mail_index_map *map = ctx->view->index->map;
const char *name;
buffer_t *missing_buf;
unsigned int i;
- if (mail_index_map_read_keywords(index, map) < 0)
- return -1;
+ if (!ctx->keywords_read) {
+ if (mail_index_map_read_keywords(ctx->view->index, map) < 0)
+ return -1;
+ ctx->keywords_read = TRUE;
+ }
missing_buf = buffer_create_dynamic(pool_datastack_create(), 64);
for (; *keywords != NULL; keywords++) {
@@ -167,8 +175,7 @@
ext = map->extensions->data;
ext += ext_id;
- ret = keywords_get_missing(ctx->view->index,
- keywords, &missing);
+ ret = keywords_get_missing(ctx, keywords, &missing);
if (ret == 0 && missing != NULL) {
/* update existing header */
buf = keywords_get_header_buf(map, ext,
@@ -242,6 +249,7 @@
buf, 0, buf->used);
map->hdr_base = map->hdr_copy_buf->data;
+ ctx->keywords_read = FALSE;
return 1;
}
@@ -249,21 +257,15 @@
keywords_make_mask(struct mail_index_map *map, const struct mail_index_ext *ext,
const char *const *keywords)
{
- const struct mail_index_keyword_header *kw_hdr;
- const struct mail_index_keyword_header_rec *kw_rec;
- const char *name, *const *n;
+ const char *const *n;
unsigned char *mask;
unsigned int i;
- kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
- kw_rec = (const void *)(kw_hdr + 1);
- name = (const char *)(kw_rec + kw_hdr->keywords_count);
-
mask = t_malloc0(ext->record_size);
- for (i = 0; i < kw_hdr->keywords_count; i++) {
+ for (i = 0; i < map->keywords_count; i++) {
for (n = keywords; *n != NULL; n++) {
- if (strcmp(name + kw_rec[i].name_offset, *n) == 0) {
+ if (strcmp(map->keywords[i], *n) == 0) {
mask[i / CHAR_BIT] |= 1 << (i % CHAR_BIT);
break;
}
@@ -333,6 +335,11 @@
}
end = CONST_PTR_OFFSET(rec, hdr->size);
+ if (*keywords == NULL && rec->modify_type != MODIFY_REPLACE) {
+ /* adding/removing empty keywords list - do nothing */
+ return 1;
+ }
+
if (rec->modify_type != MODIFY_REMOVE) {
ret = keywords_update_header(ctx, keywords);
if (ret <= 0)
@@ -349,9 +356,30 @@
ext = ctx->view->map->extensions->data;
ext += ext_id;
+ if (ext->record_size == 0) {
+ /* nothing to do */
+ i_assert(*keywords == NULL);
+ i_assert(rec->modify_type == MODIFY_REPLACE);
+ return 1;
+ }
+
+ if (!ctx->keywords_read) {
+ if (mail_index_map_read_keywords(ctx->view->index,
+ ctx->view->map) < 0)
+ return -1;
+ ctx->keywords_read = TRUE;
+ }
+
mask = keywords_make_mask(ctx->view->map, ext, keywords);
while (uid+2 <= end) {
+ if (uid[0] > uid[1] || uid[0] == 0) {
+ mail_transaction_log_view_set_corrupted(
+ ctx->view->log_view,
+ "Keyword record UIDs are broken");
+ return -1;
+ }
+
ret = keywords_update_records(ctx->view, ext, mask,
rec->modify_type,
uid[0], uid[1]);
Index: mail-index-sync-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync-private.h,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- mail-index-sync-private.h 26 Dec 2004 09:12:40 -0000 1.18
+++ mail-index-sync-private.h 10 Jan 2005 17:37:22 -0000 1.19
@@ -48,6 +48,7 @@
unsigned int expunge_handlers_set:1;
unsigned int expunge_handlers_used:1;
unsigned int cur_ext_ignore:1;
+ unsigned int keywords_read:1;
};
extern struct mail_transaction_map_functions mail_index_map_sync_funcs;
Index: mail-index-transaction-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-transaction-private.h,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -d -r1.19 -r1.20
--- mail-index-transaction-private.h 26 Dec 2004 09:12:40 -0000 1.19
+++ mail-index-transaction-private.h 10 Jan 2005 17:37:22 -0000 1.20
@@ -3,15 +3,16 @@
#include "mail-transaction-log.h"
-struct mail_index_keyword_transaction {
- struct mail_keywords keywords;
- pool_t pool;
+struct mail_keyword_transaction {
+ /* mail_keywords points to first mail_index_keyword_transaction.
+ this points to next keyword transaction using the same keywords */
+ struct mail_keyword_transaction *next;
+
enum modify_type modify_type;
- buffer_t *messages;
+ struct mail_index_transaction *transaction;
- /* mail_keywords points to first mail_index_keyword_transaction.
- this points to next transaction using the same keywords */
- struct mail_index_keyword_transaction *next;
+ struct mail_keywords *keywords;
+ buffer_t *messages;
};
struct mail_index_transaction {
@@ -33,7 +34,7 @@
buffer_t *ext_rec_updates; /* buffer[] */
buffer_t *ext_resizes; /* struct mail_transaction_ext_intro[] */
buffer_t *ext_resets; /* uint32_t[] */
- buffer_t *keyword_updates; /* struct mail_index_keyword_transaction[] */
+ buffer_t *keyword_updates; /* struct mail_keyword_transaction[] */
struct mail_cache_transaction_ctx *cache_trans_ctx;
Index: mail-index-transaction.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-transaction.c,v
retrieving revision 1.37
retrieving revision 1.38
diff -u -d -r1.37 -r1.38
--- mail-index-transaction.c 26 Dec 2004 09:12:40 -0000 1.37
+++ mail-index-transaction.c 10 Jan 2005 17:37:22 -0000 1.38
@@ -41,6 +41,27 @@
return t;
}
+static void mail_keyword_transaction_free(struct mail_keyword_transaction *kt)
+{
+ struct mail_keyword_transaction **p;
+
+ for (p = &kt->keywords->kt; *p != NULL; p = &(*p)->next) {
+ if (*p == kt) {
+ *p = kt->next;
+ break;
+ }
+ }
+
+ if (*p == NULL) {
+ /* no transactions left, free mail_keywords */
+ i_assert(kt->keywords->kt == NULL);
+ i_free(kt->keywords);
+ }
+
+ if (kt->messages != NULL)
+ buffer_free(kt->messages);
+}
+
static void mail_index_transaction_free(struct mail_index_transaction *t)
{
buffer_t **recs;
@@ -60,13 +81,13 @@
}
if (t->keyword_updates != NULL) {
- struct mail_index_keyword_transaction *kt;
+ struct mail_keyword_transaction *kt;
kt = buffer_get_modifyable_data(t->keyword_updates, &size);
size /= sizeof(*kt);
for (i = 0; i < size; i++)
- pool_unref(kt[i].pool);
+ mail_keyword_transaction_free(&kt[i]);
buffer_free(t->keyword_updates);
}
@@ -154,7 +175,7 @@
}
if (t->keyword_updates != NULL) {
- struct mail_index_keyword_transaction *kt;
+ struct mail_keyword_transaction *kt;
kt = buffer_get_modifyable_data(t->keyword_updates, &size);
size /= sizeof(*kt);
@@ -716,101 +737,142 @@
}
}
-static int keywords_match(const struct mail_keywords *keywords,
- const char *const *list)
+static struct mail_keyword_transaction *
+mail_keyword_transaction_new(struct mail_index_transaction *t,
+ struct mail_keywords *keywords)
{
- unsigned int i;
+ struct mail_keyword_transaction *kt;
- for (i = 0; list[i] != NULL; i++) {
- if (strcasecmp(keywords->keywords[i], list[i]) != 0)
- return FALSE;
- }
- i_assert(i == keywords->count);
- return TRUE;
+ if (t->keyword_updates == NULL)
+ t->keyword_updates = buffer_create_dynamic(default_pool, 512);
+
+ kt = buffer_append_space_unsafe(t->keyword_updates, sizeof(*kt));
+ kt->transaction = t;
+ kt->keywords = keywords;
+
+ kt->next = keywords->kt;
+ keywords->kt = kt;
+ return kt;
}
-static struct mail_index_keyword_transaction *
-mail_index_keyword_transaction_new(struct mail_index_transaction *t,
- const char *const sorted_keywords[],
- unsigned int count)
+static struct mail_keywords *
+mail_index_keywords_build(struct mail_index *index,
+ const char *const keywords[], unsigned int count)
{
- struct mail_index_keyword_transaction *kt;
- unsigned int i;
+ struct mail_keywords k;
+ const char **missing_keywords, *keyword;
+ buffer_t *keyword_buf;
+ unsigned int i, j, bitmask_offset, missing_count = 0;
+ size_t size;
+ uint8_t *b;
- kt = buffer_append_space_unsafe(t->keyword_updates, sizeof(*kt));
- kt->pool = pool_alloconly_create("keywords", 128);
- kt->keywords.count = count;
- kt->keywords.keywords = p_new(kt->pool, const char *, count + 1);
- kt->keywords.transaction = kt;
+ if (count == 0)
+ return i_new(struct mail_keywords, 1);
+
+ /* @UNSAFE */
+ t_push();
+
+ missing_keywords = t_new(const char *, count + 1);
+ memset(&k, 0, sizeof(k));
+
+ /* keywords are sorted in index. look up the existing ones and add
+ new ones. build a bitmap pointing to them. keywords are never
+ removed from index's keyword list. */
+ bitmask_offset = sizeof(k) - sizeof(k.bitmask);
+ keyword_buf =
+ buffer_create_static_hard(default_pool,
+ bitmask_offset + (count + 7) / 8);
for (i = 0; i < count; i++) {
- kt->keywords.keywords[i] =
- p_strdup(kt->pool, sorted_keywords[i]);
+ for (j = 0; index->keywords[j] != NULL; j++) {
+ if (strcasecmp(keywords[i], index->keywords[j]) == 0)
+ break;
+ }
+
+ if (index->keywords[j] != NULL) {
+ if (keyword_buf->used == 0) {
+ /* first one */
+ k.start = j;
+ buffer_append(keyword_buf, &k, bitmask_offset);
+ }
+ b = buffer_get_space_unsafe(keyword_buf,
+ bitmask_offset +
+ (j - k.start) / 8, 1);
+ *b |= 1 << ((j - k.start) % 8);
+ k.end = j;
+ k.count++;
+ } else {
+ /* arrays are sorted, can't match anymore */
+ missing_keywords[missing_count++] = keywords[i];
+ }
}
- return kt;
-}
+ if (missing_count > 0) {
+ /* add missing keywords. first drop the trailing NULL. */
+ size = index->keywords_buf->used - sizeof(const char *);
+ buffer_set_used_size(index->keywords_buf, size);
-static struct mail_index_keyword_transaction *
-mail_index_keyword_transaction_clone(struct mail_index_transaction *t,
- struct mail_index_keyword_transaction *kt)
-{
- struct mail_index_keyword_transaction *new_kt;
+ j = size / sizeof(const char *);
+ for (; *missing_keywords != NULL; missing_keywords++, j++) {
+ keyword = p_strdup(index->keywords_pool,
+ *missing_keywords);
+ buffer_append(index->keywords_buf,
+ &keyword, sizeof(keyword));
- new_kt = buffer_append_space_unsafe(t->keyword_updates, sizeof(*kt));
- new_kt->pool = pool_alloconly_create("keywords", 128);
- new_kt->keywords = kt->keywords;
- return new_kt;
+ b = buffer_get_space_unsafe(keyword_buf,
+ bitmask_offset +
+ (j - k.start) / 8, 1);
+ *b |= 1 << ((j - k.start) % 8);
+ k.end = j;
+ k.count++;
+ }
+
+ buffer_append_zero(index->keywords_buf, sizeof(const char *));
+ index->keywords = index->keywords_buf->data;
+ }
+ buffer_write(keyword_buf, 0, &k, bitmask_offset);
+
+ t_pop();
+ return buffer_free_without_data(keyword_buf);
}
struct mail_keywords *
mail_index_keywords_create(struct mail_index_transaction *t,
const char *const keywords[])
{
- struct mail_index_keyword_transaction *kt;
- const char **sorted_keywords;
- unsigned int i, count;
- size_t size;
-
- count = strarray_length(keywords);
-
- sorted_keywords = t_new(const char *, count + 1);
- memcpy(sorted_keywords, keywords, count * sizeof(*sorted_keywords));
- qsort(sorted_keywords, count, sizeof(*sorted_keywords), strcasecmp_p);
-
- if (t->keyword_updates == NULL)
- t->keyword_updates = buffer_create_dynamic(default_pool, 512);
-
- kt = buffer_get_modifyable_data(t->keyword_updates, &size);
- size /= sizeof(*kt);
+ struct mail_keywords *k;
+ const char *const null_keywords[] = { NULL };
- /* try to use existing ones */
- for (i = 0; i < size; i++) {
- if (kt[i].keywords.count == count &&
- keywords_match(&kt[i].keywords, sorted_keywords))
- return &kt[i].keywords;
- }
+ if (keywords == NULL)
+ keywords = null_keywords;
- kt = mail_index_keyword_transaction_new(t, sorted_keywords, count);
- return &kt->keywords;
+ k = mail_index_keywords_build(t->view->index, keywords,
+ strarray_length(keywords));
+ (void)mail_keyword_transaction_new(t, k);
+ return k;
}
void mail_index_update_keywords(struct mail_index_transaction *t, uint32_t seq,
enum modify_type modify_type,
- const struct mail_keywords *keywords)
+ struct mail_keywords *keywords)
{
- struct mail_index_keyword_transaction *kt;
+ struct mail_keyword_transaction *kt;
- kt = keywords->transaction;
- while (kt->modify_type != modify_type && kt->messages != NULL) {
- if (kt->next == NULL) {
- kt = mail_index_keyword_transaction_clone(t, kt);
+ i_assert(seq > 0 &&
+ (seq <= mail_index_view_get_messages_count(t->view) ||
+ seq <= t->last_new_seq));
+ i_assert(keywords->count > 0 || modify_type == MODIFY_REPLACE);
+
+ for (kt = keywords->kt; kt != NULL; kt = kt->next) {
+ if (kt->transaction == t &&
+ (kt->modify_type == modify_type || kt->messages == NULL))
break;
- }
- kt = kt->next;
}
+ if (kt == NULL)
+ kt = mail_keyword_transaction_new(t, keywords);
+
if (kt->messages == NULL) {
- kt->messages = buffer_create_dynamic(kt->pool, 32);
+ kt->messages = buffer_create_dynamic(default_pool, 32);
kt->modify_type = modify_type;
buffer_append(kt->messages, &seq, sizeof(seq));
buffer_append(kt->messages, &seq, sizeof(seq));
Index: mail-index.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.184
retrieving revision 1.185
diff -u -d -r1.184 -r1.185
--- mail-index.c 26 Dec 2004 09:12:40 -0000 1.184
+++ mail-index.c 10 Jan 2005 17:37:22 -0000 1.185
@@ -37,6 +37,10 @@
index->keywords_ext_id =
mail_index_ext_register(index, "keywords", 128, 2, 1);
+ index->keywords_pool = pool_alloconly_create("keywords", 512);
+ index->keywords_buf = buffer_create_dynamic(default_pool, 64);
+ buffer_append_zero(index->keywords_buf, sizeof(const char *));
+ index->keywords = index->keywords_buf->data;
return index;
}
@@ -44,10 +48,12 @@
{
mail_index_close(index);
pool_unref(index->extension_pool);
+ pool_unref(index->keywords_pool);
buffer_free(index->sync_handlers);
buffer_free(index->sync_lost_handlers);
buffer_free(index->expunge_handlers);
+ buffer_free(index->keywords_buf);
i_free(index->error);
i_free(index->dir);
Index: mail-index.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.h,v
retrieving revision 1.140
retrieving revision 1.141
diff -u -d -r1.140 -r1.141
--- mail-index.h 26 Dec 2004 09:12:40 -0000 1.140
+++ mail-index.h 10 Jan 2005 17:37:22 -0000 1.141
@@ -292,7 +292,7 @@
/* Update keywords for given message. */
void mail_index_update_keywords(struct mail_index_transaction *t, uint32_t seq,
enum modify_type modify_type,
- const struct mail_keywords *keywords);
+ struct mail_keywords *keywords);
/* Update field in header. */
void mail_index_update_header(struct mail_index_transaction *t,
Index: mail-transaction-log.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.c,v
retrieving revision 1.80
retrieving revision 1.81
diff -u -d -r1.80 -r1.81
--- mail-transaction-log.c 29 Dec 2004 15:48:35 -0000 1.80
+++ mail-transaction-log.c 10 Jan 2005 17:37:22 -0000 1.81
@@ -1191,34 +1191,53 @@
static int log_append_keyword_updates(struct mail_transaction_log_file *file,
struct mail_index_transaction *t)
{
- struct mail_index_keyword_transaction *kt;
- struct mail_transaction_keyword_update *kt_hdr;
+ struct mail_index *index = t->view->index;
+ struct mail_keyword_transaction *kt;
+ struct mail_transaction_keyword_update kt_hdr;
buffer_t *buf;
- size_t i, size;
- unsigned int j;
+ size_t i, size, size_offset, name_offset;
+ unsigned int idx, last_idx, first_keyword;
buf = buffer_create_dynamic(pool_datastack_create(), 128);
kt = buffer_get_modifyable_data(t->keyword_updates, &size);
size /= sizeof(*kt);
for (i = 0; i < size; i++) {
- if (kt[i].messages == NULL || kt[i].keywords.count == 0)
- continue;
-
buffer_set_used_size(buf, 0);
- kt_hdr = buffer_append_space_unsafe(buf, sizeof(*kt_hdr));
- kt_hdr->keywords_count = kt[i].keywords.count;
- kt_hdr->modify_type = kt[i].modify_type;
- kt_hdr->name_size[0] = strlen(kt[i].keywords.keywords[0]);
- for (j = 1; j < kt[i].keywords.count; j++) {
- uint16_t name_size = strlen(kt[i].keywords.keywords[j]);
- buffer_append(buf, &name_size, sizeof(name_size));
- }
- for (j = 0; j < kt[i].keywords.count; j++) {
- const char *name = kt[i].keywords.keywords[j];
- buffer_append(buf, name, strlen(name));
+ memset(&kt_hdr, 0, sizeof(kt_hdr));
+ kt_hdr.keywords_count = kt[i].keywords->count;
+ kt_hdr.modify_type = kt[i].modify_type;
+ buffer_append(buf, &kt_hdr,
+ sizeof(kt_hdr) - sizeof(kt_hdr.name_size));
+
+ size_offset = buf->used;
+ name_offset = buf->used +
+ kt[i].keywords->count * sizeof(uint16_t);
+
+ idx = 0;
+ first_keyword = kt[i].keywords->start;
+ last_idx = kt[i].keywords->end - first_keyword;
+
+ for (; idx <= last_idx; idx++) {
+ uint16_t name_size;
+ const char *keyword;
+
+ if ((kt[i].keywords->bitmask[idx / 8] &
+ (1 << (idx % 8))) == 0)
+ continue;
+
+ keyword = index->keywords[first_keyword + idx];
+
+ name_size = strlen(keyword);
+ buffer_write(buf, size_offset,
+ &name_size, sizeof(name_size));
+ size_offset += sizeof(name_size);
+
+ buffer_write(buf, name_offset, keyword, name_size);
+ name_offset += name_size;
}
+
if ((buf->used % 4) != 0)
buffer_append_zero(buf, 4 - (buf->used % 4));
buffer_append_buf(buf, kt[i].messages, 0, (size_t)-1);
More information about the dovecot-cvs
mailing list