[dovecot-cvs] dovecot/src/lib-index mail-cache-compress.c, 1.12,
1.13 mail-cache-decisions.c, 1.5, 1.6 mail-cache-fields.c, 1.1,
1.2 mail-cache-lookup.c, 1.14, 1.15 mail-cache-private.h, 1.10,
1.11 mail-cache-transaction.c, 1.15, 1.16 mail-cache.c, 1.41,
1.42 mail-cache.h, 1.19, 1.20
cras at dovecot.org
cras at dovecot.org
Tue Jul 20 19:50:58 EEST 2004
Update of /home/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv22286/lib-index
Modified Files:
mail-cache-compress.c mail-cache-decisions.c
mail-cache-fields.c mail-cache-lookup.c mail-cache-private.h
mail-cache-transaction.c mail-cache.c mail-cache.h
Log Message:
Cache fixes. Decisions are saved again.
Index: mail-cache-compress.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-compress.c,v
retrieving revision 1.12
retrieving revision 1.13
diff -u -d -r1.12 -r1.13
--- mail-cache-compress.c 18 Jul 2004 02:25:07 -0000 1.12
+++ mail-cache-compress.c 20 Jul 2004 16:50:56 -0000 1.13
@@ -32,7 +32,8 @@
*field_seen = ctx->field_seen_value;
field = view->cache->file_field_map[file_field];
- dec = view->cache->fields[field].decision & ~MAIL_CACHE_DECISION_FORCED;
+ dec = view->cache->fields[field].field.decision &
+ ~MAIL_CACHE_DECISION_FORCED;
if (ctx->new_msg) {
if (dec == MAIL_CACHE_DECISION_NO)
return 1;
@@ -43,7 +44,7 @@
buffer_append(ctx->buffer, &file_field, sizeof(file_field));
- if (view->cache->fields[field].field_size == (unsigned int)-1) {
+ if (view->cache->fields[field].field.field_size == (unsigned int)-1) {
size32 = (uint32_t)data_size;
buffer_append(ctx->buffer, &size32, sizeof(size32));
}
@@ -65,7 +66,6 @@
struct mail_cache_record cache_rec;
struct ostream *output;
buffer_t *buffer;
- size_t size;
uint32_t message_count, seq, first_new_seq, old_offset;
uoff_t offset;
@@ -93,14 +93,12 @@
hdr.version = MAIL_CACHE_VERSION;
hdr.indexid = idx_hdr->indexid;
hdr.file_seq = idx_hdr->cache_file_seq + 1;
+ o_stream_send(output, &hdr, sizeof(hdr));
if (cache->fields_count != 0) {
hdr.field_header_offset =
- mail_cache_uint32_to_offset(sizeof(hdr));
- }
- o_stream_send(output, &hdr, sizeof(hdr));
+ mail_cache_uint32_to_offset(output->offset);
- if (cache->fields_count != 0) {
t_push();
buffer = buffer_create_dynamic(pool_datastack_create(),
256, (size_t)-1);
Index: mail-cache-decisions.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-decisions.c,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- mail-cache-decisions.c 18 Jul 2004 02:25:07 -0000 1.5
+++ mail-cache-decisions.c 20 Jul 2004 16:50:56 -0000 1.6
@@ -79,7 +79,7 @@
i_assert(field < cache->fields_count);
- if (cache->fields[field].decision != MAIL_CACHE_DECISION_TEMP) {
+ if (cache->fields[field].field.decision != MAIL_CACHE_DECISION_TEMP) {
/* a) forced decision
b) not cached, mail_cache_decision_add() will handle this
c) permanently cached already, okay. */
@@ -106,7 +106,7 @@
client with no local cache. if it was just a new client
generating the local cache for the first time, we'll
drop back to TEMP within few months. */
- cache->fields[field].decision = MAIL_CACHE_DECISION_YES;
+ cache->fields[field].field.decision = MAIL_CACHE_DECISION_YES;
cache->field_header_write_pending = TRUE;
} else {
cache->fields[field].uid_highwater = uid;
@@ -124,14 +124,14 @@
if (MAIL_CACHE_IS_UNUSABLE(cache))
return;
- if (cache->fields[field].decision != MAIL_CACHE_DECISION_NO) {
+ if (cache->fields[field].field.decision != MAIL_CACHE_DECISION_NO) {
/* a) forced decision
b) we're already caching it, so it just wasn't in cache */
return;
}
/* field used the first time */
- cache->fields[field].decision = MAIL_CACHE_DECISION_TEMP;
+ cache->fields[field].field.decision = MAIL_CACHE_DECISION_TEMP;
cache->field_header_write_pending = TRUE;
if (mail_index_lookup_uid(view->view, seq, &uid) == 0)
Index: mail-cache-fields.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-fields.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mail-cache-fields.c 18 Jul 2004 02:25:07 -0000 1.1
+++ mail-cache-fields.c 20 Jul 2004 16:50:56 -0000 1.2
@@ -3,6 +3,7 @@
#include "lib.h"
#include "buffer.h"
#include "hash.h"
+#include "write-full.h"
#include "mail-cache-private.h"
#include <stddef.h>
@@ -50,24 +51,24 @@
continue;
/* new index - save it */
- cache->fields[idx] = fields[i];
- cache->fields[idx].name =
+ cache->fields[idx].field = fields[i];
+ cache->fields[idx].field.name =
p_strdup(cache->field_pool, fields[i].name);
cache->field_file_map[idx] = (uint32_t)-1;
- switch (cache->fields[idx].type) {
+ switch (cache->fields[idx].field.type) {
case MAIL_CACHE_FIELD_FIXED_SIZE:
case MAIL_CACHE_FIELD_BITMASK:
break;
case MAIL_CACHE_FIELD_VARIABLE_SIZE:
case MAIL_CACHE_FIELD_STRING:
case MAIL_CACHE_FIELD_HEADER:
- cache->fields[idx].field_size = (unsigned int)-1;
+ cache->fields[idx].field.field_size = (unsigned int)-1;
break;
}
hash_insert(cache->field_name_hash,
- (char *)cache->fields[idx].name,
+ (char *)cache->fields[idx].field.name,
POINTER_CAST(idx));
}
cache->fields_count = new_idx;
@@ -101,6 +102,11 @@
next_offset =
mail_cache_offset_to_uint32(cache->hdr->field_header_offset);
while (next_offset != 0) {
+ if (next_offset == offset) {
+ mail_cache_set_corrupted(cache,
+ "next_offset in field header loops");
+ return -1;
+ }
offset = next_offset;
if (mail_cache_map(cache, offset,
@@ -123,6 +129,7 @@
const uint32_t *last_used, *sizes;
const uint8_t *types, *decisions;
const char *p, *names, *end;
+ void *orig_key, *orig_value;
uint32_t offset, i;
if (mail_cache_header_fields_get_offset(cache, &offset) < 0)
@@ -160,11 +167,15 @@
field_hdr->fields_count * sizeof(unsigned int));
cache->file_fields_count = field_hdr->fields_count;
- last_used = MAIL_CACHE_FIELD_LAST_USED(field_hdr);
- sizes = MAIL_CACHE_FIELD_SIZE(field_hdr);
- types = MAIL_CACHE_FIELD_TYPE(field_hdr);
- decisions = MAIL_CACHE_FIELD_DECISION(field_hdr);
- names = MAIL_CACHE_FIELD_NAMES(field_hdr);
+ last_used = CONST_PTR_OFFSET(field_hdr, MAIL_CACHE_FIELD_LAST_USED());
+ sizes = CONST_PTR_OFFSET(field_hdr,
+ MAIL_CACHE_FIELD_SIZE(field_hdr->fields_count));
+ types = CONST_PTR_OFFSET(field_hdr,
+ MAIL_CACHE_FIELD_TYPE(field_hdr->fields_count));
+ decisions = CONST_PTR_OFFSET(field_hdr,
+ MAIL_CACHE_FIELD_DECISION(field_hdr->fields_count));
+ names = CONST_PTR_OFFSET(field_hdr,
+ MAIL_CACHE_FIELD_NAMES(field_hdr->fields_count));
end = CONST_PTR_OFFSET(field_hdr, field_hdr->size);
/* clear the old mapping */
@@ -174,81 +185,152 @@
memset(&field, 0, sizeof(field));
for (i = 0; i < field_hdr->fields_count; i++) {
for (p = names; p != end && *p != '\0'; p++) ;
- if (p == end) {
+ if (p == end || *names == '\0') {
mail_cache_set_corrupted(cache,
"field header names corrupted");
return -1;
}
- field.name = names;
- field.type = types[i];
- field.field_size = sizes[i];
- field.decision = decisions[i];
- field.last_used = (time_t)last_used[i];
- mail_cache_register_fields(cache, &field, 1);
+ if (hash_lookup_full(cache->field_name_hash, names,
+ &orig_key, &orig_value)) {
+ /* already exists, see if decision can be updated */
+ field.idx = POINTER_CAST_TO(orig_value, unsigned int);
+ if (!cache->fields[field.idx].decision_dirty) {
+ cache->fields[field.idx].field.decision =
+ decisions[i];
+ }
+ if (cache->fields[field.idx].field.type != types[i]) {
+ mail_cache_set_corrupted(cache,
+ "registered field type changed");
+ return -1;
+ }
+ } else {
+ field.name = names;
+ field.type = types[i];
+ field.field_size = sizes[i];
+ field.decision = decisions[i];
+ mail_cache_register_fields(cache, &field, 1);
+ }
cache->field_file_map[field.idx] = i;
cache->file_field_map[i] = field.idx;
- names = p + 1;
+ /* update last_used if it's newer than ours */
+ if ((time_t)last_used[i] > cache->fields[field.idx].last_used) {
+ cache->fields[field.idx].last_used =
+ (time_t)last_used[i];
+ }
+
+ names = p + 1;
}
return 0;
}
+static void copy_to_buf(struct mail_cache *cache, buffer_t *dest,
+ size_t offset, size_t size)
+{
+ const void *data;
+ unsigned int i, field;
+
+ for (i = 0; i < cache->file_fields_count; i++) {
+ field = cache->file_field_map[i];
+ data = CONST_PTR_OFFSET(&cache->fields[field], offset);
+ buffer_append(dest, data, size);
+ }
+ for (i = 0; i < cache->fields_count; i++) {
+ if (cache->field_file_map[i] != (uint32_t)-1)
+ continue;
+ data = CONST_PTR_OFFSET(&cache->fields[i], offset);
+ buffer_append(dest, data, size);
+ }
+}
+
int mail_cache_header_fields_update(struct mail_cache *cache)
{
int locked = cache->locked;
+ buffer_t *buffer;
+ uint32_t i, offset;
+ int ret = 0;
if (!locked) {
if (mail_cache_lock(cache) <= 0)
return -1;
}
- // FIXME
+ if (mail_cache_header_fields_read(cache) < 0 ||
+ mail_cache_header_fields_get_offset(cache, &offset) < 0) {
+ mail_cache_unlock(cache);
+ return -1;
+ }
+
+ t_push();
+ buffer = buffer_create_dynamic(pool_datastack_create(),
+ 256, (size_t)-1);
+
+ copy_to_buf(cache, buffer,
+ offsetof(struct mail_cache_field_private, last_used),
+ sizeof(uint32_t));
+ ret = pwrite_full(cache->fd, buffer_get_data(buffer, NULL),
+ sizeof(uint32_t) * cache->file_fields_count,
+ offset + MAIL_CACHE_FIELD_LAST_USED());
+ if (ret == 0) {
+ buffer_set_used_size(buffer, 0);
+ copy_to_buf(cache, buffer,
+ offsetof(struct mail_cache_field, decision),
+ sizeof(uint8_t));
+
+ ret = pwrite_full(cache->fd, buffer_get_data(buffer, NULL),
+ sizeof(uint8_t) * cache->file_fields_count, offset +
+ MAIL_CACHE_FIELD_DECISION(cache->file_fields_count));
+
+ if (ret == 0) {
+ for (i = 0; i < cache->fields_count; i++)
+ cache->fields[i].decision_dirty = FALSE;
+ }
+ }
+ t_pop();
+
+ if (ret == 0)
+ cache->field_header_write_pending = FALSE;
if (!locked)
mail_cache_unlock(cache);
+ return ret;
}
-#define UGLY_COPY_MACRO(field_name, type) \
- for (i = 0; i < cache->file_fields_count; i++) { \
- field = cache->file_field_map[i]; \
- field_name = (type)cache->fields[field].field_name; \
- buffer_append(dest, &field_name, sizeof(field_name)); \
- } \
- for (i = 0; i < cache->fields_count; i++) { \
- if (cache->field_file_map[i] != (uint32_t)-1) \
- continue; \
- field_name = (type)cache->fields[i].field_name; \
- buffer_append(dest, &field_name, sizeof(field_name)); \
- }
-
void mail_cache_header_fields_get(struct mail_cache *cache, buffer_t *dest)
{
struct mail_cache_header_fields hdr;
unsigned int field;
const char *name;
- uint32_t i, last_used, field_size;
- uint8_t type, decision;
+ uint32_t i;
memset(&hdr, 0, sizeof(hdr));
hdr.fields_count = cache->fields_count;
buffer_append(dest, &hdr, sizeof(hdr));
/* we have to keep the field order for the existing fields. */
- UGLY_COPY_MACRO(last_used, uint32_t);
- UGLY_COPY_MACRO(field_size, uint32_t);
- UGLY_COPY_MACRO(type, uint8_t);
- UGLY_COPY_MACRO(decision, uint8_t);
+ copy_to_buf(cache, dest,
+ offsetof(struct mail_cache_field_private, last_used),
+ sizeof(uint32_t));
+ copy_to_buf(cache, dest, offsetof(struct mail_cache_field, field_size),
+ sizeof(uint32_t));
+ copy_to_buf(cache, dest, offsetof(struct mail_cache_field, type),
+ sizeof(uint8_t));
+ copy_to_buf(cache, dest, offsetof(struct mail_cache_field, decision),
+ sizeof(uint8_t));
+
+ i_assert(buffer_get_used_size(dest) == sizeof(hdr) +
+ (sizeof(uint32_t)*2 + 2) * hdr.fields_count);
for (i = 0; i < cache->file_fields_count; i++) {
field = cache->file_field_map[i];
- name = cache->fields[field].name;
+ name = cache->fields[field].field.name;
buffer_append(dest, name, strlen(name)+1);
}
for (i = 0; i < cache->fields_count; i++) {
if (cache->field_file_map[i] != (uint32_t)-1)
continue;
- name = cache->fields[i].name;
+ name = cache->fields[i].field.name;
buffer_append(dest, name, strlen(name)+1);
}
Index: mail-cache-lookup.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-lookup.c,v
retrieving revision 1.14
retrieving revision 1.15
diff -u -d -r1.14 -r1.15
--- mail-cache-lookup.c 18 Jul 2004 02:25:07 -0000 1.14
+++ mail-cache-lookup.c 20 Jul 2004 16:50:56 -0000 1.15
@@ -103,7 +103,7 @@
}
field = cache->file_field_map[file_field];
- data_size = cache->fields[field].field_size;
+ data_size = cache->fields[field].field.field_size;
if (data_size == (unsigned int)-1) {
data_size = *((const uint32_t *)
CONST_PTR_OFFSET(cache_rec, pos));
@@ -225,7 +225,7 @@
{
i_assert(field < cache->fields_count);
- return cache->fields[field].decision;
+ return cache->fields[field].field.decision;
}
struct mail_cache_lookup_context {
Index: mail-cache-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-private.h,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- mail-cache-private.h 18 Jul 2004 02:25:07 -0000 1.10
+++ mail-cache-private.h 20 Jul 2004 16:50:56 -0000 1.11
@@ -73,20 +73,16 @@
#endif
};
-#define MAIL_CACHE_FIELD_LAST_USED(field_hdr) \
- CONST_PTR_OFFSET(field_hdr, sizeof(uint32_t) * 3)
-#define MAIL_CACHE_FIELD_SIZE(field_hdr) \
- CONST_PTR_OFFSET(MAIL_CACHE_FIELD_LAST_USED(field_hdr), \
- sizeof(uint32_t) * (field_hdr)->fields_count)
-#define MAIL_CACHE_FIELD_TYPE(field_hdr) \
- CONST_PTR_OFFSET(MAIL_CACHE_FIELD_SIZE(field_hdr), \
- sizeof(uint32_t) * (field_hdr)->fields_count)
-#define MAIL_CACHE_FIELD_DECISION(field_hdr) \
- CONST_PTR_OFFSET(MAIL_CACHE_FIELD_TYPE(field_hdr), \
- sizeof(uint8_t) * (field_hdr)->fields_count)
-#define MAIL_CACHE_FIELD_NAMES(field_hdr) \
- CONST_PTR_OFFSET(MAIL_CACHE_FIELD_DECISION(field_hdr), \
- sizeof(uint8_t) * (field_hdr)->fields_count)
+#define MAIL_CACHE_FIELD_LAST_USED() \
+ (sizeof(uint32_t) * 3)
+#define MAIL_CACHE_FIELD_SIZE(count) \
+ (MAIL_CACHE_FIELD_LAST_USED() + sizeof(uint32_t) * (count))
+#define MAIL_CACHE_FIELD_TYPE(count) \
+ (MAIL_CACHE_FIELD_SIZE(count) + sizeof(uint32_t) * (count))
+#define MAIL_CACHE_FIELD_DECISION(count) \
+ (MAIL_CACHE_FIELD_TYPE(count) + sizeof(uint8_t) * (count))
+#define MAIL_CACHE_FIELD_NAMES(count) \
+ (MAIL_CACHE_FIELD_DECISION(count) + sizeof(uint8_t) * (count))
struct mail_cache_record {
uint32_t prev_offset;
@@ -105,6 +101,15 @@
uint32_t magic;
};
+struct mail_cache_field_private {
+ struct mail_cache_field field;
+
+ uint32_t uid_highwater;
+ time_t last_used;
+
+ unsigned int decision_dirty:1;
+};
+
struct mail_cache {
struct mail_index *index;
@@ -118,7 +123,7 @@
struct mail_cache_header hdr_copy;
pool_t field_pool;
- struct mail_cache_field *fields;
+ struct mail_cache_field_private *fields;
uint32_t *field_file_map;
unsigned int fields_count;
struct hash_table *field_name_hash; /* name -> idx */
Index: mail-cache-transaction.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-transaction.c,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- mail-cache-transaction.c 18 Jul 2004 02:25:07 -0000 1.15
+++ mail-cache-transaction.c 20 Jul 2004 16:50:56 -0000 1.16
@@ -555,7 +555,7 @@
i_assert(field < ctx->cache->fields_count);
i_assert(data_size < (uint32_t)-1);
- if (ctx->cache->fields[field].decision ==
+ if (ctx->cache->fields[field].field.decision ==
(MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED))
return;
@@ -571,7 +571,7 @@
mail_cache_decision_add(ctx->view, seq, field);
- fixed_size = ctx->cache->fields[field].field_size;
+ fixed_size = ctx->cache->fields[field].field.field_size;
i_assert(fixed_size == (unsigned int)-1 || fixed_size == data_size);
data_size32 = (uint32_t)data_size;
Index: mail-cache.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache.c,v
retrieving revision 1.41
retrieving revision 1.42
diff -u -d -r1.41 -r1.42
--- mail-cache.c 18 Jul 2004 02:25:07 -0000 1.41
+++ mail-cache.c 20 Jul 2004 16:50:56 -0000 1.42
@@ -327,6 +327,9 @@
{
i_assert(cache->locked);
+ if (cache->field_header_write_pending)
+ (void)mail_cache_header_fields_update(cache);
+
cache->locked = FALSE;
if (cache->hdr_modified) {
@@ -358,6 +361,9 @@
void mail_cache_view_close(struct mail_cache_view *view)
{
+ if (view->cache->field_header_write_pending)
+ (void)mail_cache_header_fields_update(view->cache);
+
buffer_free(view->cached_exists_buf);
i_free(view);
}
Index: mail-cache.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache.h,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -d -r1.19 -r1.20
--- mail-cache.h 18 Jul 2004 02:25:07 -0000 1.19
+++ mail-cache.h 20 Jul 2004 16:50:56 -0000 1.20
@@ -36,10 +36,6 @@
enum mail_cache_field_type type;
unsigned int field_size;
enum mail_cache_decision_type decision;
-
- /* internal: */
- uint32_t uid_highwater;
- time_t last_used;
};
struct mail_cache *mail_cache_open_or_create(struct mail_index *index);
More information about the dovecot-cvs
mailing list