From dovecot at dovecot.org Thu Apr 3 09:55:04 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 09:55:04 +0000 Subject: dovecot-2.2: lib-ssl-iostream: Error handling fix. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/72b50ae2f0c3 changeset: 17183:72b50ae2f0c3 user: Timo Sirainen date: Thu Apr 03 12:50:43 2014 +0300 description: lib-ssl-iostream: Error handling fix. Don't set last_error="(null)" when the connection was simply disconnected. diffstat: src/lib-ssl-iostream/istream-openssl.c | 12 +++++++----- 1 files changed, 7 insertions(+), 5 deletions(-) diffs (25 lines): diff -r d55ae5d268a2 -r 72b50ae2f0c3 src/lib-ssl-iostream/istream-openssl.c --- a/src/lib-ssl-iostream/istream-openssl.c Sat Mar 29 22:45:16 2014 +0200 +++ b/src/lib-ssl-iostream/istream-openssl.c Thu Apr 03 12:50:43 2014 +0300 @@ -71,14 +71,16 @@ /* failed to read anything */ ret = openssl_iostream_handle_error(ssl_io, ret, "SSL_read"); if (ret <= 0) { - if (ret < 0) { + if (ret == 0) + return 0; + if (ssl_io->last_error != NULL) { io_stream_set_error(&stream->iostream, "%s", ssl_io->last_error); - stream->istream.stream_errno = errno; - stream->istream.eof = TRUE; - sstream->seen_eof = TRUE; } - return ret; + stream->istream.stream_errno = errno; + stream->istream.eof = TRUE; + sstream->seen_eof = TRUE; + return -1; } /* we did some BIO I/O, try reading again */ } From dovecot at dovecot.org Thu Apr 3 09:55:04 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 09:55:04 +0000 Subject: dovecot-2.2: i_stream_read(): If stream_errno is set, fail immed... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2e275f4e51c3 changeset: 17184:2e275f4e51c3 user: Timo Sirainen date: Thu Apr 03 12:54:18 2014 +0300 description: i_stream_read(): If stream_errno is set, fail immediately. Previously the stream_errno was reset to 0, but this required the istream implementation to correctly set the stream_errno back, which wasn't often done. diffstat: src/lib/istream.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 72b50ae2f0c3 -r 2e275f4e51c3 src/lib/istream.c --- a/src/lib/istream.c Thu Apr 03 12:50:43 2014 +0300 +++ b/src/lib/istream.c Thu Apr 03 12:54:18 2014 +0300 @@ -152,13 +152,12 @@ size_t old_size; ssize_t ret; - if (unlikely(stream->closed)) { + if (unlikely(stream->closed || stream->stream_errno != 0)) { errno = stream->stream_errno; return -1; } stream->eof = FALSE; - stream->stream_errno = 0; if (_stream->parent != NULL) i_stream_seek(_stream->parent, _stream->parent_expected_offset); From dovecot at dovecot.org Thu Apr 3 11:41:51 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 11:41:51 +0000 Subject: dovecot-2.2: DEBUG: Make sure errno isn't changed by vsnprintf()... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9b50caaa4467 changeset: 17185:9b50caaa4467 user: Timo Sirainen date: Thu Apr 03 14:41:25 2014 +0300 description: DEBUG: Make sure errno isn't changed by vsnprintf()/malloc()/free() implementations. They shouldn't, so this should only be a sanity check when running with devel-checks. Various parts in the code rely on errno not being changed by *_strdup_printf() calls. diffstat: src/lib/data-stack.c | 4 ++++ src/lib/mempool-system.c | 14 ++++++++++++++ src/lib/strfuncs.c | 7 +++++++ 3 files changed, 25 insertions(+), 0 deletions(-) diffs (88 lines): diff -r 2e275f4e51c3 -r 9b50caaa4467 src/lib/data-stack.c --- a/src/lib/data-stack.c Thu Apr 03 12:54:18 2014 +0300 +++ b/src/lib/data-stack.c Thu Apr 03 14:41:25 2014 +0300 @@ -317,6 +317,7 @@ size_t alloc_size; #ifdef DEBUG bool warn = FALSE; + int old_errno = errno; #endif if (unlikely(size == 0 || size > SSIZE_T_MAX)) @@ -390,6 +391,9 @@ had used t_buffer_get(). */ memset(PTR_OFFSET(ret, size), CLEAR_CHR, MEM_ALIGN(size + SENTRY_COUNT) - size); + + /* we rely on errno not changing. it shouldn't. */ + i_assert(errno == old_errno); #endif return ret; } diff -r 2e275f4e51c3 -r 9b50caaa4467 src/lib/mempool-system.c --- a/src/lib/mempool-system.c Thu Apr 03 12:54:18 2014 +0300 +++ b/src/lib/mempool-system.c Thu Apr 03 14:41:25 2014 +0300 @@ -73,6 +73,9 @@ static void *pool_system_malloc(pool_t pool ATTR_UNUSED, size_t size) { void *mem; +#ifdef DEBUG + int old_errno = errno; +#endif if (unlikely(size == 0 || size > SSIZE_T_MAX)) i_panic("Trying to allocate %"PRIuSIZE_T" bytes", size); @@ -86,18 +89,29 @@ i_fatal_status(FATAL_OUTOFMEM, "pool_system_malloc(%"PRIuSIZE_T "): Out of memory", size); } +#ifdef DEBUG + /* we rely on errno not changing. it shouldn't. */ + i_assert(errno == old_errno); +#endif return mem; } static void pool_system_free(pool_t pool ATTR_UNUSED, void *mem ATTR_UNUSED) { +#ifdef DEBUG + int old_errno = errno; +#endif #if !defined(USE_GC) && defined(HAVE_MALLOC_USABLE_SIZE) && defined(DEBUG) safe_memset(mem, CLEAR_CHR, malloc_usable_size(mem)); #endif #ifndef USE_GC free(mem); #endif +#ifdef DEBUG + /* we rely on errno not changing. it shouldn't. */ + i_assert(errno == old_errno); +#endif } static void *pool_system_realloc(pool_t pool ATTR_UNUSED, void *mem, diff -r 2e275f4e51c3 -r 9b50caaa4467 src/lib/strfuncs.c --- a/src/lib/strfuncs.c Thu Apr 03 12:54:18 2014 +0300 +++ b/src/lib/strfuncs.c Thu Apr 03 14:41:25 2014 +0300 @@ -106,6 +106,9 @@ char *tmp; unsigned int init_size; int ret; +#ifdef DEBUG + int old_errno = errno; +#endif VA_COPY(args2, args); @@ -127,6 +130,10 @@ ret = vsnprintf(tmp, *size_r, format, args2); i_assert((unsigned int)ret == *size_r-1); } +#ifdef DEBUG + /* we rely on errno not changing. it shouldn't. */ + i_assert(errno == old_errno); +#endif return tmp; } From dovecot at dovecot.org Thu Apr 3 15:21:57 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 15:21:57 +0000 Subject: dovecot-2.2: dsync: Don't assert-crash if we have the same mailb... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8328d67e2329 changeset: 17186:8328d67e2329 user: Timo Sirainen date: Thu Apr 03 18:21:39 2014 +0300 description: dsync: Don't assert-crash if we have the same mailbox name in conflicting namespaces. diffstat: src/doveadm/dsync/dsync-brain-mailbox-tree.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diffs (19 lines): diff -r 9b50caaa4467 -r 8328d67e2329 src/doveadm/dsync/dsync-brain-mailbox-tree.c --- a/src/doveadm/dsync/dsync-brain-mailbox-tree.c Thu Apr 03 14:41:25 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c Thu Apr 03 18:21:39 2014 +0300 @@ -387,7 +387,14 @@ memcpy(other_node->mailbox_guid, node->mailbox_guid, sizeof(other_node->mailbox_guid)); } - i_assert(other_node->ns == NULL || other_node->ns == node->ns); + if (other_node->ns != node->ns && other_node->ns != NULL) { + /* namespace mismatch for this node. this shouldn't happen + normally, but especially during some misconfigurations it's + possible that one side has created mailboxes that conflict + with another namespace's prefix. since we're here because + one of the mailboxes was deleted, we'll just ignore this. */ + return; + } other_node->ns = node->ns; if (other_del->type != DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE) other_node->existence = DSYNC_MAILBOX_NODE_DELETED; From dovecot at dovecot.org Thu Apr 3 16:52:23 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 16:52:23 +0000 Subject: dovecot-2.2: ioloop: Added io_set_pending() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d2a6f57e174f changeset: 17187:d2a6f57e174f user: Timo Sirainen date: Thu Apr 03 19:47:49 2014 +0300 description: ioloop: Added io_set_pending() diffstat: src/lib/ioloop-epoll.c | 2 +- src/lib/ioloop-kqueue.c | 2 +- src/lib/ioloop-poll.c | 2 +- src/lib/ioloop-private.h | 7 +++++++ src/lib/ioloop-select.c | 2 +- src/lib/ioloop.c | 42 ++++++++++++++++++++++++++++++++++++++++++ src/lib/ioloop.h | 6 ++++++ 7 files changed, 59 insertions(+), 4 deletions(-) diffs (182 lines): diff -r 8328d67e2329 -r d2a6f57e174f src/lib/ioloop-epoll.c --- a/src/lib/ioloop-epoll.c Thu Apr 03 18:21:39 2014 +0300 +++ b/src/lib/ioloop-epoll.c Thu Apr 03 19:47:49 2014 +0300 @@ -161,7 +161,7 @@ i_free(io); } -void io_loop_handler_run(struct ioloop *ioloop) +void io_loop_handler_run_internal(struct ioloop *ioloop) { struct ioloop_handler_context *ctx = ioloop->handler_context; struct epoll_event *events; diff -r 8328d67e2329 -r d2a6f57e174f src/lib/ioloop-kqueue.c --- a/src/lib/ioloop-kqueue.c Thu Apr 03 18:21:39 2014 +0300 +++ b/src/lib/ioloop-kqueue.c Thu Apr 03 19:47:49 2014 +0300 @@ -108,7 +108,7 @@ i_free(io); } -void io_loop_handler_run(struct ioloop *ioloop) +void io_loop_handler_run_internal(struct ioloop *ioloop) { struct ioloop_handler_context *ctx = ioloop->handler_context; struct kevent *events; diff -r 8328d67e2329 -r d2a6f57e174f src/lib/ioloop-poll.c --- a/src/lib/ioloop-poll.c Thu Apr 03 18:21:39 2014 +0300 +++ b/src/lib/ioloop-poll.c Thu Apr 03 19:47:49 2014 +0300 @@ -143,7 +143,7 @@ } } -void io_loop_handler_run(struct ioloop *ioloop) +void io_loop_handler_run_internal(struct ioloop *ioloop) { struct ioloop_handler_context *ctx = ioloop->handler_context; struct pollfd *pollfd; diff -r 8328d67e2329 -r d2a6f57e174f src/lib/ioloop-private.h --- a/src/lib/ioloop-private.h Thu Apr 03 18:21:39 2014 +0300 +++ b/src/lib/ioloop-private.h Thu Apr 03 19:47:49 2014 +0300 @@ -24,6 +24,8 @@ io_loop_time_moved_callback_t *time_moved_callback; time_t next_max_time; + unsigned int io_pending_count; + unsigned int running:1; unsigned int iolooping:1; }; @@ -31,6 +33,9 @@ struct io { enum io_condition condition; unsigned int source_linenum; + /* trigger I/O callback even if OS doesn't think there is input + pending */ + bool pending; io_callback_t *callback; void *context; @@ -79,6 +84,8 @@ void io_loop_handle_timeouts(struct ioloop *ioloop); void io_loop_call_io(struct io *io); +void io_loop_handler_run_internal(struct ioloop *ioloop); + /* I/O handler calls */ void io_loop_handle_add(struct io_file *io); void io_loop_handle_remove(struct io_file *io, bool closed); diff -r 8328d67e2329 -r d2a6f57e174f src/lib/ioloop-select.c --- a/src/lib/ioloop-select.c Thu Apr 03 18:21:39 2014 +0300 +++ b/src/lib/ioloop-select.c Thu Apr 03 19:47:49 2014 +0300 @@ -103,7 +103,7 @@ (FD_ISSET((fd), &(ctx)->tmp_write_fds) && ((cond) & IO_WRITE)) || \ (FD_ISSET((fd), &(ctx)->tmp_except_fds))) -void io_loop_handler_run(struct ioloop *ioloop) +void io_loop_handler_run_internal(struct ioloop *ioloop) { struct ioloop_handler_context *ctx = ioloop->handler_context; struct timeval tv; diff -r 8328d67e2329 -r d2a6f57e174f src/lib/ioloop.c --- a/src/lib/ioloop.c Thu Apr 03 18:21:39 2014 +0300 +++ b/src/lib/ioloop.c Thu Apr 03 19:47:49 2014 +0300 @@ -93,6 +93,11 @@ kqueue code relies on this. */ io->callback = NULL; + if (io->pending) { + i_assert(io->ioloop->io_pending_count > 0); + io->ioloop->io_pending_count--; + } + if (io->ctx != NULL) io_loop_context_unref(&io->ctx); @@ -118,6 +123,16 @@ io_remove_full(io, TRUE); } +void io_set_pending(struct io *io) +{ + i_assert((io->condition & IO_NOTIFY) == 0); + + if (!io->pending) { + io->pending = TRUE; + io->ioloop->io_pending_count++; + } +} + static void timeout_update_next(struct timeout *timeout, struct timeval *tv_now) { if (tv_now == NULL) { @@ -382,6 +397,12 @@ struct ioloop *ioloop = io->ioloop; unsigned int t_id; + if (io->pending) { + i_assert(ioloop->io_pending_count > 0); + ioloop->io_pending_count--; + io->pending = FALSE; + } + if (io->ctx != NULL) io_loop_context_activate(io->ctx); t_id = t_push(); @@ -413,6 +434,24 @@ ioloop->iolooping = FALSE; } +static void io_loop_call_pending(struct ioloop *ioloop) +{ + struct io_file *io; + + for (io = ioloop->io_files; ioloop->io_pending_count > 0; ) { + ioloop->next_io_file = io->next; + if (io->io.pending) + io_loop_call_io(&io->io); + io = ioloop->next_io_file; + } +} + +void io_loop_handler_run(struct ioloop *ioloop) +{ + io_loop_handler_run_internal(ioloop); + io_loop_call_pending(ioloop); +} + void io_loop_stop(struct ioloop *ioloop) { ioloop->running = FALSE; @@ -484,6 +523,7 @@ io->io.source_linenum, io->fd); io_remove(&_io); } + i_assert(ioloop->io_pending_count == 0); while ((item = priorityq_pop(ioloop->timeouts)) != NULL) { struct timeout *to = (struct timeout *)item; @@ -686,6 +726,8 @@ new_io = io_add(old_io_file->fd, old_io->condition, old_io->source_linenum, old_io->callback, old_io->context); + if (old_io->pending) + io_set_pending(new_io); io_remove(_io); return new_io; } diff -r 8328d67e2329 -r d2a6f57e174f src/lib/ioloop.h --- a/src/lib/ioloop.h Thu Apr 03 18:21:39 2014 +0300 +++ b/src/lib/ioloop.h Thu Apr 03 19:47:49 2014 +0300 @@ -67,6 +67,12 @@ With some backends this simply frees the memory. */ void io_remove_closed(struct io **io); +/* Make sure the I/O callback is called by io_loop_run() even if there isn't + any input actually pending currently as seen by the OS. This may be useful + if some of the input has already read into some internal buffer and the + caller wants to handle it the same way as if the fd itself had input. */ +void io_set_pending(struct io *io); + /* Timeout handlers */ struct timeout * timeout_add(unsigned int msecs, unsigned int source_linenum, From dovecot at dovecot.org Thu Apr 3 16:52:23 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 16:52:23 +0000 Subject: dovecot-2.2: Added io_add_istream() and related functionality fo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fd186eff7325 changeset: 17188:fd186eff7325 user: Timo Sirainen date: Thu Apr 03 19:51:52 2014 +0300 description: Added io_add_istream() and related functionality for combining the ioloop/istream. The idea here is that it's possible to implement asynchronous istreams when there isn't a file descriptor that can be used to reliably receive updates. diffstat: src/lib/ioloop-private.h | 3 ++ src/lib/ioloop.c | 63 ++++++++++++++++++++++++++++++++++++++-------- src/lib/ioloop.h | 7 +++++ src/lib/istream-private.h | 8 +++++ src/lib/istream.c | 44 ++++++++++++++++++++++++++++++++ src/lib/istream.h | 6 ++++ 6 files changed, 120 insertions(+), 11 deletions(-) diffs (252 lines): diff -r d2a6f57e174f -r fd186eff7325 src/lib/ioloop-private.h --- a/src/lib/ioloop-private.h Thu Apr 03 19:47:49 2014 +0300 +++ b/src/lib/ioloop-private.h Thu Apr 03 19:51:52 2014 +0300 @@ -52,6 +52,9 @@ int refcount; int fd; + + /* only for io_add_istream(), a bit kludgy to be here.. */ + struct istream *istream; }; struct timeout { diff -r d2a6f57e174f -r fd186eff7325 src/lib/ioloop.c --- a/src/lib/ioloop.c Thu Apr 03 19:47:49 2014 +0300 +++ b/src/lib/ioloop.c Thu Apr 03 19:51:52 2014 +0300 @@ -3,6 +3,7 @@ #include "lib.h" #include "array.h" #include "time-util.h" +#include "istream-private.h" #include "ioloop-private.h" #include @@ -28,10 +29,10 @@ io_loop_handler_init(ioloop, initial_fd_count); } -#undef io_add -struct io *io_add(int fd, enum io_condition condition, - unsigned int source_linenum, - io_callback_t *callback, void *context) +static struct io_file * +io_add_file(int fd, enum io_condition condition, + unsigned int source_linenum, + io_callback_t *callback, void *context) { struct io_file *io; @@ -62,6 +63,31 @@ io->next = io->io.ioloop->io_files; } io->io.ioloop->io_files = io; + return io; +} + +#undef io_add +struct io *io_add(int fd, enum io_condition condition, + unsigned int source_linenum, + io_callback_t *callback, void *context) +{ + struct io_file *io; + + io = io_add_file(fd, condition, source_linenum, callback, context); + return &io->io; +} + +#undef io_add_istream +struct io *io_add_istream(struct istream *input, unsigned int source_linenum, + io_callback_t *callback, void *context) +{ + struct io_file *io; + + io = io_add_file(i_stream_get_fd(input), IO_READ, source_linenum, + callback, context); + io->istream = input; + i_stream_ref(io->istream); + i_stream_set_io(io->istream, &io->io); return &io->io; } @@ -106,6 +132,12 @@ else { struct io_file *io_file = (struct io_file *)io; + if (io_file->istream != NULL) { + i_stream_unset_io(io_file->istream, io); + i_stream_unref(&io_file->istream); + io_file->istream = NULL; + } + io_file_unlink(io_file); io_loop_handle_remove(io_file, closed); } @@ -714,8 +746,8 @@ struct io *io_loop_move_io(struct io **_io) { - struct io *new_io, *old_io = *_io; - struct io_file *old_io_file; + struct io *old_io = *_io; + struct io_file *old_io_file, *new_io_file; i_assert((old_io->condition & IO_NOTIFY) == 0); @@ -723,13 +755,22 @@ return old_io; old_io_file = (struct io_file *)old_io; - new_io = io_add(old_io_file->fd, old_io->condition, - old_io->source_linenum, - old_io->callback, old_io->context); + new_io_file = io_add_file(old_io_file->fd, old_io->condition, + old_io->source_linenum, + old_io->callback, old_io->context); + if (old_io_file->istream != NULL) { + /* reference before io_remove() */ + new_io_file->istream = old_io_file->istream; + i_stream_ref(new_io_file->istream); + } if (old_io->pending) - io_set_pending(new_io); + io_set_pending(&new_io_file->io); io_remove(_io); - return new_io; + if (new_io_file->istream != NULL) { + /* update istream io after it was removed with io_remove() */ + i_stream_set_io(new_io_file->istream, &new_io_file->io); + } + return &new_io_file->io; } struct timeout *io_loop_move_timeout(struct timeout **_timeout) diff -r d2a6f57e174f -r fd186eff7325 src/lib/ioloop.h --- a/src/lib/ioloop.h Thu Apr 03 19:47:49 2014 +0300 +++ b/src/lib/ioloop.h Thu Apr 03 19:51:52 2014 +0300 @@ -7,6 +7,7 @@ struct io; struct timeout; struct ioloop; +struct istream; enum io_condition { IO_READ = 0x01, @@ -60,6 +61,12 @@ io_add_notify(path + \ CALLBACK_TYPECHECK(callback, void (*)(typeof(context))), \ (io_callback_t *)callback, context, io_r) +struct io *io_add_istream(struct istream *input, unsigned int source_linenum, + io_callback_t *callback, void *context) ATTR_NULL(3); +#define io_add_istream(input, callback, context) \ + io_add_istream(input, __LINE__ + \ + CALLBACK_TYPECHECK(callback, void (*)(typeof(context))), \ + (io_callback_t *)callback, context) /* Remove I/O handler, and set io pointer to NULL. */ void io_remove(struct io **io); diff -r d2a6f57e174f -r fd186eff7325 src/lib/istream-private.h --- a/src/lib/istream-private.h Thu Apr 03 19:47:49 2014 +0300 +++ b/src/lib/istream-private.h Thu Apr 03 19:51:52 2014 +0300 @@ -6,6 +6,8 @@ #define I_STREAM_MIN_SIZE IO_BLOCK_SIZE +struct io; + struct istream_private { /* inheritance: */ struct iostream_private iostream; @@ -17,6 +19,7 @@ void (*sync)(struct istream_private *stream); int (*stat)(struct istream_private *stream, bool exact); int (*get_size)(struct istream_private *stream, bool exact, uoff_t *size_r); + void (*switch_ioloop)(struct istream_private *stream); /* data: */ struct istream istream; @@ -24,6 +27,8 @@ int fd; uoff_t abs_start_offset; struct stat statbuf; + /* added by io_add_istream() -> i_stream_set_io() */ + struct io *io; const unsigned char *buffer; unsigned char *w_buffer; /* may be NULL */ @@ -66,4 +71,7 @@ void i_stream_default_seek_nonseekable(struct istream_private *stream, uoff_t v_offset, bool mark); +void i_stream_set_io(struct istream *stream, struct io *io); +void i_stream_unset_io(struct istream *stream, struct io *io); + #endif diff -r d2a6f57e174f -r fd186eff7325 src/lib/istream.c --- a/src/lib/istream.c Thu Apr 03 19:47:49 2014 +0300 +++ b/src/lib/istream.c Thu Apr 03 19:51:52 2014 +0300 @@ -608,6 +608,50 @@ return TRUE; } +void i_stream_set_input_pending(struct istream *stream, bool pending) +{ + if (!pending) + return; + + while (stream->real_stream->parent != NULL) { + i_assert(stream->real_stream->io == NULL); + stream = stream->real_stream->parent; + } + if (stream->real_stream->io != NULL) + io_set_pending(stream->real_stream->io); +} + +void i_stream_switch_ioloop(struct istream *stream) +{ + do { + if (stream->real_stream->switch_ioloop != NULL) + stream->real_stream->switch_ioloop(stream->real_stream); + stream = stream->real_stream->parent; + } while (stream != NULL); +} + +void i_stream_set_io(struct istream *stream, struct io *io) +{ + while (stream->real_stream->parent != NULL) { + i_assert(stream->real_stream->io == NULL); + stream = stream->real_stream->parent; + } + + i_assert(stream->real_stream->io == NULL); + stream->real_stream->io = io; +} + +void i_stream_unset_io(struct istream *stream, struct io *io) +{ + while (stream->real_stream->parent != NULL) { + i_assert(stream->real_stream->io == NULL); + stream = stream->real_stream->parent; + } + + i_assert(stream->real_stream->io == io); + stream->real_stream->io = NULL; +} + static void i_stream_default_set_max_buffer_size(struct iostream_private *stream, size_t max_size) diff -r d2a6f57e174f -r fd186eff7325 src/lib/istream.h --- a/src/lib/istream.h Thu Apr 03 19:47:49 2014 +0300 +++ b/src/lib/istream.h Thu Apr 03 19:51:52 2014 +0300 @@ -157,4 +157,10 @@ bool i_stream_add_data(struct istream *stream, const unsigned char *data, size_t size); +void i_stream_set_input_pending(struct istream *stream, bool pending); + +/* If there are any I/O loop items associated with the stream, move all of + them to current_ioloop. */ +void i_stream_switch_ioloop(struct istream *stream); + #endif From dovecot at dovecot.org Thu Apr 3 16:55:06 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 16:55:06 +0000 Subject: dovecot-2.2: Added istream-timeout, which triggers I/O event and... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b1a756176ed2 changeset: 17189:b1a756176ed2 user: Timo Sirainen date: Thu Apr 03 19:53:13 2014 +0300 description: Added istream-timeout, which triggers I/O event and fails with ETIMEDOUT after the timeout. diffstat: src/lib/Makefile.am | 2 + src/lib/istream-timeout.c | 123 ++++++++++++++++++++++++++++++++++++++++++++++ src/lib/istream-timeout.h | 9 +++ 3 files changed, 134 insertions(+), 0 deletions(-) diffs (159 lines): diff -r fd186eff7325 -r b1a756176ed2 src/lib/Makefile.am --- a/src/lib/Makefile.am Thu Apr 03 19:51:52 2014 +0300 +++ b/src/lib/Makefile.am Thu Apr 03 19:53:13 2014 +0300 @@ -70,6 +70,7 @@ istream-seekable.c \ istream-sized.c \ istream-tee.c \ + istream-timeout.c \ ioloop.c \ ioloop-iolist.c \ ioloop-notify-none.c \ @@ -197,6 +198,7 @@ istream-seekable.h \ istream-sized.h \ istream-tee.h \ + istream-timeout.h \ ioloop.h \ ioloop-iolist.h \ ioloop-private.h \ diff -r fd186eff7325 -r b1a756176ed2 src/lib/istream-timeout.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/istream-timeout.c Thu Apr 03 19:53:13 2014 +0300 @@ -0,0 +1,123 @@ +/* Copyright (c) 2014 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "time-util.h" +#include "istream-private.h" +#include "istream-timeout.h" + +struct timeout_istream { + struct istream_private istream; + + struct timeout *to; + struct timeval last_read_timestamp; + + unsigned int timeout_msecs; + bool update_timestamp; +}; + +static void i_stream_timeout_close(struct iostream_private *stream, + bool close_parent) +{ + struct timeout_istream *tstream = (struct timeout_istream *)stream; + + if (tstream->to != NULL) + timeout_remove(&tstream->to); + if (close_parent) + i_stream_close(tstream->istream.parent); +} + +static void i_stream_timeout_switch_ioloop(struct istream_private *stream) +{ + struct timeout_istream *tstream = (struct timeout_istream *)stream; + + if (tstream->to != NULL) + tstream->to = io_loop_move_timeout(&tstream->to); +} + +static void i_stream_timeout(struct timeout_istream *tstream) +{ + unsigned int msecs; + int diff; + + timeout_remove(&tstream->to); + + diff = timeval_diff_msecs(&ioloop_timeval, &tstream->last_read_timestamp); + if (diff < (int)tstream->timeout_msecs) { + /* we haven't reached the read timeout yet, update it */ + if (diff < 0) + diff = 0; + tstream->to = timeout_add(tstream->timeout_msecs - diff, + i_stream_timeout, tstream); + return; + } + + msecs = tstream->timeout_msecs % 1000; + io_stream_set_error(&tstream->istream.iostream, + "Read timeout in %u%s s after %"PRIuUOFF_T" bytes", + tstream->timeout_msecs/1000, + msecs == 0 ? "" : t_strdup_printf(".%u", msecs), + tstream->istream.istream.v_offset); + tstream->istream.istream.stream_errno = ETIMEDOUT; + + i_stream_set_input_pending(tstream->istream.parent, TRUE); +} + +static ssize_t +i_stream_timeout_read(struct istream_private *stream) +{ + struct timeout_istream *tstream = (struct timeout_istream *)stream; + ssize_t ret; + + i_stream_seek(stream->parent, stream->parent_start_offset + + stream->istream.v_offset); + + ret = i_stream_read_copy_from_parent(&stream->istream); + if (ret < 0) { + /* failed */ + } else if (tstream->to == NULL) { + /* first read. add the timeout here instead of in init + in case the stream is created long before it's actually + read from. */ + tstream->to = tstream->timeout_msecs == 0 ? NULL : + timeout_add(tstream->timeout_msecs, + i_stream_timeout, tstream); + tstream->update_timestamp = TRUE; + tstream->last_read_timestamp = ioloop_timeval; + } else if (ret > 0 && tstream->to != NULL) { + /* we read something, reset the timeout */ + timeout_reset(tstream->to); + /* make sure we get called again on the next ioloop run. + this updates the timeout to the timestamp where we actually + would have wanted to start waiting for more data (so if + there is long-running code outside the ioloop it's not + counted) */ + tstream->update_timestamp = TRUE; + tstream->last_read_timestamp = ioloop_timeval; + i_stream_set_input_pending(&stream->istream, TRUE); + } else if (tstream->update_timestamp) { + tstream->update_timestamp = FALSE; + tstream->last_read_timestamp = ioloop_timeval; + } + return ret; +} + +struct istream * +i_stream_create_timeout(struct istream *input, unsigned int timeout_msecs) +{ + struct timeout_istream *tstream; + + tstream = i_new(struct timeout_istream, 1); + tstream->timeout_msecs = timeout_msecs; + tstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + tstream->istream.stream_size_passthrough = TRUE; + + tstream->istream.read = i_stream_timeout_read; + tstream->istream.switch_ioloop = i_stream_timeout_switch_ioloop; + tstream->istream.iostream.close = i_stream_timeout_close; + + tstream->istream.istream.blocking = input->blocking; + tstream->istream.istream.seekable = input->seekable; + return i_stream_create(&tstream->istream, input, + i_stream_get_fd(input)); +} diff -r fd186eff7325 -r b1a756176ed2 src/lib/istream-timeout.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/istream-timeout.h Thu Apr 03 19:53:13 2014 +0300 @@ -0,0 +1,9 @@ +#ifndef ISTREAM_TIMEOUT_H +#define ISTREAM_TIMEOUT_H + +/* Return ETIMEDOUT error if read() doesn't return anything for timeout_msecs. + If timeout_msecs=0, there is no timeout. */ +struct istream * +i_stream_create_timeout(struct istream *input, unsigned int timeout_msecs); + +#endif From dovecot at dovecot.org Thu Apr 3 16:55:06 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 16:55:06 +0000 Subject: dovecot-2.2: lib-ssl-iostream: Make sure I/O input event is trig... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e0a3b902cd3f changeset: 17190:e0a3b902cd3f user: Timo Sirainen date: Thu Apr 03 19:54:27 2014 +0300 description: lib-ssl-iostream: Make sure I/O input event is triggered after ostream-ssl has read some data to buffer. diffstat: src/lib-ssl-iostream/iostream-openssl.c | 2 ++ src/lib-ssl-iostream/iostream-openssl.h | 1 + src/lib-ssl-iostream/istream-openssl.c | 1 + 3 files changed, 4 insertions(+), 0 deletions(-) diffs (41 lines): diff -r b1a756176ed2 -r e0a3b902cd3f src/lib-ssl-iostream/iostream-openssl.c --- a/src/lib-ssl-iostream/iostream-openssl.c Thu Apr 03 19:53:13 2014 +0300 +++ b/src/lib-ssl-iostream/iostream-openssl.c Thu Apr 03 19:54:27 2014 +0300 @@ -261,6 +261,7 @@ if (ssl_io->plain_output->real_stream->error_handling_disabled) o_stream_set_no_error_handling(*output, TRUE); + ssl_io->ssl_input = *input; ssl_io->ssl_output = *output; *iostream_r = ssl_io; return 0; @@ -427,6 +428,7 @@ ssl_io->ostream_flush_waiting_input = FALSE; o_stream_set_flush_pending(ssl_io->plain_output, TRUE); } + i_stream_set_input_pending(ssl_io->ssl_input, TRUE); ssl_io->want_read = FALSE; } return bytes_read; diff -r b1a756176ed2 -r e0a3b902cd3f src/lib-ssl-iostream/iostream-openssl.h --- a/src/lib-ssl-iostream/iostream-openssl.h Thu Apr 03 19:53:13 2014 +0300 +++ b/src/lib-ssl-iostream/iostream-openssl.h Thu Apr 03 19:54:27 2014 +0300 @@ -26,6 +26,7 @@ struct istream *plain_input; struct ostream *plain_output; + struct istream *ssl_input; struct ostream *ssl_output; char *host; diff -r b1a756176ed2 -r e0a3b902cd3f src/lib-ssl-iostream/istream-openssl.c --- a/src/lib-ssl-iostream/istream-openssl.c Thu Apr 03 19:53:13 2014 +0300 +++ b/src/lib-ssl-iostream/istream-openssl.c Thu Apr 03 19:54:27 2014 +0300 @@ -24,6 +24,7 @@ struct ssl_istream *sstream = (struct ssl_istream *)stream; i_free(sstream->istream.w_buffer); + sstream->ssl_io->ssl_input = NULL; ssl_iostream_unref(&sstream->ssl_io); } From dovecot at dovecot.org Thu Apr 3 16:56:56 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 16:56:56 +0000 Subject: dovecot-2.2: lib-http: Wrap response payload istream into a istr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/790afcf05f0d changeset: 17191:790afcf05f0d user: Timo Sirainen date: Thu Apr 03 19:56:47 2014 +0300 description: lib-http: Wrap response payload istream into a istream-timeout. diffstat: src/lib-http/http-client-connection.c | 6 +++++- src/plugins/fts-solr/solr-connection.c | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diffs (43 lines): diff -r e0a3b902cd3f -r 790afcf05f0d src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Thu Apr 03 19:54:27 2014 +0300 +++ b/src/lib-http/http-client-connection.c Thu Apr 03 19:56:47 2014 +0300 @@ -8,6 +8,7 @@ #include "array.h" #include "ioloop.h" #include "istream.h" +#include "istream-timeout.h" #include "ostream.h" #include "time-util.h" #include "iostream-rawlog.h" @@ -433,7 +434,8 @@ /* wrap the stream to capture the destroy event without destroying the actual payload stream. */ conn->incoming_payload = response->payload = - i_stream_create_limit(response->payload, (uoff_t)-1); + i_stream_create_timeout(response->payload, + conn->client->set.request_timeout_msecs); i_stream_add_destroy_callback(response->payload, http_client_payload_destroyed, req); @@ -1195,5 +1197,7 @@ conn->to_idle = io_loop_move_timeout(&conn->to_idle); if (conn->to_response != NULL) conn->to_response = io_loop_move_timeout(&conn->to_response); + if (conn->incoming_payload != NULL) + i_stream_switch_ioloop(conn->incoming_payload); connection_switch_ioloop(&conn->conn); } diff -r e0a3b902cd3f -r 790afcf05f0d src/plugins/fts-solr/solr-connection.c --- a/src/plugins/fts-solr/solr-connection.c Thu Apr 03 19:54:27 2014 +0300 +++ b/src/plugins/fts-solr/solr-connection.c Thu Apr 03 19:56:47 2014 +0300 @@ -391,8 +391,8 @@ i_stream_ref(response->payload); conn->payload = response->payload; - conn->io = io_add(i_stream_get_fd(response->payload), IO_READ, - solr_connection_payload_input, conn); + conn->io = io_add_istream(response->payload, + solr_connection_payload_input, conn); solr_connection_payload_input(conn); } From dovecot at dovecot.org Thu Apr 3 16:59:39 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 16:59:39 +0000 Subject: dovecot-2.2: Use io_add_istream() wherever possible. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a1c153c70bb9 changeset: 17192:a1c153c70bb9 user: Timo Sirainen date: Thu Apr 03 19:59:30 2014 +0300 description: Use io_add_istream() wherever possible. This shouldn't fix anything, but might make some functionality easier to implement in future. diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 4 +--- src/imap/cmd-append.c | 3 +-- src/imap/cmd-idle.c | 7 +++---- src/imap/imap-client.c | 8 +++----- src/lib-http/http-client-connection.c | 8 ++++---- src/lib-http/http-client-request.c | 7 ++----- src/lib-http/test-http-client.c | 3 +-- src/pop3/pop3-client.c | 6 +++--- 8 files changed, 18 insertions(+), 28 deletions(-) diffs (183 lines): diff -r 790afcf05f0d -r a1c153c70bb9 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Thu Apr 03 19:56:47 2014 +0300 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Thu Apr 03 19:59:30 2014 +0300 @@ -134,7 +134,6 @@ struct dsync_ibc ibc; char *name, *temp_path_prefix; - int fd_in, fd_out; struct istream *input; struct ostream *output; struct io *io; @@ -292,7 +291,7 @@ { unsigned int i; - ibc->io = io_add(ibc->fd_in, IO_READ, dsync_ibc_stream_input, ibc); + ibc->io = io_add_istream(ibc->input, dsync_ibc_stream_input, ibc); o_stream_set_no_error_handling(ibc->output, TRUE); o_stream_set_flush_callback(ibc->output, dsync_ibc_stream_output, ibc); ibc->to = timeout_add(DSYNC_IBC_STREAM_TIMEOUT_MSECS, @@ -1845,7 +1844,6 @@ ibc->ibc.v = dsync_ibc_stream_vfuncs; ibc->input = input; ibc->output = output; - ibc->fd_in = i_stream_get_fd(input); ibc->name = i_strdup(name); ibc->temp_path_prefix = i_strdup(temp_path_prefix); ibc->ret_pool = pool_alloconly_create("ibc stream data", 2048); diff -r 790afcf05f0d -r a1c153c70bb9 src/imap/cmd-append.c --- a/src/imap/cmd-append.c Thu Apr 03 19:56:47 2014 +0300 +++ b/src/imap/cmd-append.c Thu Apr 03 19:59:30 2014 +0300 @@ -914,8 +914,7 @@ } io_remove(&client->io); - client->io = io_add(i_stream_get_fd(client->input), IO_READ, - client_input_append, cmd); + client->io = io_add_istream(client->input, client_input_append, cmd); /* append is special because we're only waiting on client input, not client output, so disable the standard output handler until we're finished */ diff -r 790afcf05f0d -r a1c153c70bb9 src/imap/cmd-idle.c --- a/src/imap/cmd-idle.c Thu Apr 03 19:56:47 2014 +0300 +++ b/src/imap/cmd-idle.c Thu Apr 03 19:59:30 2014 +0300 @@ -284,8 +284,8 @@ } if (client->io == NULL) { /* input is pending */ - client->io = io_add(i_stream_get_fd(client->input), - IO_READ, idle_client_input, ctx); + client->io = io_add_istream(client->input, + idle_client_input, ctx); idle_client_input_more(ctx); } return FALSE; @@ -306,8 +306,7 @@ client_send_line(client, "+ idling"); io_remove(&client->io); - client->io = io_add(i_stream_get_fd(client->input), - IO_READ, idle_client_input, ctx); + client->io = io_add_istream(client->input, idle_client_input, ctx); cmd->func = cmd_idle_continue; cmd->context = ctx; diff -r 790afcf05f0d -r a1c153c70bb9 src/imap/imap-client.c --- a/src/imap/imap-client.c Thu Apr 03 19:56:47 2014 +0300 +++ b/src/imap/imap-client.c Thu Apr 03 19:59:30 2014 +0300 @@ -89,7 +89,7 @@ o_stream_set_flush_callback(client->output, client_output, client); p_array_init(&client->module_contexts, client->pool, 5); - client->io = io_add(fd_in, IO_READ, client_input, client); + client->io = io_add_istream(client->input, client_input, client); client->last_input = ioloop_time; client->to_idle = timeout_add(CLIENT_IDLE_TIMEOUT_MSECS, client_idle_timeout, client); @@ -617,10 +617,8 @@ static void client_add_missing_io(struct client *client) { - if (client->io == NULL && !client->disconnected) { - client->io = io_add(client->fd_in, - IO_READ, client_input, client); - } + if (client->io == NULL && !client->disconnected) + client->io = io_add_istream(client->input, client_input, client); } void client_command_free(struct client_command_context **_cmd) diff -r 790afcf05f0d -r a1c153c70bb9 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Thu Apr 03 19:56:47 2014 +0300 +++ b/src/lib-http/http-client-connection.c Thu Apr 03 19:59:30 2014 +0300 @@ -375,8 +375,8 @@ static void http_client_payload_finished(struct http_client_connection *conn) { timeout_remove(&conn->to_input); - conn->conn.io = io_add(conn->conn.fd_in, IO_READ, - http_client_connection_input, &conn->conn); + conn->conn.io = io_add_istream(conn->conn.input, + http_client_connection_input, &conn->conn); } static void @@ -464,7 +464,7 @@ i_stream_remove_destroy_callback(conn->incoming_payload, http_client_payload_destroyed); i_stream_unref(&conn->incoming_payload); - conn->conn.io = io_add(conn->conn.fd_in, IO_READ, + conn->conn.io = io_add_istream(conn->conn.input, http_client_connection_input, &conn->conn); } @@ -997,7 +997,7 @@ o_stream_set_no_error_handling(conn->output, TRUE); o_stream_set_name(conn->output, conn->name); - conn->io = io_add(conn->fd_in, IO_READ, *list->v.input, conn); + conn->io = io_add_istream(conn->input, *list->v.input, conn); DLLIST_PREPEND(&list->connections, conn); list->connections_count++; diff -r 790afcf05f0d -r a1c153c70bb9 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Thu Apr 03 19:56:47 2014 +0300 +++ b/src/lib-http/http-client-request.c Thu Apr 03 19:59:30 2014 +0300 @@ -548,7 +548,6 @@ struct http_client_connection *conn = req->conn; struct ostream *output = req->payload_output; off_t ret; - int fd; i_assert(req->payload_input != NULL); i_assert(req->payload_output != NULL); @@ -608,11 +607,9 @@ http_client_request_debug(req, "Partially sent payload"); } else { /* input is blocking */ - fd = i_stream_get_fd(req->payload_input); conn->output_locked = TRUE; - i_assert(fd >= 0); - conn->io_req_payload = io_add - (fd, IO_READ, http_client_request_payload_input, req); + conn->io_req_payload = io_add_istream(req->payload_input, + http_client_request_payload_input, req); } return ret < 0 ? -1 : 0; } diff -r 790afcf05f0d -r a1c153c70bb9 src/lib-http/test-http-client.c --- a/src/lib-http/test-http-client.c Thu Apr 03 19:56:47 2014 +0300 +++ b/src/lib-http/test-http-client.c Thu Apr 03 19:59:30 2014 +0300 @@ -63,8 +63,7 @@ i_info("DEBUG: REQUEST: Got payload"); i_stream_ref(response->payload); req->payload = response->payload; - req->io = io_add(i_stream_get_fd(response->payload), IO_READ, - payload_input, req); + req->io = io_add_istream(response->payload, payload_input, req); payload_input(req); } diff -r 790afcf05f0d -r a1c153c70bb9 src/pop3/pop3-client.c --- a/src/pop3/pop3-client.c Thu Apr 03 19:56:47 2014 +0300 +++ b/src/pop3/pop3-client.c Thu Apr 03 19:59:30 2014 +0300 @@ -394,7 +394,7 @@ o_stream_set_flush_callback(client->output, client_output, client); p_array_init(&client->module_contexts, client->pool, 5); - client->io = io_add(fd_in, IO_READ, client_input, client); + client->io = io_add_istream(client->input, client_input, client); client->last_input = ioloop_time; client->to_idle = timeout_add(CLIENT_IDLE_TIMEOUT_MSECS, client_idle_timeout, client); @@ -812,8 +812,8 @@ if (o_stream_get_buffer_used_size(client->output) < POP3_OUTBUF_THROTTLE_SIZE/2 && client->io == NULL) { /* enable input again */ - client->io = io_add(i_stream_get_fd(client->input), - IO_READ, client_input, client); + client->io = io_add_istream(client->input, client_input, + client); } if (client->io != NULL && client->waiting_input) { if (!client_handle_input(client)) { From dovecot at dovecot.org Thu Apr 3 20:38:30 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 20:38:30 +0000 Subject: dovecot-2.2: maildir: Fixed auto-creating INBOX that was missing... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0ca197d2da42 changeset: 17193:0ca197d2da42 user: Timo Sirainen date: Thu Apr 03 23:38:17 2014 +0300 description: maildir: Fixed auto-creating INBOX that was missing tmp/ directory (but not cur/) diffstat: src/lib-storage/index/maildir/maildir-storage.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r a1c153c70bb9 -r 0ca197d2da42 src/lib-storage/index/maildir/maildir-storage.c --- a/src/lib-storage/index/maildir/maildir-storage.c Thu Apr 03 19:59:30 2014 +0300 +++ b/src/lib-storage/index/maildir/maildir-storage.c Thu Apr 03 23:38:17 2014 +0300 @@ -361,8 +361,8 @@ } root_dir = mailbox_list_get_root_forced(box->list, MAILBOX_LIST_PATH_TYPE_MAILBOX); - if (strcmp(box_path, root_dir) == 0) { - /* root directory. either INBOX or some other namespace root */ + if (strcmp(box_path, root_dir) == 0 && !box->inbox_any) { + /* root directory for some namespace. */ errno = ENOENT; } else if (stat(box_path, &st) == 0) { /* yes, we'll need to create the missing dirs */ From dovecot at dovecot.org Thu Apr 3 21:14:56 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 21:14:56 +0000 Subject: dovecot-2.2: lib-storage: When trying to get body size quickly, ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2338a2c96a22 changeset: 17194:2338a2c96a22 user: Timo Sirainen date: Fri Apr 04 00:13:35 2014 +0300 description: lib-storage: When trying to get body size quickly, make sure we don't end up reading the mail body. diffstat: src/lib-storage/index/index-mail.c | 15 +++++++++++---- 1 files changed, 11 insertions(+), 4 deletions(-) diffs (25 lines): diff -r 0ca197d2da42 -r 2338a2c96a22 src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Thu Apr 03 23:38:17 2014 +0300 +++ b/src/lib-storage/index/index-mail.c Fri Apr 04 00:13:35 2014 +0300 @@ -435,10 +435,17 @@ return; if (!data->body_size_set) { - if (mail_get_physical_size(&mail->mail.mail, &tmp) < 0) - return; - /* we should have everything now. try again. */ - (void)index_mail_get_cached_virtual_size(mail, &tmp); + enum mail_lookup_abort old_abort = mail->mail.mail.lookup_abort; + + /* get the physical size, but not if it requires reading + through the whole message */ + if (mail->mail.mail.lookup_abort < MAIL_LOOKUP_ABORT_READ_MAIL) + mail->mail.mail.lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL; + if (mail_get_physical_size(&mail->mail.mail, &tmp) == 0) { + /* we should have everything now. try again. */ + (void)index_mail_get_cached_virtual_size(mail, &tmp); + } + mail->mail.mail.lookup_abort = old_abort; } } From dovecot at dovecot.org Thu Apr 3 21:14:57 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 21:14:57 +0000 Subject: dovecot-2.2: maildir: If istream_opened hook is set, don't retur... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1adbd576f320 changeset: 17195:1adbd576f320 user: Timo Sirainen date: Fri Apr 04 00:14:42 2014 +0300 description: maildir: If istream_opened hook is set, don't return stat()ed file size as physical size. That's pretty much always wrong. The hook is set with e.g. zlib and mail-filter and similar plugins that modify the mail stream. diffstat: src/lib-storage/index/maildir/maildir-mail.c | 11 ++++++++++- 1 files changed, 10 insertions(+), 1 deletions(-) diffs (28 lines): diff -r 2338a2c96a22 -r 1adbd576f320 src/lib-storage/index/maildir/maildir-mail.c --- a/src/lib-storage/index/maildir/maildir-mail.c Fri Apr 04 00:13:35 2014 +0300 +++ b/src/lib-storage/index/maildir/maildir-mail.c Fri Apr 04 00:14:42 2014 +0300 @@ -421,6 +421,8 @@ struct maildir_mailbox *mbox = (struct maildir_mailbox *)_mail->box; struct index_mail_data *data = &mail->data; struct stat st; + struct message_size hdr_size, body_size; + struct istream *input; const char *path; int ret; @@ -448,7 +450,14 @@ return 0; } - if (!_mail->saving) { + if (mail->mail.v.istream_opened != NULL) { + /* we can't use stat(), because this may be a mail that some + plugin has changed (e.g. zlib). need to do it the slow + way. */ + if (mail_get_stream(_mail, &hdr_size, &body_size, &input) < 0) + return -1; + st.st_size = hdr_size.physical_size + body_size.physical_size; + } else if (!_mail->saving) { ret = maildir_file_do(mbox, _mail->uid, do_stat, &st); if (ret <= 0) { if (ret == 0) From dovecot at dovecot.org Thu Apr 3 21:30:20 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 21:30:20 +0000 Subject: dovecot-2.2: lib-imap-storage: Minor code cleanup Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1b71a0128d2d changeset: 17196:1b71a0128d2d user: Timo Sirainen date: Fri Apr 04 00:30:11 2014 +0300 description: lib-imap-storage: Minor code cleanup diffstat: src/lib-imap-storage/imap-msgpart.c | 21 +++++++++++---------- 1 files changed, 11 insertions(+), 10 deletions(-) diffs (58 lines): diff -r 1adbd576f320 -r 1b71a0128d2d src/lib-imap-storage/imap-msgpart.c --- a/src/lib-imap-storage/imap-msgpart.c Fri Apr 04 00:14:42 2014 +0300 +++ b/src/lib-imap-storage/imap-msgpart.c Fri Apr 04 00:30:11 2014 +0300 @@ -371,6 +371,7 @@ i_stream_seek(input, 0); result_r->input = input; result_r->size = hdr_size_r->virtual_size; + result_r->size_field = 0; return 0; } @@ -558,14 +559,13 @@ case FETCH_MIME_BODY: i_unreached(); case FETCH_HEADER: - case FETCH_HEADER_FIELDS_NOT: /* fetch the message's header */ if (mail_get_hdr_stream(mail, &hdr_size, &input) < 0) return -1; result_r->size_field = MAIL_FETCH_MESSAGE_PARTS; break; case FETCH_HEADER_FIELDS: - /* try to lookup the headers from cache */ + /* return specific headers */ if (msgpart->header_ctx == NULL) { msgpart->header_ctx = mailbox_header_lookup_init(mail->box, @@ -574,8 +574,14 @@ if (mail_get_header_stream(mail, msgpart->header_ctx, &input) < 0) return -1; - result_r->size_field = 0; - break; + return imap_msgpart_get_partial_header(mail, input, msgpart, + part_size_r, result_r); + case FETCH_HEADER_FIELDS_NOT: + /* return specific headers */ + if (mail_get_hdr_stream(mail, NULL, &input) < 0) + return -1; + return imap_msgpart_get_partial_header(mail, input, msgpart, + part_size_r, result_r); case FETCH_BODY: /* fetch the message's body */ if (mail_get_stream(mail, &hdr_size, &body_size, &input) < 0) @@ -583,12 +589,7 @@ result_r->size_field = MAIL_FETCH_MESSAGE_PARTS; break; } - - if (msgpart->headers != NULL) { - /* return specific headers */ - return imap_msgpart_get_partial_header(mail, input, msgpart, - part_size_r, result_r); - } + i_assert(msgpart->headers == NULL); switch (msgpart->fetch_type) { case FETCH_FULL: From dovecot at dovecot.org Thu Apr 3 22:03:29 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 22:03:29 +0000 Subject: dovecot-2.2: lib-imap-storage: Reverted previous change. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cba3890dafdb changeset: 17197:cba3890dafdb user: Timo Sirainen date: Fri Apr 04 00:53:34 2014 +0300 description: lib-imap-storage: Reverted previous change. It broke FETCH BODY [n.HEADER.FIELDS ..] diffstat: src/lib-imap-storage/imap-msgpart.c | 21 ++++++++++----------- 1 files changed, 10 insertions(+), 11 deletions(-) diffs (58 lines): diff -r 1b71a0128d2d -r cba3890dafdb src/lib-imap-storage/imap-msgpart.c --- a/src/lib-imap-storage/imap-msgpart.c Fri Apr 04 00:30:11 2014 +0300 +++ b/src/lib-imap-storage/imap-msgpart.c Fri Apr 04 00:53:34 2014 +0300 @@ -371,7 +371,6 @@ i_stream_seek(input, 0); result_r->input = input; result_r->size = hdr_size_r->virtual_size; - result_r->size_field = 0; return 0; } @@ -559,13 +558,14 @@ case FETCH_MIME_BODY: i_unreached(); case FETCH_HEADER: + case FETCH_HEADER_FIELDS_NOT: /* fetch the message's header */ if (mail_get_hdr_stream(mail, &hdr_size, &input) < 0) return -1; result_r->size_field = MAIL_FETCH_MESSAGE_PARTS; break; case FETCH_HEADER_FIELDS: - /* return specific headers */ + /* try to lookup the headers from cache */ if (msgpart->header_ctx == NULL) { msgpart->header_ctx = mailbox_header_lookup_init(mail->box, @@ -574,14 +574,8 @@ if (mail_get_header_stream(mail, msgpart->header_ctx, &input) < 0) return -1; - return imap_msgpart_get_partial_header(mail, input, msgpart, - part_size_r, result_r); - case FETCH_HEADER_FIELDS_NOT: - /* return specific headers */ - if (mail_get_hdr_stream(mail, NULL, &input) < 0) - return -1; - return imap_msgpart_get_partial_header(mail, input, msgpart, - part_size_r, result_r); + result_r->size_field = 0; + break; case FETCH_BODY: /* fetch the message's body */ if (mail_get_stream(mail, &hdr_size, &body_size, &input) < 0) @@ -589,7 +583,12 @@ result_r->size_field = MAIL_FETCH_MESSAGE_PARTS; break; } - i_assert(msgpart->headers == NULL); + + if (msgpart->headers != NULL) { + /* return specific headers */ + return imap_msgpart_get_partial_header(mail, input, msgpart, + part_size_r, result_r); + } switch (msgpart->fetch_type) { case FETCH_FULL: From dovecot at dovecot.org Thu Apr 3 22:03:30 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 22:03:30 +0000 Subject: dovecot-2.2: lib-imap-storage: Don't waste effort getting mail's... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/95ae49692ccb changeset: 17198:95ae49692ccb user: Timo Sirainen date: Fri Apr 04 01:03:17 2014 +0300 description: lib-imap-storage: Don't waste effort getting mail's physical size if it can't be done quickly. diffstat: src/lib-imap-storage/imap-msgpart.c | 71 +++++++++++++++++++++--------------- 1 files changed, 42 insertions(+), 29 deletions(-) diffs (199 lines): diff -r cba3890dafdb -r 95ae49692ccb src/lib-imap-storage/imap-msgpart.c --- a/src/lib-imap-storage/imap-msgpart.c Fri Apr 04 00:53:34 2014 +0300 +++ b/src/lib-imap-storage/imap-msgpart.c Fri Apr 04 01:03:17 2014 +0300 @@ -336,11 +336,12 @@ static int imap_msgpart_get_partial_header(struct mail *mail, struct istream *mail_input, const struct imap_msgpart *msgpart, - struct message_size *hdr_size_r, + uoff_t *virtual_size_r, bool *have_crlfs_r, struct imap_msgpart_open_result *result_r) { const char *const *hdr_fields = msgpart->headers; unsigned int hdr_count = str_array_length(hdr_fields); + struct message_size hdr_size; struct istream *input; bool has_nuls; @@ -361,7 +362,7 @@ (void *)NULL); } - if (message_get_header_size(input, hdr_size_r, &has_nuls) < 0) { + if (message_get_header_size(input, &hdr_size, &has_nuls) < 0) { errno = input->stream_errno; mail_storage_set_critical(mail->box->storage, "read(%s) failed: %m", i_stream_get_name(mail_input)); @@ -370,7 +371,10 @@ } i_stream_seek(input, 0); result_r->input = input; - result_r->size = hdr_size_r->virtual_size; + result_r->size = hdr_size.virtual_size; + result_r->size_field = 0; + *virtual_size_r = hdr_size.virtual_size; + *have_crlfs_r = hdr_size.physical_size == hdr_size.virtual_size; return 0; } @@ -426,7 +430,7 @@ static void imap_msgpart_get_partial(struct mail *mail, const struct imap_msgpart *msgpart, bool convert_nuls, bool use_partial_cache, - const struct message_size *part_size, + uoff_t virtual_size, bool have_crlfs, struct imap_msgpart_open_result *result) { struct istream *input2; @@ -434,7 +438,7 @@ /* input is already seeked to the beginning of the wanted data */ - if (msgpart->partial_offset >= part_size->virtual_size) { + if (msgpart->partial_offset >= virtual_size) { /* can't seek past the MIME part */ i_stream_unref(&result->input); result->input = i_stream_create_from_data("", 0); @@ -442,7 +446,7 @@ return; } - if (part_size->virtual_size == part_size->physical_size) { + if (have_crlfs) { /* input has CRLF linefeeds, we can quickly seek to wanted position */ i_stream_skip(result->input, msgpart->partial_offset); @@ -454,7 +458,7 @@ msgpart); } - bytes_left = part_size->virtual_size - msgpart->partial_offset; + bytes_left = virtual_size - msgpart->partial_offset; if (msgpart->partial_size <= bytes_left) { /* limit output to specified number of bytes */ result->size = msgpart->partial_size; @@ -527,15 +531,16 @@ static int imap_msgpart_open_normal(struct mail *mail, struct imap_msgpart *msgpart, const struct message_part *part, - struct message_size *part_size_r, + uoff_t *virtual_size_r, bool *have_crlfs_r, struct imap_msgpart_open_result *result_r) { - struct message_size hdr_size, body_size; + struct message_size hdr_size, body_size, part_size; struct istream *input = NULL; + bool unknown_crlfs = FALSE; memset(&hdr_size, 0, sizeof(hdr_size)); memset(&body_size, 0, sizeof(body_size)); - memset(part_size_r, 0, sizeof(*part_size_r)); + memset(&part_size, 0, sizeof(part_size)); if (*msgpart->section_number != '\0') { /* find the MIME part */ @@ -549,10 +554,15 @@ case FETCH_FULL: /* fetch the whole message */ if (mail_get_stream(mail, NULL, NULL, &input) < 0 || - mail_get_virtual_size(mail, &body_size.virtual_size) < 0 || - mail_get_physical_size(mail, &body_size.physical_size) < 0) + mail_get_virtual_size(mail, &body_size.virtual_size) < 0) return -1; result_r->size_field = MAIL_FETCH_VIRTUAL_SIZE; + + i_assert(mail->lookup_abort == MAIL_LOOKUP_ABORT_NEVER); + mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL; + if (mail_get_physical_size(mail, &body_size.physical_size) < 0) + unknown_crlfs = TRUE; + mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER; break; case FETCH_MIME: case FETCH_MIME_BODY: @@ -587,18 +597,19 @@ if (msgpart->headers != NULL) { /* return specific headers */ return imap_msgpart_get_partial_header(mail, input, msgpart, - part_size_r, result_r); + virtual_size_r, + have_crlfs_r, result_r); } switch (msgpart->fetch_type) { case FETCH_FULL: - part_size_r->physical_size += body_size.physical_size; - part_size_r->virtual_size += body_size.virtual_size; + part_size.physical_size += body_size.physical_size; + part_size.virtual_size += body_size.virtual_size; /* fall through */ case FETCH_MIME: case FETCH_HEADER: - part_size_r->physical_size += hdr_size.physical_size; - part_size_r->virtual_size += hdr_size.virtual_size; + part_size.physical_size += hdr_size.physical_size; + part_size.virtual_size += hdr_size.virtual_size; break; case FETCH_HEADER_FIELDS: case FETCH_HEADER_FIELDS_NOT: @@ -606,13 +617,16 @@ case FETCH_BODY: case FETCH_MIME_BODY: i_stream_skip(input, hdr_size.physical_size); - part_size_r->physical_size += body_size.physical_size; - part_size_r->virtual_size += body_size.virtual_size; + part_size.physical_size += body_size.physical_size; + part_size.virtual_size += body_size.virtual_size; break; } result_r->input = input; i_stream_ref(input); + *virtual_size_r = part_size.virtual_size; + *have_crlfs_r = !unknown_crlfs && + part_size.virtual_size == part_size.physical_size; return 0; } @@ -620,9 +634,8 @@ struct imap_msgpart_open_result *result_r) { struct message_part *part; - struct message_size part_size; - uoff_t size; - bool include_hdr, binary, use_partial_cache; + uoff_t virtual_size; + bool include_hdr, binary, use_partial_cache, have_crlfs; int ret; memset(result_r, 0, sizeof(*result_r)); @@ -645,15 +658,15 @@ if (mail_get_parts(mail, &part) < 0) return -1; } - if (mail_get_binary_stream(mail, part, include_hdr, &size, - &binary, &result_r->input) < 0) + if (mail_get_binary_stream(mail, part, include_hdr, + &virtual_size, &binary, + &result_r->input) < 0) return -1; - part_size.virtual_size = size; - part_size.physical_size = size; + have_crlfs = TRUE; use_partial_cache = FALSE; } else { - if (imap_msgpart_open_normal(mail, msgpart, part, &part_size, - result_r) < 0) + if (imap_msgpart_open_normal(mail, msgpart, part, &virtual_size, + &have_crlfs, result_r) < 0) return -1; binary = FALSE; use_partial_cache = TRUE; @@ -663,7 +676,7 @@ result_r->binary_decoded_input_has_nuls = TRUE; imap_msgpart_get_partial(mail, msgpart, !binary, use_partial_cache, - &part_size, result_r); + virtual_size, have_crlfs, result_r); return 0; } From dovecot at dovecot.org Thu Apr 3 22:23:38 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Apr 2014 22:23:38 +0000 Subject: dovecot-2.2: maildir: If istream_opened is set by a plugin, don'... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3b9935fe9cb7 changeset: 17199:3b9935fe9cb7 user: Timo Sirainen date: Fri Apr 04 01:23:29 2014 +0300 description: maildir: If istream_opened is set by a plugin, don't try to fix S=size by renaming. Although this could be fixed in a nicer way by actually calculating the correct S=size and using that for renaming. diffstat: src/lib-storage/index/maildir/maildir-mail.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (19 lines): diff -r 95ae49692ccb -r 3b9935fe9cb7 src/lib-storage/index/maildir/maildir-mail.c --- a/src/lib-storage/index/maildir/maildir-mail.c Fri Apr 04 01:03:17 2014 +0300 +++ b/src/lib-storage/index/maildir/maildir-mail.c Fri Apr 04 01:23:29 2014 +0300 @@ -695,12 +695,14 @@ enum mail_fetch_field field) { struct maildir_mailbox *mbox = (struct maildir_mailbox *)mail->box; + struct mail_private *pmail = (struct mail_private *)mail; enum maildir_uidlist_rec_flag flags; const char *fname; uoff_t size; char wrong_key; - if (mbox->storage->set->maildir_broken_filename_sizes) { + if (mbox->storage->set->maildir_broken_filename_sizes || + pmail->v.istream_opened != NULL) { /* never try to fix sizes in maildir filenames */ return; } From dovecot at dovecot.org Mon Apr 7 21:08:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Apr 2014 21:08:53 +0000 Subject: dovecot-2.2: dsync: Fixed using -n "" parameter Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3e1a69e0cda9 changeset: 17200:3e1a69e0cda9 user: Timo Sirainen date: Tue Apr 08 00:08:21 2014 +0300 description: dsync: Fixed using -n "" parameter diffstat: src/doveadm/dsync/dsync-brain.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (16 lines): diff -r 3b9935fe9cb7 -r 3e1a69e0cda9 src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Fri Apr 04 01:23:29 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain.c Tue Apr 08 00:08:21 2014 +0300 @@ -410,6 +410,12 @@ if (ibc_set->sync_ns_prefixes != NULL) { p_array_init(&brain->sync_namespaces, brain->pool, 4); prefixes = t_strsplit(ibc_set->sync_ns_prefixes, "\n"); + if (prefixes[0] == NULL) { + /* ugly workaround for strsplit API: there was one + prefix="" entry */ + static const char *empty_prefix[] = { "", NULL }; + prefixes = empty_prefix; + } for (; *prefixes != NULL; prefixes++) { ns = mail_namespace_find(brain->user->namespaces, *prefixes); From dovecot at dovecot.org Mon Apr 7 21:09:33 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Apr 2014 21:09:33 +0000 Subject: dovecot-2.2: liblib: Added unit test for t_strsplit(data="") Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3bff23b6c187 changeset: 17201:3bff23b6c187 user: Timo Sirainen date: Tue Apr 08 00:09:03 2014 +0300 description: liblib: Added unit test for t_strsplit(data="") diffstat: src/lib/strfuncs.h | 3 ++- src/lib/test-strfuncs.c | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletions(-) diffs (47 lines): diff -r 3e1a69e0cda9 -r 3bff23b6c187 src/lib/strfuncs.h --- a/src/lib/strfuncs.h Tue Apr 08 00:08:21 2014 +0300 +++ b/src/lib/strfuncs.h Tue Apr 08 00:09:03 2014 +0300 @@ -56,7 +56,8 @@ int i_strcmp_p(const char *const *p1, const char *const *p2) ATTR_PURE; int i_strcasecmp_p(const char *const *p1, const char *const *p2) ATTR_PURE; -/* separators is an array of separator characters, not a separator string. */ +/* separators is an array of separator characters, not a separator string. + an empty data string results in an array containing only NULL. */ char **p_strsplit(pool_t pool, const char *data, const char *separators) ATTR_MALLOC; const char **t_strsplit(const char *data, const char *separators) diff -r 3e1a69e0cda9 -r 3bff23b6c187 src/lib/test-strfuncs.c --- a/src/lib/test-strfuncs.c Tue Apr 08 00:08:21 2014 +0300 +++ b/src/lib/test-strfuncs.c Tue Apr 08 00:09:03 2014 +0300 @@ -29,6 +29,23 @@ test_end(); } +static void test_t_strsplit(void) +{ + const char *const *args; + + test_begin("t_strsplit"); + /* empty string -> empty array. was this perhaps a mistake for the + API to do this originally?.. can't really change now anyway. */ + args = t_strsplit("", "\n"); + test_assert(args[0] == NULL); + /* two empty strings */ + args = t_strsplit("\n", "\n"); + test_assert(args[0][0] == '\0'); + test_assert(args[1][0] == '\0'); + test_assert(args[2] == NULL); + test_end(); +} + static void strsplit_verify(const char *str) { T_BEGIN { @@ -82,5 +99,6 @@ void test_strfuncs(void) { test_p_strarray_dup(); + test_t_strsplit(); test_t_strsplit_tab(); } From dovecot at dovecot.org Mon Apr 7 21:21:11 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Apr 2014 21:21:11 +0000 Subject: dovecot-2.2: doveadm backup: Fix to handling mailbox deletions. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/57a490780b1e changeset: 17202:57a490780b1e user: Timo Sirainen date: Tue Apr 08 00:20:57 2014 +0300 description: doveadm backup: Fix to handling mailbox deletions. diffstat: src/doveadm/dsync/dsync-mailbox-tree-sync.c | 47 ++++++++++++++++++++++++---- 1 files changed, 39 insertions(+), 8 deletions(-) diffs (88 lines): diff -r 3bff23b6c187 -r 57a490780b1e src/doveadm/dsync/dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c Tue Apr 08 00:09:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Tue Apr 08 00:20:57 2014 +0300 @@ -199,10 +199,16 @@ sync_delete_mailbox_node(ctx, tree, node, reason); } +enum del_sync_type { + DEL_SYNC_TYPE_TWOWAY, + DEL_SYNC_TYPE_RESTORE, + DEL_SYNC_TYPE_IGNORE +}; + static void sync_tree_sort_and_delete_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, struct dsync_mailbox_tree *tree, - bool ignore_deletes) + enum del_sync_type del_sync_type) { struct dsync_mailbox_tree_bfs_iter *iter; struct dsync_mailbox_node *node, *parent = NULL; @@ -219,15 +225,20 @@ } if (node->existence == DSYNC_MAILBOX_NODE_DELETED && !dsync_mailbox_node_is_dir(node)) { - if (!ignore_deletes) { + switch (del_sync_type) { + case DEL_SYNC_TYPE_TWOWAY: /* this mailbox was deleted. delete it from the other side as well */ sync_delete_mailbox(ctx, tree, node, "Mailbox has been deleted"); - } else { + break; + case DEL_SYNC_TYPE_RESTORE: /* we want to restore the mailbox back. just treat it as if it didn't exist */ sync_set_node_deleted(tree, node); + break; + case DEL_SYNC_TYPE_IGNORE: + break; } } array_append(&siblings, &node, 1); @@ -1202,7 +1213,7 @@ { struct dsync_mailbox_tree_sync_ctx *ctx; pool_t pool; - bool ignore_deletes; + enum del_sync_type del_sync_type; i_assert(hash_table_is_created(local_tree->guid_hash)); i_assert(hash_table_is_created(remote_tree->guid_hash)); @@ -1217,10 +1228,30 @@ ctx->sync_flags = sync_flags; i_array_init(&ctx->changes, 128); - ignore_deletes = sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE; - sync_tree_sort_and_delete_mailboxes(ctx, remote_tree, ignore_deletes); - ignore_deletes = sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL; - sync_tree_sort_and_delete_mailboxes(ctx, local_tree, ignore_deletes); + switch (sync_type) { + case DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY: + del_sync_type = DEL_SYNC_TYPE_TWOWAY; + break; + case DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE: + del_sync_type = DEL_SYNC_TYPE_IGNORE; + break; + case DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL: + del_sync_type = DEL_SYNC_TYPE_RESTORE; + break; + } + sync_tree_sort_and_delete_mailboxes(ctx, remote_tree, del_sync_type); + switch (sync_type) { + case DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY: + del_sync_type = DEL_SYNC_TYPE_TWOWAY; + break; + case DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE: + del_sync_type = DEL_SYNC_TYPE_RESTORE; + break; + case DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL: + del_sync_type = DEL_SYNC_TYPE_IGNORE; + break; + } + sync_tree_sort_and_delete_mailboxes(ctx, local_tree, del_sync_type); dsync_mailbox_tree_update_child_timestamps(&local_tree->root, 0); dsync_mailbox_tree_update_child_timestamps(&remote_tree->root, 0); From dovecot at dovecot.org Tue Apr 8 06:50:05 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Apr 2014 06:50:05 +0000 Subject: dovecot-2.2: Compiler warning fix Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0cafeddf3bf0 changeset: 17203:0cafeddf3bf0 user: Timo Sirainen date: Tue Apr 08 08:49:53 2014 +0200 description: Compiler warning fix diffstat: src/doveadm/dsync/dsync-mailbox-tree-sync.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (21 lines): diff -r 57a490780b1e -r 0cafeddf3bf0 src/doveadm/dsync/dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c Tue Apr 08 00:20:57 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Tue Apr 08 08:49:53 2014 +0200 @@ -1238,6 +1238,8 @@ case DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL: del_sync_type = DEL_SYNC_TYPE_RESTORE; break; + default: + i_unreached(); } sync_tree_sort_and_delete_mailboxes(ctx, remote_tree, del_sync_type); switch (sync_type) { @@ -1250,6 +1252,8 @@ case DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL: del_sync_type = DEL_SYNC_TYPE_IGNORE; break; + default: + i_unreached(); } sync_tree_sort_and_delete_mailboxes(ctx, local_tree, del_sync_type); From dovecot at dovecot.org Tue Apr 8 15:07:45 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Apr 2014 15:07:45 +0000 Subject: dovecot-2.2: liblib: Fixed crash using str_c() in different stac... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/49c7cc58fb1b changeset: 17204:49c7cc58fb1b user: Timo Sirainen date: Tue Apr 08 17:07:31 2014 +0200 description: liblib: Fixed crash using str_c() in different stack frame with str_new(0) allocated string. diffstat: src/lib/str.c | 5 ++++- src/lib/test-str.c | 5 +++++ 2 files changed, 9 insertions(+), 1 deletions(-) diffs (30 lines): diff -r 0cafeddf3bf0 -r 49c7cc58fb1b src/lib/str.c --- a/src/lib/str.c Tue Apr 08 08:49:53 2014 +0200 +++ b/src/lib/str.c Tue Apr 08 17:07:31 2014 +0200 @@ -9,7 +9,10 @@ string_t *str_new(pool_t pool, size_t initial_size) { - return buffer_create_dynamic(pool, initial_size); + /* never allocate a 0 byte size buffer. this is especially important + when str_c() is called on an empty string from a different stack + frame (see the comment in buffer.c about this). */ + return buffer_create_dynamic(pool, I_MAX(initial_size, 1)); } string_t *str_new_const(pool_t pool, const char *str, size_t len) diff -r 0cafeddf3bf0 -r 49c7cc58fb1b src/lib/test-str.c --- a/src/lib/test-str.c Tue Apr 08 08:49:53 2014 +0200 +++ b/src/lib/test-str.c Tue Apr 08 17:07:31 2014 +0200 @@ -9,6 +9,11 @@ unsigned int i, j; test_begin("str_c()"); + str = t_str_new(0); + T_BEGIN { + (void)str_c(str); + } T_END; + for (i = 0; i < 32; i++) T_BEGIN { str = t_str_new(15); for (j = 0; j < i; j++) From dovecot at dovecot.org Wed Apr 9 14:05:31 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Apr 2014 14:05:31 +0000 Subject: dovecot-2.2: maildir: If GUID can't be preserved in filename due... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a7e3ea778ce1 changeset: 17205:a7e3ea778ce1 user: Timo Sirainen date: Wed Apr 09 17:05:12 2014 +0300 description: maildir: If GUID can't be preserved in filename due to size change, set it in dovecot-uidlist This fixes problems with dsync when zlib is used and filenames/GUIDs don't contain (correct) S=sizes. diffstat: src/lib-storage/index/maildir/maildir-save.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 49c7cc58fb1b -r a7e3ea778ce1 src/lib-storage/index/maildir/maildir-save.c --- a/src/lib-storage/index/maildir/maildir-save.c Tue Apr 08 17:07:31 2014 +0200 +++ b/src/lib-storage/index/maildir/maildir-save.c Wed Apr 09 17:05:12 2014 +0300 @@ -606,6 +606,9 @@ /* e.g. zlib plugin was used. the "physical size" must be in the maildir filename, since stat() will return wrong size */ ctx->file_last->preserve_filename = FALSE; + /* preserve the GUID if needed */ + if (ctx->file_last->guid == NULL) + ctx->file_last->guid = ctx->file_last->dest_basename; /* reset the base name as well, just in case there's a ,W=vsize */ ctx->file_last->dest_basename = ctx->file_last->tmp_name; From dovecot at dovecot.org Wed Apr 9 15:14:50 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Apr 2014 15:14:50 +0000 Subject: dovecot-2.2: doveadm backup: Another attempt at fixing assert-cr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/20462a116704 changeset: 17206:20462a116704 user: Timo Sirainen date: Wed Apr 09 18:14:33 2014 +0300 description: doveadm backup: Another attempt at fixing assert-crashes when deleting mailboxes. Only two-way syncs really care about the mailbox deletion logs which set existence=DELETED. So actually the log reading could be skipped entirely for doveadm backup to speed things up. In any case the code currently relies that after sync_tree_sort_and_delete_mailboxes() the tree no longer has nodes with existence=DELETED and mailbox_guid!=0, because that assert-crashes in sync_create_mailboxes(). The assert() could probably simply be changed into an if(), but might as well keep the code as it was originally intended and not have the sync/backup behave more differently than needed. diffstat: src/doveadm/dsync/dsync-mailbox-tree-sync.c | 58 ++++++---------------------- 1 files changed, 13 insertions(+), 45 deletions(-) diffs (97 lines): diff -r a7e3ea778ce1 -r 20462a116704 src/doveadm/dsync/dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c Wed Apr 09 17:05:12 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Wed Apr 09 18:14:33 2014 +0300 @@ -199,16 +199,10 @@ sync_delete_mailbox_node(ctx, tree, node, reason); } -enum del_sync_type { - DEL_SYNC_TYPE_TWOWAY, - DEL_SYNC_TYPE_RESTORE, - DEL_SYNC_TYPE_IGNORE -}; - static void sync_tree_sort_and_delete_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, struct dsync_mailbox_tree *tree, - enum del_sync_type del_sync_type) + bool twoway_sync) { struct dsync_mailbox_tree_bfs_iter *iter; struct dsync_mailbox_node *node, *parent = NULL; @@ -225,20 +219,19 @@ } if (node->existence == DSYNC_MAILBOX_NODE_DELETED && !dsync_mailbox_node_is_dir(node)) { - switch (del_sync_type) { - case DEL_SYNC_TYPE_TWOWAY: + if (twoway_sync) { /* this mailbox was deleted. delete it from the other side as well */ sync_delete_mailbox(ctx, tree, node, "Mailbox has been deleted"); - break; - case DEL_SYNC_TYPE_RESTORE: - /* we want to restore the mailbox back. - just treat it as if it didn't exist */ + } else { + /* treat the node as if it didn't exist. it'll + get either recreated or deleted later. in + any case this function must handle all + existence=DELETED mailbox nodes by changing + them into directories (setting GUID=0) or + we'll assert-crash later */ sync_set_node_deleted(tree, node); - break; - case DEL_SYNC_TYPE_IGNORE: - break; } } array_append(&siblings, &node, 1); @@ -1213,7 +1206,6 @@ { struct dsync_mailbox_tree_sync_ctx *ctx; pool_t pool; - enum del_sync_type del_sync_type; i_assert(hash_table_is_created(local_tree->guid_hash)); i_assert(hash_table_is_created(remote_tree->guid_hash)); @@ -1228,34 +1220,10 @@ ctx->sync_flags = sync_flags; i_array_init(&ctx->changes, 128); - switch (sync_type) { - case DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY: - del_sync_type = DEL_SYNC_TYPE_TWOWAY; - break; - case DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE: - del_sync_type = DEL_SYNC_TYPE_IGNORE; - break; - case DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL: - del_sync_type = DEL_SYNC_TYPE_RESTORE; - break; - default: - i_unreached(); - } - sync_tree_sort_and_delete_mailboxes(ctx, remote_tree, del_sync_type); - switch (sync_type) { - case DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY: - del_sync_type = DEL_SYNC_TYPE_TWOWAY; - break; - case DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE: - del_sync_type = DEL_SYNC_TYPE_RESTORE; - break; - case DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL: - del_sync_type = DEL_SYNC_TYPE_IGNORE; - break; - default: - i_unreached(); - } - sync_tree_sort_and_delete_mailboxes(ctx, local_tree, del_sync_type); + sync_tree_sort_and_delete_mailboxes(ctx, remote_tree, + sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY); + sync_tree_sort_and_delete_mailboxes(ctx, local_tree, + sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY); dsync_mailbox_tree_update_child_timestamps(&local_tree->root, 0); dsync_mailbox_tree_update_child_timestamps(&remote_tree->root, 0); From dovecot at dovecot.org Wed Apr 9 20:17:24 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Apr 2014 20:17:24 +0000 Subject: dovecot-2.2: lazy_expunge: If lazy-expunge mailbox open/create f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ce30e5ac42a9 changeset: 17207:ce30e5ac42a9 user: Timo Sirainen date: Wed Apr 09 23:17:09 2014 +0300 description: lazy_expunge: If lazy-expunge mailbox open/create fails, show the mailbox name in error message. diffstat: src/plugins/lazy-expunge/lazy-expunge-plugin.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (22 lines): diff -r 20462a116704 -r ce30e5ac42a9 src/plugins/lazy-expunge/lazy-expunge-plugin.c --- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c Wed Apr 09 18:14:33 2014 +0300 +++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c Wed Apr 09 23:17:09 2014 +0300 @@ -104,6 +104,8 @@ *error_r = mailbox_get_last_error(box, &error); if (error != MAIL_ERROR_NOTFOUND) { + *error_r = t_strdup_printf("Failed to open mailbox %s: %s", + name, *error_r); mailbox_free(&box); return NULL; } @@ -111,7 +113,8 @@ /* try creating and re-opening it. */ if (mailbox_create(box, NULL, FALSE) < 0 || mailbox_open(box) < 0) { - *error_r = mailbox_get_last_error(box, NULL); + *error_r = t_strdup_printf("Failed to create mailbox %s: %s", name, + mailbox_get_last_error(box, NULL)); mailbox_free(&box); return NULL; } From dovecot at dovecot.org Wed Apr 9 20:19:44 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Apr 2014 20:19:44 +0000 Subject: dovecot-2.2: lazy_expunge: If setting is empty, disable the plugin. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/978871ca81e7 changeset: 17208:978871ca81e7 user: Timo Sirainen date: Wed Apr 09 23:19:29 2014 +0300 description: lazy_expunge: If setting is empty, disable the plugin. Instead of assuming that we want to use prefix="" as the lazy_expunge namespace, which is highly unlikely anyone will ever want to do. diffstat: src/plugins/lazy-expunge/lazy-expunge-plugin.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r ce30e5ac42a9 -r 978871ca81e7 src/plugins/lazy-expunge/lazy-expunge-plugin.c --- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c Wed Apr 09 23:17:09 2014 +0300 +++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c Wed Apr 09 23:19:29 2014 +0300 @@ -442,7 +442,7 @@ const char *env; env = mail_user_plugin_getenv(user, "lazy_expunge"); - if (env != NULL) { + if (env != NULL && env[0] != '\0') { luser = p_new(user->pool, struct lazy_expunge_mail_user, 1); luser->module_ctx.super = *v; user->vlast = &luser->module_ctx.super; From dovecot at dovecot.org Tue Apr 15 13:32:27 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 13:32:27 +0000 Subject: dovecot-2.2: lib-storage: Avoid rebuilding the whole mailbox lis... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6fd12fbe6848 changeset: 17209:6fd12fbe6848 user: Timo Sirainen date: Tue Apr 15 15:13:28 2014 +0200 description: lib-storage: Avoid rebuilding the whole mailbox list index when one mailbox changes. diffstat: src/lib-storage/list/mailbox-list-index-status.c | 21 +++++++++++++++------ src/lib-storage/list/mailbox-list-index.h | 1 + 2 files changed, 16 insertions(+), 6 deletions(-) diffs (65 lines): diff -r 978871ca81e7 -r 6fd12fbe6848 src/lib-storage/list/mailbox-list-index-status.c --- a/src/lib-storage/list/mailbox-list-index-status.c Wed Apr 09 23:19:29 2014 +0300 +++ b/src/lib-storage/list/mailbox-list-index-status.c Tue Apr 15 15:13:28 2014 +0200 @@ -53,7 +53,10 @@ if (ret != 0) { /* error / mailbox has changed. we'll need to sync it. */ - mailbox_list_index_refresh_later(box->list); + if (ret < 0) + mailbox_list_index_refresh_later(box->list); + else + ilist->index_last_check_changed = TRUE; mail_index_view_close(&view); return ret < 0 ? -1 : 0; } @@ -366,10 +369,14 @@ list_view = mail_index_view_open(ilist->index); if (!index_list_update_fill_changes(box, list_view, &changes)) ret = -1; - else if (!index_list_has_changed(box, list_view, &changes)) - ret = 0; - else + else if (index_list_has_changed(box, list_view, &changes)) ret = 1; + else { + /* if backend state changed on the last check, update it here + now. we probably don't need to bother checking again if the + state had changed? */ + ret = ilist->index_last_check_changed; + } mail_index_view_close(&list_view); if (ret <= 0) { if (ret < 0) @@ -394,9 +401,10 @@ if (!index_list_update_fill_changes(box, list_view, &changes)) mailbox_list_index_refresh_later(box->list); - else if (index_list_has_changed(box, list_view, &changes)) { + else { ilist->updating_status = TRUE; - index_list_update(box, list_view, list_trans, &changes); + if (index_list_has_changed(box, list_view, &changes)) + index_list_update(box, list_view, list_trans, &changes); if (box->v.list_index_update_sync != NULL) { box->v.list_index_update_sync(box, list_trans, changes.seq); @@ -408,6 +416,7 @@ mailbox_set_index_error(box); return -1; } + ilist->index_last_check_changed = FALSE; return 0; } diff -r 978871ca81e7 -r 6fd12fbe6848 src/lib-storage/list/mailbox-list-index.h --- a/src/lib-storage/list/mailbox-list-index.h Wed Apr 09 23:19:29 2014 +0300 +++ b/src/lib-storage/list/mailbox-list-index.h Tue Apr 15 15:13:28 2014 +0200 @@ -107,6 +107,7 @@ unsigned int syncing:1; unsigned int updating_status:1; unsigned int has_backing_store:1; + unsigned int index_last_check_changed:1; }; struct mailbox_list_index_iterate_context { From dovecot at dovecot.org Tue Apr 15 13:32:32 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 13:32:32 +0000 Subject: dovecot-2.2: lib-storage: Don't open mailbox on mailbox_get_stat... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3a6b64bff7ea changeset: 17210:3a6b64bff7ea user: Timo Sirainen date: Tue Apr 15 15:21:37 2014 +0200 description: lib-storage: Don't open mailbox on mailbox_get_status() if we have no status items left to get. diffstat: src/lib-storage/index/index-status.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (16 lines): diff -r 6fd12fbe6848 -r 3a6b64bff7ea src/lib-storage/index/index-status.c --- a/src/lib-storage/index/index-status.c Tue Apr 15 15:13:28 2014 +0200 +++ b/src/lib-storage/index/index-status.c Tue Apr 15 15:21:37 2014 +0200 @@ -33,6 +33,12 @@ enum mailbox_status_items items, struct mailbox_status *status_r) { + if (items == 0) { + /* caller could have wanted only e.g. mailbox_status.have_* + flags */ + return 0; + } + if (!box->opened) { if (mailbox_open(box) < 0) return -1; From dovecot at dovecot.org Tue Apr 15 15:20:49 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 15:20:49 +0000 Subject: dovecot-2.2: Removed unused code. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ac26bb157938 changeset: 17211:ac26bb157938 user: Timo Sirainen date: Tue Apr 15 16:37:26 2014 +0200 description: Removed unused code. diffstat: src/lib/primes.c | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diffs (12 lines): diff -r 3a6b64bff7ea -r ac26bb157938 src/lib/primes.c --- a/src/lib/primes.c Tue Apr 15 15:21:37 2014 +0200 +++ b/src/lib/primes.c Tue Apr 15 16:37:26 2014 +0200 @@ -36,8 +36,6 @@ 4294967291U /* previous from 2^32 */ }; -static const unsigned int primes_count = N_ELEMENTS(primes); - unsigned int primes_closest(unsigned int num) { unsigned int i; From dovecot at dovecot.org Tue Apr 15 15:20:50 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 15:20:50 +0000 Subject: dovecot-2.2: lib-storage: Replaced mail_get_real_mail() with mai... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e47b3e215dec changeset: 17212:e47b3e215dec user: Timo Sirainen date: Tue Apr 15 17:06:57 2014 +0200 description: lib-storage: Replaced mail_get_real_mail() with mail_get_backend_mail() that can fail. For now the mail_get_real_mail() can still be used for backwards compatibility. diffstat: src/lib-storage/fail-mail.c | 5 +++-- src/lib-storage/index/index-mail.c | 5 +++-- src/lib-storage/index/index-mail.h | 2 +- src/lib-storage/index/index-search.c | 5 ++++- src/lib-storage/mail-storage-private.h | 2 +- src/lib-storage/mail-storage.c | 9 ++++++--- src/lib-storage/mail-storage.h | 3 +++ src/lib-storage/mail.c | 15 +++++++++++++-- src/plugins/virtual/virtual-mail.c | 5 +++-- 9 files changed, 37 insertions(+), 14 deletions(-) diffs (163 lines): diff -r ac26bb157938 -r e47b3e215dec src/lib-storage/fail-mail.c --- a/src/lib-storage/fail-mail.c Tue Apr 15 16:37:26 2014 +0200 +++ b/src/lib-storage/fail-mail.c Tue Apr 15 17:06:57 2014 +0200 @@ -196,9 +196,10 @@ return -1; } -static struct mail *fail_mail_get_real_mail(struct mail *mail) +static int fail_mail_get_real_mail(struct mail *mail, struct mail **real_mail_r) { - return mail; + *real_mail_r = mail; + return 0; } static void diff -r ac26bb157938 -r e47b3e215dec src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Tue Apr 15 16:37:26 2014 +0200 +++ b/src/lib-storage/index/index-mail.c Tue Apr 15 17:06:57 2014 +0200 @@ -1211,9 +1211,10 @@ } } -struct mail *index_mail_get_real_mail(struct mail *mail) +int index_mail_get_real_mail(struct mail *mail, struct mail **real_mail_r) { - return mail; + *real_mail_r = mail; + return 0; } struct mail * diff -r ac26bb157938 -r e47b3e215dec src/lib-storage/index/index-mail.h --- a/src/lib-storage/index/index-mail.h Tue Apr 15 16:37:26 2014 +0200 +++ b/src/lib-storage/index/index-mail.h Tue Apr 15 17:06:57 2014 +0200 @@ -216,7 +216,7 @@ struct istream **stream_r); int index_mail_get_special(struct mail *_mail, enum mail_fetch_field field, const char **value_r); -struct mail *index_mail_get_real_mail(struct mail *mail); +int index_mail_get_real_mail(struct mail *mail, struct mail **real_mail_r); void index_mail_update_flags(struct mail *mail, enum modify_type modify_type, enum mail_flags flags); diff -r ac26bb157938 -r e47b3e215dec src/lib-storage/index/index-search.c --- a/src/lib-storage/index/index-search.c Tue Apr 15 16:37:26 2014 +0200 +++ b/src/lib-storage/index/index-search.c Tue Apr 15 17:06:57 2014 +0200 @@ -640,6 +640,7 @@ const enum message_header_parser_flags hdr_parser_flags = MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE; struct index_mail *imail = (struct index_mail *)ctx->cur_mail; + struct mail *real_mail; struct istream *input = NULL; struct mailbox_header_lookup_ctx *headers_ctx; struct search_header_context hdr_ctx; @@ -657,7 +658,9 @@ hdr_ctx.index_ctx = ctx; /* hdr_ctx.imail is different from imail for mails in virtual mailboxes */ - hdr_ctx.imail = (struct index_mail *)mail_get_real_mail(ctx->cur_mail); + if (mail_get_backend_mail(ctx->cur_mail, &real_mail) < 0) + return -1; + hdr_ctx.imail = (struct index_mail *)real_mail; hdr_ctx.custom_header = TRUE; hdr_ctx.args = args; diff -r ac26bb157938 -r e47b3e215dec src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Tue Apr 15 16:37:26 2014 +0200 +++ b/src/lib-storage/mail-storage-private.h Tue Apr 15 17:06:57 2014 +0200 @@ -391,7 +391,7 @@ int (*get_special)(struct mail *mail, enum mail_fetch_field field, const char **value_r); - struct mail *(*get_real_mail)(struct mail *mail); + int (*get_real_mail)(struct mail *mail, struct mail **real_mail_r); void (*update_flags)(struct mail *mail, enum modify_type modify_type, enum mail_flags flags); diff -r ac26bb157938 -r e47b3e215dec src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Tue Apr 15 16:37:26 2014 +0200 +++ b/src/lib-storage/mail-storage.c Tue Apr 15 17:06:57 2014 +0200 @@ -2116,7 +2116,7 @@ struct mailbox_transaction_context *t = ctx->transaction; struct mail_keywords *keywords = ctx->data.keywords; enum mail_flags pvt_flags = ctx->data.pvt_flags; - struct mail *real_mail; + struct mail *backend_mail; int ret; *_ctx = NULL; @@ -2129,8 +2129,11 @@ /* bypass virtual storage, so hard linking can be used whenever possible */ - real_mail = mail_get_real_mail(mail); - ret = t->box->v.copy(ctx, real_mail); + if (mail_get_backend_mail(mail, &backend_mail) < 0) { + mailbox_save_cancel(&ctx); + return -1; + } + ret = t->box->v.copy(ctx, backend_mail); if (ret == 0) { if (pvt_flags != 0) mailbox_save_add_pvt_flags(t, pvt_flags); diff -r ac26bb157938 -r e47b3e215dec src/lib-storage/mail-storage.h --- a/src/lib-storage/mail-storage.h Tue Apr 15 16:37:26 2014 +0200 +++ b/src/lib-storage/mail-storage.h Tue Apr 15 17:06:57 2014 +0200 @@ -909,6 +909,9 @@ const char **value_r); /* Returns the mail for the physical message. Normally this is the mail itself, but in virtual mailboxes it points to the backend mailbox. */ +int mail_get_backend_mail(struct mail *mail, struct mail **real_mail_r); +/* FIXME: For backwards compatibility for now, use mail_get_backend_mail() + instead. */ struct mail *mail_get_real_mail(struct mail *mail); /* Update message flags. */ diff -r ac26bb157938 -r e47b3e215dec src/lib-storage/mail.c --- a/src/lib-storage/mail.c Tue Apr 15 16:37:26 2014 +0200 +++ b/src/lib-storage/mail.c Tue Apr 15 17:06:57 2014 +0200 @@ -272,11 +272,22 @@ return 0; } -struct mail *mail_get_real_mail(struct mail *mail) +int mail_get_backend_mail(struct mail *mail, struct mail **real_mail_r) { struct mail_private *p = (struct mail_private *)mail; - return p->v.get_real_mail(mail); + return p->v.get_real_mail(mail, real_mail_r); +} + +struct mail *mail_get_real_mail(struct mail *mail) +{ + struct mail *backend_mail; + + if (mail_get_backend_mail(mail, &backend_mail) < 0) { + i_panic("FIXME: Error occurred in mail_get_real_mail(), " + "switch to using mail_get_backend_mail() instead"); + } + return backend_mail; } void mail_update_flags(struct mail *mail, enum modify_type modify_type, diff -r ac26bb157938 -r e47b3e215dec src/plugins/virtual/virtual-mail.c --- a/src/plugins/virtual/virtual-mail.c Tue Apr 15 16:37:26 2014 +0200 +++ b/src/plugins/virtual/virtual-mail.c Tue Apr 15 17:06:57 2014 +0200 @@ -387,11 +387,12 @@ return 0; } -static struct mail *virtual_mail_get_real_mail(struct mail *mail) +static int +virtual_mail_get_real_mail(struct mail *mail, struct mail **real_mail_r) { struct virtual_mail *vmail = (struct virtual_mail *)mail; - return mail_get_real_mail(vmail->backend_mail); + return mail_get_backend_mail(vmail->backend_mail, real_mail_r); } static void virtual_mail_update_pop3_uidl(struct mail *mail, const char *uidl) From dovecot at dovecot.org Tue Apr 15 15:20:50 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 15:20:50 +0000 Subject: dovecot-2.2: Replaced mail_get_real_mail() callers with mail_get... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/13ead8c6aed5 changeset: 17213:13ead8c6aed5 user: Timo Sirainen date: Tue Apr 15 17:08:34 2014 +0200 description: Replaced mail_get_real_mail() callers with mail_get_backend_mail() diffstat: src/imap/imap-fetch.c | 7 +++++-- src/plugins/lazy-expunge/lazy-expunge-plugin.c | 7 ++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diffs (40 lines): diff -r e47b3e215dec -r 13ead8c6aed5 src/imap/imap-fetch.c --- a/src/imap/imap-fetch.c Tue Apr 15 17:06:57 2014 +0200 +++ b/src/imap/imap-fetch.c Tue Apr 15 17:08:34 2014 +0200 @@ -879,8 +879,11 @@ static int fetch_x_real_uid(struct imap_fetch_context *ctx, struct mail *mail, void *context ATTR_UNUSED) { - str_printfa(ctx->state.cur_str, "X-REAL-UID %u ", - mail_get_real_mail(mail)->uid); + struct mail *real_mail; + + if (mail_get_backend_mail(mail, &real_mail) < 0) + return -1; + str_printfa(ctx->state.cur_str, "X-REAL-UID %u ", real_mail->uid); return 1; } diff -r e47b3e215dec -r 13ead8c6aed5 src/plugins/lazy-expunge/lazy-expunge-plugin.c --- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c Tue Apr 15 17:06:57 2014 +0200 +++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c Tue Apr 15 17:08:34 2014 +0200 @@ -200,13 +200,18 @@ LAZY_EXPUNGE_CONTEXT(_mail->transaction); struct lazy_expunge_mailbox_list *llist; struct mailbox *real_box; + struct mail *real_mail; struct mail_save_context *save_ctx; const char *error; int ret; /* don't copy the mail if we're expunging from lazy_expunge namespace (even if it's via a virtual mailbox) */ - real_box = mail_get_real_mail(_mail)->box; + if (mail_get_backend_mail(_mail, &real_mail) < 0) { + lt->failed = TRUE; + return; + } + real_box = real_mail->box; llist = LAZY_EXPUNGE_LIST_CONTEXT(real_box->list); if (llist != NULL && llist->internal_namespace) { mmail->super.expunge(_mail); From dovecot at dovecot.org Tue Apr 15 15:20:51 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 15:20:51 +0000 Subject: dovecot-2.2: virtual: Delay opening the backend mailboxes until ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b9bc20b1ef91 changeset: 17214:b9bc20b1ef91 user: Timo Sirainen date: Tue Apr 15 17:20:02 2014 +0200 description: virtual: Delay opening the backend mailboxes until it's really necessary. This should improve the performance a lot with mailbox_list_index=yes when opening virtual mailboxes whose backend mailboxes are mostly unchanged. diffstat: src/plugins/virtual/virtual-mail.c | 228 ++++++++++++++++++++------------- src/plugins/virtual/virtual-save.c | 33 +++- src/plugins/virtual/virtual-storage.c | 75 +++++++---- src/plugins/virtual/virtual-storage.h | 1 + src/plugins/virtual/virtual-sync.c | 55 ++++++- 5 files changed, 256 insertions(+), 136 deletions(-) diffs (truncated from 776 to 300 lines): diff -r 13ead8c6aed5 -r b9bc20b1ef91 src/plugins/virtual/virtual-mail.c --- a/src/plugins/virtual/virtual-mail.c Tue Apr 15 17:08:34 2014 +0200 +++ b/src/plugins/virtual/virtual-mail.c Tue Apr 15 17:20:02 2014 +0200 @@ -13,12 +13,14 @@ struct mailbox_header_lookup_ctx *wanted_headers; /* currently active mail */ - struct mail *backend_mail; + struct mail *cur_backend_mail; + struct virtual_mail_index_record cur_vrec; + /* all allocated mails */ ARRAY(struct mail *) backend_mails; /* mail is lost if backend_mail doesn't point to correct mail */ - unsigned int lost:1; + unsigned int cur_lost:1; }; struct mail * @@ -85,6 +87,44 @@ return NULL; } +static int backend_mail_get(struct virtual_mail *vmail, + struct mail **backend_mail_r) +{ + struct mail *mail = &vmail->imail.mail.mail; + struct virtual_mailbox *mbox = (struct virtual_mailbox *)mail->box; + struct virtual_backend_box *bbox; + + *backend_mail_r = NULL; + + if (vmail->cur_backend_mail != NULL) { + if (vmail->cur_lost) { + mail_set_expunged(&vmail->imail.mail.mail); + return -1; + } + *backend_mail_r = vmail->cur_backend_mail; + return 0; + } + + bbox = virtual_backend_box_lookup(mbox, vmail->cur_vrec.mailbox_id); + vmail->cur_backend_mail = backend_mail_find(vmail, bbox->box); + if (vmail->cur_backend_mail == NULL) { + if (mailbox_open(bbox->box) < 0) { + virtual_box_copy_error(mail->box, bbox->box); + return -1; + } + (void)virtual_mail_set_backend_mail(mail, bbox); + } + vmail->cur_lost = !mail_set_uid(vmail->cur_backend_mail, + vmail->cur_vrec.real_uid); + mail->expunged = vmail->cur_lost || vmail->cur_backend_mail->expunged; + if (vmail->cur_lost) { + mail_set_expunged(&vmail->imail.mail.mail); + return -1; + } + *backend_mail_r = vmail->cur_backend_mail; + return 0; +} + struct mail * virtual_mail_set_backend_mail(struct mail *mail, struct virtual_backend_box *bbox) @@ -94,41 +134,36 @@ struct mailbox_transaction_context *backend_trans; struct mailbox_header_lookup_ctx *backend_headers; + i_assert(bbox->box->opened); + backend_trans = virtual_transaction_get(mail->transaction, bbox->box); backend_headers = vmail->wanted_headers == NULL ? NULL : mailbox_header_lookup_init(bbox->box, vmail->wanted_headers->name); - vmail->backend_mail = mail_alloc(backend_trans, vmail->wanted_fields, - backend_headers); + vmail->cur_backend_mail = + mail_alloc(backend_trans, vmail->wanted_fields, backend_headers); if (backend_headers != NULL) mailbox_header_lookup_unref(&backend_headers); - backend_pmail = (struct mail_private *)vmail->backend_mail; + backend_pmail = (struct mail_private *)vmail->cur_backend_mail; backend_pmail->vmail = mail; - array_append(&vmail->backend_mails, &vmail->backend_mail, 1); - return vmail->backend_mail; + array_append(&vmail->backend_mails, &vmail->cur_backend_mail, 1); + return vmail->cur_backend_mail; } static void virtual_mail_set_seq(struct mail *mail, uint32_t seq, bool saving) { struct virtual_mail *vmail = (struct virtual_mail *)mail; struct virtual_mailbox *mbox = (struct virtual_mailbox *)mail->box; - struct virtual_backend_box *bbox; - const struct virtual_mail_index_record *vrec; const void *data; i_assert(!saving); mail_index_lookup_ext(mail->transaction->view, seq, mbox->virtual_ext_id, &data, NULL); - vrec = data; + memcpy(&vmail->cur_vrec, data, sizeof(vmail->cur_vrec)); - bbox = virtual_backend_box_lookup(mbox, vrec->mailbox_id); - vmail->backend_mail = backend_mail_find(vmail, bbox->box); - if (vmail->backend_mail == NULL) - (void)virtual_mail_set_backend_mail(mail, bbox); - vmail->lost = !mail_set_uid(vmail->backend_mail, vrec->real_uid); memset(&vmail->imail.data, 0, sizeof(vmail->imail.data)); p_clear(vmail->imail.mail.data_pool); @@ -136,15 +171,7 @@ mail->seq = seq; mail_index_lookup_uid(mail->transaction->view, seq, &mail->uid); - if (!vmail->lost) { - mail->expunged = vmail->backend_mail->expunged; - mail->has_nuls = vmail->backend_mail->has_nuls; - mail->has_no_nuls = vmail->backend_mail->has_no_nuls; - } else { - mail->expunged = TRUE; - mail->has_nuls = FALSE; - mail->has_no_nuls = FALSE; - } + vmail->cur_backend_mail = NULL; } static bool virtual_mail_set_uid(struct mail *mail, uint32_t uid) @@ -161,25 +188,37 @@ static void virtual_mail_set_uid_cache_updates(struct mail *mail, bool set) { struct virtual_mail *vmail = (struct virtual_mail *)mail; - struct mail_private *p = (struct mail_private *)vmail->backend_mail; + struct mail *backend_mail; + struct mail_private *p; - p->v.set_uid_cache_updates(vmail->backend_mail, set); + if (backend_mail_get(vmail, &backend_mail) < 0) + return; + p = (struct mail_private *)backend_mail; + p->v.set_uid_cache_updates(backend_mail, set); } static bool virtual_mail_prefetch(struct mail *mail) { struct virtual_mail *vmail = (struct virtual_mail *)mail; - struct mail_private *p = (struct mail_private *)vmail->backend_mail; + struct mail *backend_mail; + struct mail_private *p; - return p->v.prefetch(vmail->backend_mail); + if (backend_mail_get(vmail, &backend_mail) < 0) + return TRUE; + p = (struct mail_private *)backend_mail; + return p->v.prefetch(backend_mail); } static void virtual_mail_precache(struct mail *mail) { struct virtual_mail *vmail = (struct virtual_mail *)mail; - struct mail_private *p = (struct mail_private *)vmail->backend_mail; + struct mail *backend_mail; + struct mail_private *p; - p->v.precache(vmail->backend_mail); + if (backend_mail_get(vmail, &backend_mail) < 0) + return; + p = (struct mail_private *)backend_mail; + p->v.precache(backend_mail); } static void @@ -188,29 +227,25 @@ struct mailbox_header_lookup_ctx *headers) { struct virtual_mail *vmail = (struct virtual_mail *)mail; - struct mail_private *p = (struct mail_private *)vmail->backend_mail; + struct mail *backend_mail; + struct mail_private *p; - p->v.add_temp_wanted_fields(vmail->backend_mail, fields, headers); -} - -static int virtual_mail_handle_lost(struct virtual_mail *vmail) -{ - if (!vmail->lost) - return 0; - - mail_set_expunged(&vmail->imail.mail.mail); - return -1; + if (backend_mail_get(vmail, &backend_mail) < 0) + return; + p = (struct mail_private *)backend_mail; + p->v.add_temp_wanted_fields(backend_mail, fields, headers); } static int virtual_mail_get_parts(struct mail *mail, struct message_part **parts_r) { struct virtual_mail *vmail = (struct virtual_mail *)mail; + struct mail *backend_mail; - if (virtual_mail_handle_lost(vmail) < 0) + if (backend_mail_get(vmail, &backend_mail) < 0) return -1; - if (mail_get_parts(vmail->backend_mail, parts_r) < 0) { - virtual_box_copy_error(mail->box, vmail->backend_mail->box); + if (mail_get_parts(backend_mail, parts_r) < 0) { + virtual_box_copy_error(mail->box, backend_mail->box); return -1; } return 0; @@ -220,15 +255,16 @@ virtual_mail_get_date(struct mail *mail, time_t *date_r, int *timezone_r) { struct virtual_mail *vmail = (struct virtual_mail *)mail; + struct mail *backend_mail; int tz; if (timezone_r == NULL) timezone_r = &tz; - if (virtual_mail_handle_lost(vmail) < 0) + if (backend_mail_get(vmail, &backend_mail) < 0) return -1; - if (mail_get_date(vmail->backend_mail, date_r, timezone_r) < 0) { - virtual_box_copy_error(mail->box, vmail->backend_mail->box); + if (mail_get_date(backend_mail, date_r, timezone_r) < 0) { + virtual_box_copy_error(mail->box, backend_mail->box); return -1; } return 0; @@ -237,11 +273,12 @@ static int virtual_mail_get_received_date(struct mail *mail, time_t *date_r) { struct virtual_mail *vmail = (struct virtual_mail *)mail; + struct mail *backend_mail; - if (virtual_mail_handle_lost(vmail) < 0) + if (backend_mail_get(vmail, &backend_mail) < 0) return -1; - if (mail_get_received_date(vmail->backend_mail, date_r) < 0) { - virtual_box_copy_error(mail->box, vmail->backend_mail->box); + if (mail_get_received_date(backend_mail, date_r) < 0) { + virtual_box_copy_error(mail->box, backend_mail->box); return -1; } return 0; @@ -250,11 +287,12 @@ static int virtual_mail_get_save_date(struct mail *mail, time_t *date_r) { struct virtual_mail *vmail = (struct virtual_mail *)mail; + struct mail *backend_mail; - if (virtual_mail_handle_lost(vmail) < 0) + if (backend_mail_get(vmail, &backend_mail) < 0) return -1; - if (mail_get_save_date(vmail->backend_mail, date_r) < 0) { - virtual_box_copy_error(mail->box, vmail->backend_mail->box); + if (mail_get_save_date(backend_mail, date_r) < 0) { + virtual_box_copy_error(mail->box, backend_mail->box); return -1; } return 0; @@ -263,11 +301,12 @@ static int virtual_mail_get_virtual_mail_size(struct mail *mail, uoff_t *size_r) { struct virtual_mail *vmail = (struct virtual_mail *)mail; + struct mail *backend_mail; - if (virtual_mail_handle_lost(vmail) < 0) + if (backend_mail_get(vmail, &backend_mail) < 0) return -1; - if (mail_get_virtual_size(vmail->backend_mail, size_r) < 0) { - virtual_box_copy_error(mail->box, vmail->backend_mail->box); + if (mail_get_virtual_size(backend_mail, size_r) < 0) { + virtual_box_copy_error(mail->box, backend_mail->box); return -1; } return 0; @@ -276,11 +315,12 @@ static int virtual_mail_get_physical_size(struct mail *mail, uoff_t *size_r) { struct virtual_mail *vmail = (struct virtual_mail *)mail; + struct mail *backend_mail; - if (virtual_mail_handle_lost(vmail) < 0) + if (backend_mail_get(vmail, &backend_mail) < 0) return -1; - if (mail_get_physical_size(vmail->backend_mail, size_r) < 0) { - virtual_box_copy_error(mail->box, vmail->backend_mail->box); + if (mail_get_physical_size(backend_mail, size_r) < 0) { + virtual_box_copy_error(mail->box, backend_mail->box); return -1; } return 0; From dovecot at dovecot.org Tue Apr 15 15:33:52 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 15:33:52 +0000 Subject: dovecot-2.2: doveadm-server: Using any command line parameters c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/24e1b79e0d3d changeset: 17215:24e1b79e0d3d user: Timo Sirainen date: Tue Apr 15 17:33:12 2014 +0200 description: doveadm-server: Using any command line parameters caused crashes due to missing getopt() reset. diffstat: src/doveadm/client-connection.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (21 lines): diff -r b9bc20b1ef91 -r 24e1b79e0d3d src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Tue Apr 15 17:20:02 2014 +0200 +++ b/src/doveadm/client-connection.c Tue Apr 15 17:33:12 2014 +0200 @@ -54,6 +54,7 @@ if (doveadm_debug) ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_DEBUG; + optind = 1; getopt_args = t_strconcat("AS:u:", ctx->getopt_args, NULL); while ((c = getopt(argc, argv, getopt_args)) > 0) { switch (c) { @@ -80,9 +81,7 @@ } } } - argv += optind; - optind = 1; if (argv[0] != NULL && cmd->usage_args == NULL) { i_error("doveadm %s: Client sent unknown parameter: %s", From dovecot at dovecot.org Tue Apr 15 15:43:14 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 15:43:14 +0000 Subject: dovecot-2.2: dsync: If remote command fails, show the full comma... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3f3d4c64d7b4 changeset: 17216:3f3d4c64d7b4 user: Timo Sirainen date: Tue Apr 15 17:42:31 2014 +0200 description: dsync: If remote command fails, show the full command in the error message. diffstat: src/doveadm/dsync/doveadm-dsync.c | 27 ++++++++++++++++++--------- 1 files changed, 18 insertions(+), 9 deletions(-) diffs (64 lines): diff -r 24e1b79e0d3d -r 3f3d4c64d7b4 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Tue Apr 15 17:33:12 2014 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Tue Apr 15 17:42:31 2014 +0200 @@ -63,6 +63,7 @@ const char *remote_name; const char *local_location; pid_t remote_pid; + const char *const *remote_cmd_args; int fd_in, fd_out, fd_err; struct io *io_err; @@ -121,6 +122,8 @@ { int fd_in[2], fd_out[2], fd_err[2]; + ctx->remote_cmd_args = p_strarray_dup(ctx->ctx.pool, args); + if (pipe(fd_in) < 0 || pipe(fd_out) < 0 || pipe(fd_err) < 0) i_fatal("pipe() failed: %m"); @@ -409,19 +412,23 @@ alarm(0); } -static void cmd_dsync_log_remote_status(int status, bool remote_errors_logged) +static void cmd_dsync_log_remote_status(int status, bool remote_errors_logged, + const char *const *remote_cmd_args) { if (status == -1) ; - else if (WIFSIGNALED(status)) - i_error("Remote command died with signal %d", WTERMSIG(status)); - else if (!WIFEXITED(status)) - i_error("Remote command failed with status %d", status); - else if (WEXITSTATUS(status) == EX_TEMPFAIL && remote_errors_logged) { + else if (WIFSIGNALED(status)) { + i_error("Remote command died with signal %d: %s", WTERMSIG(status), + t_strarray_join(remote_cmd_args, " ")); + } else if (!WIFEXITED(status)) { + i_error("Remote command failed with status %d: %s", status, + t_strarray_join(remote_cmd_args, " ")); + } else if (WEXITSTATUS(status) == EX_TEMPFAIL && remote_errors_logged) { /* remote most likely already logged the error. don't bother logging another line about it */ } else if (WEXITSTATUS(status) != 0) { - i_error("Remote command returned error %d", WEXITSTATUS(status)); + i_error("Remote command returned error %d: %s", WEXITSTATUS(status), + t_strarray_join(remote_cmd_args, " ")); } } @@ -629,8 +636,10 @@ remote_errors_logged = ctx->err_stream->v_offset > 0; i_stream_destroy(&ctx->err_stream); } - if (ctx->run_type == DSYNC_RUN_TYPE_CMD) - cmd_dsync_log_remote_status(status, remote_errors_logged); + if (ctx->run_type == DSYNC_RUN_TYPE_CMD) { + cmd_dsync_log_remote_status(status, remote_errors_logged, + ctx->remote_cmd_args); + } if (ctx->io_err != NULL) io_remove(&ctx->io_err); if (ctx->fd_err != -1) From dovecot at dovecot.org Tue Apr 15 16:25:01 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 16:25:01 +0000 Subject: dovecot-2.2: replicator: Added "doveadm replicator dsync-status"... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0ec7e1e1db48 changeset: 17217:0ec7e1e1db48 user: Timo Sirainen date: Tue Apr 15 18:24:31 2014 +0200 description: replicator: Added "doveadm replicator dsync-status" command. diffstat: src/doveadm/doveadm-replicator.c | 35 +++++++++++ src/replication/replicator/doveadm-connection.c | 73 ++++++++++++++++++++---- src/replication/replicator/doveadm-connection.h | 4 +- src/replication/replicator/dsync-client.c | 38 +++++++++++++ src/replication/replicator/dsync-client.h | 14 ++++ src/replication/replicator/replicator-brain.c | 20 ++++++- src/replication/replicator/replicator-brain.h | 9 +++ src/replication/replicator/replicator.c | 2 +- 8 files changed, 178 insertions(+), 17 deletions(-) diffs (truncated from 442 to 300 lines): diff -r 3f3d4c64d7b4 -r 0ec7e1e1db48 src/doveadm/doveadm-replicator.c --- a/src/doveadm/doveadm-replicator.c Tue Apr 15 17:42:31 2014 +0200 +++ b/src/doveadm/doveadm-replicator.c Tue Apr 15 18:24:31 2014 +0200 @@ -177,6 +177,39 @@ replicator_disconnect(ctx); } +static void cmd_replicator_dsync_status(int argc, char *argv[]) +{ + struct replicator_context *ctx; + const char *line; + unsigned int i; + + ctx = cmd_replicator_init(argc, argv, "a:", cmd_replicator_dsync_status); + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header("username", "username", + DOVEADM_PRINT_HEADER_FLAG_EXPAND); + doveadm_print_header_simple("type"); + doveadm_print_header_simple("status"); + + replicator_send(ctx, "STATUS-DSYNC\n"); + while ((line = i_stream_read_next_line(ctx->input)) != NULL) { + if (*line == '\0') + break; + T_BEGIN { + const char *const *args = t_strsplit_tab(line); + + for (i = 0; i < 3; i++) { + if (args[i] == NULL) + break; + doveadm_print(args[i]); + } + for (; i < 3; i++) + doveadm_print(""); + } T_END; + } + replicator_disconnect(ctx); +} + static void cmd_replicator_replicate(int argc, char *argv[]) { struct replicator_context *ctx; @@ -247,6 +280,8 @@ struct doveadm_cmd doveadm_cmd_replicator[] = { { cmd_replicator_status, "replicator status", "[-a ] []" }, + { cmd_replicator_dsync_status, "replicator dsync-status", + "[-a ]" }, { cmd_replicator_replicate, "replicator replicate", "[-a ] [-p ] " }, { cmd_replicator_remove, "replicator remove", diff -r 3f3d4c64d7b4 -r 0ec7e1e1db48 src/replication/replicator/doveadm-connection.c --- a/src/replication/replicator/doveadm-connection.c Tue Apr 15 17:42:31 2014 +0200 +++ b/src/replication/replicator/doveadm-connection.c Tue Apr 15 18:24:31 2014 +0200 @@ -1,13 +1,16 @@ /* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "array.h" #include "connection.h" #include "ostream.h" #include "str.h" #include "strescape.h" #include "wildcard-match.h" #include "master-service.h" +#include "replicator-brain.h" #include "replicator-queue.h" +#include "dsync-client.h" #include "doveadm-connection.h" #include @@ -17,12 +20,14 @@ struct doveadm_connection { struct connection conn; - struct replicator_queue *queue; + struct replicator_brain *brain; }; static struct connection_list *doveadm_connections; static int client_input_status_overview(struct doveadm_connection *client) { + struct replicator_queue *queue = + replicator_brain_get_queue(client->brain); struct replicator_queue_iter *iter; struct replicator_user *user; enum replication_priority priority; @@ -36,12 +41,11 @@ pending_full_resync_count = 0; user_count = 0; - iter = replicator_queue_iter_init(client->queue); + iter = replicator_queue_iter_init(queue); while ((user = replicator_queue_iter_next(iter)) != NULL) { if (user->priority != REPLICATION_PRIORITY_NONE) pending_counts[user->priority]++; - else if (replicator_queue_want_sync_now(client->queue, - user, &next_secs)) { + else if (replicator_queue_want_sync_now(queue, user, &next_secs)) { if (user->last_sync_failed) pending_failed_count++; else @@ -74,6 +78,8 @@ static int client_input_status(struct doveadm_connection *client, const char *const *args) { + struct replicator_queue *queue = + replicator_brain_get_queue(client->brain); struct replicator_queue_iter *iter; struct replicator_user *user; const char *mask = args[0]; @@ -82,7 +88,7 @@ if (mask == NULL) return client_input_status_overview(client); - iter = replicator_queue_iter_init(client->queue); + iter = replicator_queue_iter_init(queue); while ((user = replicator_queue_iter_next(iter)) != NULL) { if (!wildcard_match(user->username, mask)) continue; @@ -103,8 +109,42 @@ } static int +client_input_status_dsyncs(struct doveadm_connection *client) +{ + string_t *str = t_str_new(256); + const ARRAY_TYPE(dsync_client) *clients; + struct dsync_client *const *clientp; + + clients = replicator_brain_get_dsync_clients(client->brain); + array_foreach(clients, clientp) { + str_append_tabescaped(str, dsync_client_get_username(*clientp)); + str_append_c(str, '\t'); + switch (dsync_client_get_type(*clientp)) { + case DSYNC_TYPE_FULL: + str_append(str, "full"); + break; + case DSYNC_TYPE_NORMAL: + str_append(str, "normal"); + break; + case DSYNC_TYPE_INCREMENTAL: + str_append(str, "incremental"); + break; + } + str_append_c(str, '\t'); + str_append_tabescaped(str, dsync_client_get_state(*clientp)); + str_append_c(str, '\n'); + } + + str_append_c(str, '\n'); + o_stream_send(client->conn.output, str_data(str), str_len(str)); + return 0; +} + +static int client_input_replicate(struct doveadm_connection *client, const char *const *args) { + struct replicator_queue *queue = + replicator_brain_get_queue(client->brain); struct replicator_queue_iter *iter; struct replicator_user *user; const char *usermask; @@ -122,17 +162,17 @@ } usermask = args[1]; if (strchr(usermask, '*') == NULL && strchr(usermask, '?') == NULL) { - replicator_queue_add(client->queue, usermask, priority); + replicator_queue_add(queue, usermask, priority); o_stream_send_str(client->conn.output, "+1\n"); return 0; } match_count = 0; - iter = replicator_queue_iter_init(client->queue); + iter = replicator_queue_iter_init(queue); while ((user = replicator_queue_iter_next(iter)) != NULL) { if (!wildcard_match(user->username, usermask)) continue; - replicator_queue_add(client->queue, user->username, priority); + replicator_queue_add(queue, user->username, priority); match_count++; } replicator_queue_iter_deinit(&iter); @@ -144,6 +184,8 @@ static int client_input_remove(struct doveadm_connection *client, const char *const *args) { + struct replicator_queue *queue = + replicator_brain_get_queue(client->brain); struct replicator_user *user; /* */ @@ -151,11 +193,11 @@ i_error("%s: REMOVE: Invalid parameters", client->conn.name); return -1; } - user = replicator_queue_lookup(client->queue, args[0]); + user = replicator_queue_lookup(queue, args[0]); if (user == NULL) o_stream_send_str(client->conn.output, "-User not found\n"); else { - replicator_queue_remove(client->queue, &user); + replicator_queue_remove(queue, &user); o_stream_send_str(client->conn.output, "+\n"); } return 0; @@ -164,6 +206,8 @@ static int client_input_notify(struct doveadm_connection *client, const char *const *args) { + struct replicator_queue *queue = + replicator_brain_get_queue(client->brain); struct replicator_user *user; /* */ @@ -172,8 +216,7 @@ return -1; } - user = replicator_queue_add(client->queue, args[0], - REPLICATION_PRIORITY_NONE); + user = replicator_queue_add(queue, args[0], REPLICATION_PRIORITY_NONE); if (args[1][0] == 'f') user->last_full_sync = ioloop_time; user->last_fast_sync = ioloop_time; @@ -200,6 +243,8 @@ if (strcmp(cmd, "STATUS") == 0) return client_input_status(client, args); + else if (strcmp(cmd, "STATUS-DSYNC") == 0) + return client_input_status_dsyncs(client); else if (strcmp(cmd, "REPLICATE") == 0) return client_input_replicate(client, args); else if (strcmp(cmd, "REMOVE") == 0) @@ -220,12 +265,12 @@ master_service_client_connection_destroyed(master_service); } -void doveadm_connection_create(struct replicator_queue *queue, int fd) +void doveadm_connection_create(struct replicator_brain *brain, int fd) { struct doveadm_connection *client; client = i_new(struct doveadm_connection, 1); - client->queue = queue; + client->brain = brain; connection_init_server(doveadm_connections, &client->conn, "(doveadm client)", fd, fd); } diff -r 3f3d4c64d7b4 -r 0ec7e1e1db48 src/replication/replicator/doveadm-connection.h --- a/src/replication/replicator/doveadm-connection.h Tue Apr 15 17:42:31 2014 +0200 +++ b/src/replication/replicator/doveadm-connection.h Tue Apr 15 18:24:31 2014 +0200 @@ -1,7 +1,9 @@ #ifndef DOVEADM_CONNECTION_H #define DOVEADM_CONNECTION_H -void doveadm_connection_create(struct replicator_queue *queue, int fd); +struct replicator_brain; + +void doveadm_connection_create(struct replicator_brain *brain, int fd); void doveadm_connections_init(void); void doveadm_connections_deinit(void); diff -r 3f3d4c64d7b4 -r 0ec7e1e1db48 src/replication/replicator/dsync-client.c --- a/src/replication/replicator/dsync-client.c Tue Apr 15 17:42:31 2014 +0200 +++ b/src/replication/replicator/dsync-client.c Tue Apr 15 18:24:31 2014 +0200 @@ -23,7 +23,9 @@ struct timeout *to; char *dsync_params; + char *username; char *state; + enum dsync_type sync_type; dsync_callback_t *callback; void *context; @@ -69,6 +71,7 @@ client->cmd_sent = FALSE; client->handshaked = FALSE; i_free_and_null(client->state); + i_free_and_null(client->username); if (client->fd == -1) return; @@ -195,9 +198,16 @@ i_assert(callback != NULL); i_assert(!dsync_client_is_busy(client)); + client->username = i_strdup(username); client->cmd_sent = TRUE; client->callback = callback; client->context = context; + if (full) + client->sync_type = DSYNC_TYPE_FULL; + else if (state != NULL && state[0] != '\0') + client->sync_type = DSYNC_TYPE_INCREMENTAL; + else + client->sync_type = DSYNC_TYPE_NORMAL; From dovecot at dovecot.org Tue Apr 15 20:28:43 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 20:28:43 +0000 Subject: dovecot-2.2: fts-lucene: Don't allocate large amounts of memory ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2ef62e1f44f2 changeset: 17218:2ef62e1f44f2 user: Timo Sirainen date: Tue Apr 15 22:28:26 2014 +0200 description: fts-lucene: Don't allocate large amounts of memory from stack. This probably only happened for huge header values. diffstat: src/plugins/fts-lucene/lucene-wrapper.cc | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-) diffs (24 lines): diff -r 0ec7e1e1db48 -r 2ef62e1f44f2 src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Tue Apr 15 18:24:31 2014 +0200 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Tue Apr 15 22:28:26 2014 +0200 @@ -535,7 +535,11 @@ } datasize = uni_utf8_strlen_n(data, size) + 1; - wchar_t dest[datasize]; + wchar_t *dest, *dest_free = NULL; + if (datasize < 4096) + dest = (wchar_t *)t_malloc(datasize); + else + dest = dest_free = (wchar_t *)i_malloc(datasize); lucene_utf8_n_to_tchar(data, size, dest, datasize); lucene_data_translate(index, dest, datasize); @@ -556,6 +560,7 @@ index->cur_analyzer = guess_analyzer(index, data, size); index->doc->add(*_CLNEW Field(_T("body"), dest, Field::STORE_NO | Field::INDEX_TOKENIZED)); } + i_free(dest_free); return 0; } From dovecot at dovecot.org Tue Apr 15 21:02:44 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 21:02:44 +0000 Subject: dovecot-2.2: dsync: Fixed giving -n "" followed by other -n para... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f0b8cf119b7f changeset: 17219:f0b8cf119b7f user: Timo Sirainen date: Tue Apr 15 23:02:11 2014 +0200 description: dsync: Fixed giving -n "" followed by other -n parameters. diffstat: src/doveadm/dsync/dsync-brain.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (17 lines): diff -r 2ef62e1f44f2 -r f0b8cf119b7f src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Tue Apr 15 22:28:26 2014 +0200 +++ b/src/doveadm/dsync/dsync-brain.c Tue Apr 15 23:02:11 2014 +0200 @@ -159,11 +159,11 @@ p_array_init(&brain->sync_namespaces, brain->pool, array_count(&set->sync_namespaces)); array_foreach(&set->sync_namespaces, nsp) { - if (str_len(sync_ns_str) > 0) - str_append_c(sync_ns_str, '\n'); str_append(sync_ns_str, (*nsp)->prefix); + str_append_c(sync_ns_str, '\n'); array_append(&brain->sync_namespaces, nsp, 1); } + str_delete(sync_ns_str, str_len(sync_ns_str)-1); } brain->sync_box = p_strdup(brain->pool, set->sync_box); brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL : From pigeonhole at rename-it.nl Tue Apr 15 21:12:50 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 15 Apr 2014 23:12:50 +0200 Subject: dovecot-2.2-pigeonhole: Dovecot change: Replaced mail_get_real_m... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/db8f7bbda283 changeset: 1846:db8f7bbda283 user: Stephan Bosch date: Tue Apr 15 23:11:46 2014 +0200 description: Dovecot change: Replaced mail_get_real_mail() with mail_get_backend_mail(). diffstat: src/lib-sieve/sieve-actions.c | 18 ++++++++++++++---- src/lib-sieve/util/edit-mail.c | 6 ++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diffs (59 lines): diff -r 8c53243b341f -r db8f7bbda283 src/lib-sieve/sieve-actions.c --- a/src/lib-sieve/sieve-actions.c Tue Mar 04 22:28:56 2014 +0100 +++ b/src/lib-sieve/sieve-actions.c Tue Apr 15 23:11:46 2014 +0200 @@ -479,9 +479,9 @@ (struct act_store_transaction *) tr_context; struct mail *mail = ( action->mail != NULL ? action->mail : aenv->msgdata->mail ); - struct mail *real_mail = mail_get_real_mail(mail); struct mail_save_context *save_ctx; struct mail_keywords *keywords = NULL; + bool backends_equal = FALSE; int status = SIEVE_EXEC_OK; /* Verify transaction */ @@ -504,12 +504,22 @@ return SIEVE_EXEC_FAILURE; } + /* If the message originates from the target mailbox, only update the flags * and keywords (if not read-only) */ - if ( mailbox_backends_equal(trans->box, mail->box) || - (real_mail != mail && mailbox_backends_equal(trans->box, real_mail->box)) ) - { + if ( mailbox_backends_equal(trans->box, mail->box) ) { + backends_equal = TRUE; + } else { + struct mail *real_mail; + + if ( mail_get_backend_mail(mail, &real_mail) < 0 ) + return SIEVE_EXEC_FAILURE; + if ( real_mail != mail && + mailbox_backends_equal(trans->box, real_mail->box) ) + backends_equal = TRUE; + } + if (backends_equal) { trans->redundant = TRUE; if ( trans->flags_altered && !mailbox_is_readonly(mail->box) ) { diff -r 8c53243b341f -r db8f7bbda283 src/lib-sieve/util/edit-mail.c --- a/src/lib-sieve/util/edit-mail.c Tue Mar 04 22:28:56 2014 +0100 +++ b/src/lib-sieve/util/edit-mail.c Tue Apr 15 23:11:46 2014 +0200 @@ -1379,11 +1379,13 @@ return edmail->wrapped->v.get_special(&edmail->wrapped->mail, field, value_r); } -static struct mail *edit_mail_get_real_mail(struct mail *mail) +static int +edit_mail_get_real_mail(struct mail *mail, struct mail **real_mail_r) { struct edit_mail *edmail = (struct edit_mail *)mail; - return edit_mail_get_mail(edmail); + *real_mail_r = edit_mail_get_mail(edmail); + return 0; } static void edit_mail_update_flags From dovecot at dovecot.org Tue Apr 15 21:23:59 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 21:23:59 +0000 Subject: dovecot-2.2: replicator: Fixed crash caused by "doveadm replicat... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/be480cfcd7bf changeset: 17220:be480cfcd7bf user: Timo Sirainen date: Tue Apr 15 23:23:37 2014 +0200 description: replicator: Fixed crash caused by "doveadm replicator dsync-status" in some situations. diffstat: src/replication/replicator/doveadm-connection.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (18 lines): diff -r f0b8cf119b7f -r be480cfcd7bf src/replication/replicator/doveadm-connection.c --- a/src/replication/replicator/doveadm-connection.c Tue Apr 15 23:02:11 2014 +0200 +++ b/src/replication/replicator/doveadm-connection.c Tue Apr 15 23:23:37 2014 +0200 @@ -114,10 +114,13 @@ string_t *str = t_str_new(256); const ARRAY_TYPE(dsync_client) *clients; struct dsync_client *const *clientp; + const char *username; clients = replicator_brain_get_dsync_clients(client->brain); array_foreach(clients, clientp) { - str_append_tabescaped(str, dsync_client_get_username(*clientp)); + username = dsync_client_get_username(*clientp); + if (username != NULL) + str_append_tabescaped(str, username); str_append_c(str, '\t'); switch (dsync_client_get_type(*clientp)) { case DSYNC_TYPE_FULL: From dovecot at dovecot.org Tue Apr 15 21:42:06 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 21:42:06 +0000 Subject: dovecot-2.2: dsync: Fix to previous commit to actually compile.. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e75851b65900 changeset: 17221:e75851b65900 user: Timo Sirainen date: Tue Apr 15 23:41:50 2014 +0200 description: dsync: Fix to previous commit to actually compile.. diffstat: src/doveadm/dsync/dsync-brain.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r be480cfcd7bf -r e75851b65900 src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Tue Apr 15 23:23:37 2014 +0200 +++ b/src/doveadm/dsync/dsync-brain.c Tue Apr 15 23:41:50 2014 +0200 @@ -163,7 +163,7 @@ str_append_c(sync_ns_str, '\n'); array_append(&brain->sync_namespaces, nsp, 1); } - str_delete(sync_ns_str, str_len(sync_ns_str)-1); + str_delete(sync_ns_str, str_len(sync_ns_str)-1, 1); } brain->sync_box = p_strdup(brain->pool, set->sync_box); brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL : From dovecot at dovecot.org Tue Apr 15 21:45:10 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 21:45:10 +0000 Subject: dovecot-2.2: lib-storage: mailbox_alloc_guid() shouldn't refresh... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/26b468dc7f11 changeset: 17222:26b468dc7f11 user: Timo Sirainen date: Tue Apr 15 23:44:54 2014 +0200 description: lib-storage: mailbox_alloc_guid() shouldn't refresh GUID cache on every nonexistent GUID. Refresh it only if we know it has had some changes (made by our process) or timestamp has changed. This should handle the cases that are currently important (=single dsync run). diffstat: src/lib-storage/mail-storage.c | 15 +++++++++++++-- src/lib-storage/mailbox-guid-cache.c | 11 ++++++++++- src/lib-storage/mailbox-list-private.h | 2 ++ 3 files changed, 25 insertions(+), 3 deletions(-) diffs (104 lines): diff -r e75851b65900 -r 26b468dc7f11 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Tue Apr 15 23:41:50 2014 +0200 +++ b/src/lib-storage/mail-storage.c Tue Apr 15 23:44:54 2014 +0200 @@ -1239,18 +1239,25 @@ box->creating = TRUE; ret = box->v.create_box(box, update, directory); box->creating = FALSE; + if (ret == 0) + box->list->guid_cache_invalidated = TRUE; return ret; } int mailbox_update(struct mailbox *box, const struct mailbox_update *update) { + int ret; + i_assert(update->min_next_uid == 0 || update->min_first_recent_uid == 0 || update->min_first_recent_uid <= update->min_next_uid); if (mailbox_verify_existing_name(box) < 0) return -1; - return box->v.update_box(box, update); + ret = box->v.update_box(box, update); + if (!guid_128_is_empty(update->mailbox_guid)) + box->list->guid_cache_invalidated = TRUE; + return ret; } int mailbox_mark_index_deleted(struct mailbox *box, bool del) @@ -1425,7 +1432,11 @@ return -1; } - return src->v.rename_box(src, dest); + if (src->v.rename_box(src, dest) < 0) + return -1; + src->list->guid_cache_invalidated = TRUE; + dest->list->guid_cache_invalidated = TRUE; + return 0; } int mailbox_set_subscribed(struct mailbox *box, bool set) diff -r e75851b65900 -r 26b468dc7f11 src/lib-storage/mailbox-guid-cache.c --- a/src/lib-storage/mailbox-guid-cache.c Tue Apr 15 23:41:50 2014 +0200 +++ b/src/lib-storage/mailbox-guid-cache.c Tue Apr 15 23:44:54 2014 +0200 @@ -1,6 +1,7 @@ /* Copyright (c) 2005-2014 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "ioloop.h" #include "hash.h" #include "mail-storage.h" #include "mailbox-list-private.h" @@ -11,6 +12,12 @@ const char *vname; }; +static bool mailbox_guid_cache_want_refresh(struct mailbox_list *list) +{ + return list->guid_cache_invalidated || + list->guid_cache_last_update < ioloop_time; +} + int mailbox_guid_cache_find(struct mailbox_list *list, const guid_128_t guid, const char **vname_r) { @@ -22,7 +29,7 @@ rec = hash_table_lookup(list->guid_cache, guid_p); } else { rec = hash_table_lookup(list->guid_cache, guid_p); - if (rec == NULL) { + if (rec == NULL && mailbox_guid_cache_want_refresh(list)) { mailbox_guid_cache_refresh(list); rec = hash_table_lookup(list->guid_cache, guid_p); } @@ -53,6 +60,8 @@ hash_table_clear(list->guid_cache, TRUE); p_clear(list->guid_cache_pool); } + list->guid_cache_last_update = ioloop_time; + list->guid_cache_invalidated = FALSE; list->guid_cache_errors = FALSE; ctx = mailbox_list_iter_init(list, "*", diff -r e75851b65900 -r 26b468dc7f11 src/lib-storage/mailbox-list-private.h --- a/src/lib-storage/mailbox-list-private.h Tue Apr 15 23:41:50 2014 +0200 +++ b/src/lib-storage/mailbox-list-private.h Tue Apr 15 23:44:54 2014 +0200 @@ -122,6 +122,7 @@ struct mailbox_log *changelog; time_t changelog_timestamp; + time_t guid_cache_last_update; pool_t guid_cache_pool; HASH_TABLE(uint8_t *, struct mailbox_guid_cache_rec *) guid_cache; bool guid_cache_errors; @@ -133,6 +134,7 @@ ARRAY(union mailbox_list_module_context *) module_contexts; unsigned int index_root_dir_created:1; + unsigned int guid_cache_invalidated:1; }; union mailbox_list_iterate_module_context { From dovecot at dovecot.org Tue Apr 15 22:04:52 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 22:04:52 +0000 Subject: dovecot-2.2: lib-storage: Further changes to mailbox_alloc_guid(... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c2755572d31e changeset: 17223:c2755572d31e user: Timo Sirainen date: Wed Apr 16 00:04:29 2014 +0200 description: lib-storage: Further changes to mailbox_alloc_guid() caching. We'll now cache the GUIDs forever and force invalidation after mailbox rename or when mailbox GUIDs have been updated. This is what dsync needs, especially for good performance, and currently there aren't any other users for this API. diffstat: src/lib-storage/mail-storage.c | 2 +- src/lib-storage/mailbox-guid-cache.c | 13 ++++--------- src/lib-storage/mailbox-list-private.h | 2 +- 3 files changed, 6 insertions(+), 11 deletions(-) diffs (72 lines): diff -r 26b468dc7f11 -r c2755572d31e src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Tue Apr 15 23:44:54 2014 +0200 +++ b/src/lib-storage/mail-storage.c Wed Apr 16 00:04:29 2014 +0200 @@ -1240,7 +1240,7 @@ ret = box->v.create_box(box, update, directory); box->creating = FALSE; if (ret == 0) - box->list->guid_cache_invalidated = TRUE; + box->list->guid_cache_updated = TRUE; return ret; } diff -r 26b468dc7f11 -r c2755572d31e src/lib-storage/mailbox-guid-cache.c --- a/src/lib-storage/mailbox-guid-cache.c Tue Apr 15 23:44:54 2014 +0200 +++ b/src/lib-storage/mailbox-guid-cache.c Wed Apr 16 00:04:29 2014 +0200 @@ -12,24 +12,19 @@ const char *vname; }; -static bool mailbox_guid_cache_want_refresh(struct mailbox_list *list) -{ - return list->guid_cache_invalidated || - list->guid_cache_last_update < ioloop_time; -} - int mailbox_guid_cache_find(struct mailbox_list *list, const guid_128_t guid, const char **vname_r) { const struct mailbox_guid_cache_rec *rec; const uint8_t *guid_p = guid; - if (!hash_table_is_created(list->guid_cache)) { + if (!hash_table_is_created(list->guid_cache) || + list->guid_cache_invalidated) { mailbox_guid_cache_refresh(list); rec = hash_table_lookup(list->guid_cache, guid_p); } else { rec = hash_table_lookup(list->guid_cache, guid_p); - if (rec == NULL && mailbox_guid_cache_want_refresh(list)) { + if (rec == NULL && list->guid_cache_updated) { mailbox_guid_cache_refresh(list); rec = hash_table_lookup(list->guid_cache, guid_p); } @@ -60,8 +55,8 @@ hash_table_clear(list->guid_cache, TRUE); p_clear(list->guid_cache_pool); } - list->guid_cache_last_update = ioloop_time; list->guid_cache_invalidated = FALSE; + list->guid_cache_updated = FALSE; list->guid_cache_errors = FALSE; ctx = mailbox_list_iter_init(list, "*", diff -r 26b468dc7f11 -r c2755572d31e src/lib-storage/mailbox-list-private.h --- a/src/lib-storage/mailbox-list-private.h Tue Apr 15 23:44:54 2014 +0200 +++ b/src/lib-storage/mailbox-list-private.h Wed Apr 16 00:04:29 2014 +0200 @@ -122,7 +122,6 @@ struct mailbox_log *changelog; time_t changelog_timestamp; - time_t guid_cache_last_update; pool_t guid_cache_pool; HASH_TABLE(uint8_t *, struct mailbox_guid_cache_rec *) guid_cache; bool guid_cache_errors; @@ -134,6 +133,7 @@ ARRAY(union mailbox_list_module_context *) module_contexts; unsigned int index_root_dir_created:1; + unsigned int guid_cache_updated:1; unsigned int guid_cache_invalidated:1; }; From dovecot at dovecot.org Tue Apr 15 22:14:01 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Apr 2014 22:14:01 +0000 Subject: dovecot-2.2: fts-lucene: Fix to previous change. We allocated wa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5e42a8c18b51 changeset: 17224:5e42a8c18b51 user: Timo Sirainen date: Wed Apr 16 00:13:42 2014 +0200 description: fts-lucene: Fix to previous change. We allocated way too little memory. diffstat: src/plugins/fts-lucene/lucene-wrapper.cc | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (15 lines): diff -r c2755572d31e -r 5e42a8c18b51 src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Wed Apr 16 00:04:29 2014 +0200 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Wed Apr 16 00:13:42 2014 +0200 @@ -537,9 +537,9 @@ datasize = uni_utf8_strlen_n(data, size) + 1; wchar_t *dest, *dest_free = NULL; if (datasize < 4096) - dest = (wchar_t *)t_malloc(datasize); + dest = t_new(wchar_t, datasize); else - dest = dest_free = (wchar_t *)i_malloc(datasize); + dest = dest_free = i_new(wchar_t, datasize); lucene_utf8_n_to_tchar(data, size, dest, datasize); lucene_data_translate(index, dest, datasize); From dovecot at dovecot.org Wed Apr 16 13:45:27 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Apr 2014 13:45:27 +0000 Subject: dovecot-2.2: replicator: Improve logging import/export errors. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/112c6951c1df changeset: 17225:112c6951c1df user: Timo Sirainen date: Wed Apr 16 10:40:28 2014 +0200 description: replicator: Improve logging import/export errors. diffstat: src/replication/replicator/replicator-queue.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 5e42a8c18b51 -r 112c6951c1df src/replication/replicator/replicator-queue.c --- a/src/replication/replicator/replicator-queue.c Wed Apr 16 00:13:42 2014 +0200 +++ b/src/replication/replicator/replicator-queue.c Wed Apr 16 10:40:28 2014 +0200 @@ -371,8 +371,10 @@ break; } } - if (input->stream_errno != 0) + if (input->stream_errno != 0) { + i_error("read(%s) failed: %s", path, i_stream_get_error(input)); ret = -1; + } i_stream_destroy(&input); return ret; } @@ -417,7 +419,7 @@ } replicator_queue_iter_deinit(&iter); if (o_stream_nfinish(output) < 0) { - i_error("write(%s) failed: %m", path); + i_error("write(%s) failed: %s", path, o_stream_get_error(output)); ret = -1; } o_stream_destroy(&output); From dovecot at dovecot.org Wed Apr 16 13:45:27 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Apr 2014 13:45:27 +0000 Subject: dovecot-2.2: fts-solr: Removed unnecessary code Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3745f5c73224 changeset: 17226:3745f5c73224 user: Timo Sirainen date: Wed Apr 16 15:44:46 2014 +0200 description: fts-solr: Removed unnecessary code diffstat: src/plugins/fts-solr/solr-connection.c | 8 -------- 1 files changed, 0 insertions(+), 8 deletions(-) diffs (21 lines): diff -r 112c6951c1df -r 3745f5c73224 src/plugins/fts-solr/solr-connection.c --- a/src/plugins/fts-solr/solr-connection.c Wed Apr 16 10:40:28 2014 +0200 +++ b/src/plugins/fts-solr/solr-connection.c Wed Apr 16 15:44:46 2014 +0200 @@ -448,17 +448,9 @@ solr_connection_update_response(const struct http_response *response, struct solr_connection *conn) { - if (response == NULL) { - /* request failed */ - i_error("fts_solr: HTTP POST request failed"); - conn->request_status = -1; - return; - } - if (response->status / 100 != 2) { i_error("fts_solr: Indexing failed: %s", response->reason); conn->request_status = -1; - return; } } From dovecot at dovecot.org Wed Apr 16 14:05:57 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Apr 2014 14:05:57 +0000 Subject: dovecot-2.2: maildir: Added maildir_empty_new setting to avoid k... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d7b0f653d1ba changeset: 17227:d7b0f653d1ba user: Timo Sirainen date: Wed Apr 16 16:05:12 2014 +0200 description: maildir: Added maildir_empty_new setting to avoid keeping mails in new/ directory. This is useful for example when using an external MDA that calls "doveadm index" and the user hasn't opened the mailbox for a long time. Also this removes the delay that user may see when opening a rarely accessed mailbox, which renames a ton of files to cur/ while the user is waiting. diffstat: doc/example-config/conf.d/10-mail.conf | 4 ++++ src/lib-storage/index/maildir/maildir-settings.c | 4 +++- src/lib-storage/index/maildir/maildir-settings.h | 1 + src/lib-storage/index/maildir/maildir-sync.c | 6 +++--- 4 files changed, 11 insertions(+), 4 deletions(-) diffs (62 lines): diff -r 3745f5c73224 -r d7b0f653d1ba doc/example-config/conf.d/10-mail.conf --- a/doc/example-config/conf.d/10-mail.conf Wed Apr 16 15:44:46 2014 +0200 +++ b/doc/example-config/conf.d/10-mail.conf Wed Apr 16 16:05:12 2014 +0200 @@ -266,6 +266,10 @@ # broken size. The performance hit for enabling this is very small. #maildir_broken_filename_sizes = no +# Always move mails from new/ directory to cur/, even when the \Recent flags +# aren't being reset. +#maildir_empty_new = no + ## ## mbox-specific settings ## diff -r 3745f5c73224 -r d7b0f653d1ba src/lib-storage/index/maildir/maildir-settings.c --- a/src/lib-storage/index/maildir/maildir-settings.c Wed Apr 16 15:44:46 2014 +0200 +++ b/src/lib-storage/index/maildir/maildir-settings.c Wed Apr 16 16:05:12 2014 +0200 @@ -15,6 +15,7 @@ DEF(SET_BOOL, maildir_copy_with_hardlinks), DEF(SET_BOOL, maildir_very_dirty_syncs), DEF(SET_BOOL, maildir_broken_filename_sizes), + DEF(SET_BOOL, maildir_empty_new), SETTING_DEFINE_LIST_END }; @@ -22,7 +23,8 @@ static const struct maildir_settings maildir_default_settings = { .maildir_copy_with_hardlinks = TRUE, .maildir_very_dirty_syncs = FALSE, - .maildir_broken_filename_sizes = FALSE + .maildir_broken_filename_sizes = FALSE, + .maildir_empty_new = FALSE }; static const struct setting_parser_info maildir_setting_parser_info = { diff -r 3745f5c73224 -r d7b0f653d1ba src/lib-storage/index/maildir/maildir-settings.h --- a/src/lib-storage/index/maildir/maildir-settings.h Wed Apr 16 15:44:46 2014 +0200 +++ b/src/lib-storage/index/maildir/maildir-settings.h Wed Apr 16 16:05:12 2014 +0200 @@ -5,6 +5,7 @@ bool maildir_copy_with_hardlinks; bool maildir_very_dirty_syncs; bool maildir_broken_filename_sizes; + bool maildir_empty_new; }; const struct setting_parser_info *maildir_get_setting_parser_info(void); diff -r 3745f5c73224 -r d7b0f653d1ba src/lib-storage/index/maildir/maildir-sync.c --- a/src/lib-storage/index/maildir/maildir-sync.c Wed Apr 16 15:44:46 2014 +0200 +++ b/src/lib-storage/index/maildir/maildir-sync.c Wed Apr 16 16:05:12 2014 +0200 @@ -469,9 +469,9 @@ src = t_str_new(1024); dest = t_str_new(1024); - move_new = new_dir && !mailbox_is_readonly(&ctx->mbox->box) && - (ctx->mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0 && - ctx->locked; + move_new = new_dir && ctx->locked && + ((ctx->mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0 || + ctx->mbox->storage->set->maildir_empty_new); errno = 0; for (; (dp = readdir(dirp)) != NULL; errno = 0) { From dovecot at dovecot.org Wed Apr 16 16:20:36 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Apr 2014 16:20:36 +0000 Subject: dovecot-2.2: doveadm replicator dsync-status: Show replication t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/195945490dc9 changeset: 17228:195945490dc9 user: Timo Sirainen date: Wed Apr 16 18:19:57 2014 +0200 description: doveadm replicator dsync-status: Show replication type as "-" for non-working dsyncs. diffstat: src/replication/replicator/doveadm-connection.c | 27 +++++++++++++----------- 1 files changed, 15 insertions(+), 12 deletions(-) diffs (38 lines): diff -r d7b0f653d1ba -r 195945490dc9 src/replication/replicator/doveadm-connection.c --- a/src/replication/replicator/doveadm-connection.c Wed Apr 16 16:05:12 2014 +0200 +++ b/src/replication/replicator/doveadm-connection.c Wed Apr 16 18:19:57 2014 +0200 @@ -119,19 +119,22 @@ clients = replicator_brain_get_dsync_clients(client->brain); array_foreach(clients, clientp) { username = dsync_client_get_username(*clientp); - if (username != NULL) + if (username != NULL) { str_append_tabescaped(str, username); - str_append_c(str, '\t'); - switch (dsync_client_get_type(*clientp)) { - case DSYNC_TYPE_FULL: - str_append(str, "full"); - break; - case DSYNC_TYPE_NORMAL: - str_append(str, "normal"); - break; - case DSYNC_TYPE_INCREMENTAL: - str_append(str, "incremental"); - break; + str_append_c(str, '\t'); + switch (dsync_client_get_type(*clientp)) { + case DSYNC_TYPE_FULL: + str_append(str, "full"); + break; + case DSYNC_TYPE_NORMAL: + str_append(str, "normal"); + break; + case DSYNC_TYPE_INCREMENTAL: + str_append(str, "incremental"); + break; + } + } else { + str_append(str, "\t-"); } str_append_c(str, '\t'); str_append_tabescaped(str, dsync_client_get_state(*clientp)); From dovecot at dovecot.org Thu Apr 17 07:46:50 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Apr 2014 07:46:50 +0000 Subject: dovecot-2.2: fts: Free fts-parser on failures. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8618c884170d changeset: 17229:8618c884170d user: Timo Sirainen date: Thu Apr 17 09:45:55 2014 +0200 description: fts: Free fts-parser on failures. diffstat: src/plugins/fts/fts-build-mail.c | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 195945490dc9 -r 8618c884170d src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Wed Apr 16 18:19:57 2014 +0200 +++ b/src/plugins/fts/fts-build-mail.c Thu Apr 17 09:45:55 2014 +0200 @@ -352,8 +352,12 @@ body_added = TRUE; } } - if (ret == 0 && ctx.body_parser != NULL) - ret = fts_body_parser_finish(&ctx); + if (ctx.body_parser != NULL) { + if (ret == 0) + ret = fts_body_parser_finish(&ctx); + else + fts_parser_deinit(&ctx.body_parser); + } if (ret == 0 && body_part && !skip_body && !body_added) { /* make sure body is added even when it doesn't exist */ block.data = NULL; block.size = 0; From dovecot at dovecot.org Thu Apr 17 08:30:11 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Apr 2014 08:30:11 +0000 Subject: dovecot-2.2: lib-http: http_client_request_finish_payload() no l... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f68ec8bd4afa changeset: 17230:f68ec8bd4afa user: Timo Sirainen date: Thu Apr 17 10:17:09 2014 +0200 description: lib-http: http_client_request_finish_payload() no longer waits for the entire result payload. The caller can now read it instead, or just discard it by not reading it. diffstat: src/lib-http/http-client-request.c | 16 ++++++++++++---- 1 files changed, 12 insertions(+), 4 deletions(-) diffs (34 lines): diff -r 8618c884170d -r f68ec8bd4afa src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Thu Apr 17 09:45:55 2014 +0200 +++ b/src/lib-http/http-client-request.c Thu Apr 17 10:17:09 2014 +0200 @@ -485,7 +485,7 @@ client->ioloop = io_loop_create(); http_client_switch_ioloop(client); - while (req->state < HTTP_REQUEST_STATE_FINISHED) { + while (req->state < HTTP_REQUEST_STATE_PAYLOAD_IN) { http_client_request_debug(req, "Waiting for request to finish"); if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT) @@ -505,10 +505,18 @@ io_loop_set_current(client->ioloop); io_loop_destroy(&client->ioloop); - if (req->state == HTTP_REQUEST_STATE_FINISHED) + switch (req->state) { + case HTTP_REQUEST_STATE_PAYLOAD_IN: + case HTTP_REQUEST_STATE_FINISHED: ret = 1; - else - ret = (req->state == HTTP_REQUEST_STATE_ABORTED ? -1 : 0); + break; + case HTTP_REQUEST_STATE_ABORTED: + ret = -1; + break; + default: + ret = 0; + break; + } req->payload_wait = FALSE; http_client_request_unref(_req); From dovecot at dovecot.org Thu Apr 17 08:30:16 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Apr 2014 08:30:16 +0000 Subject: dovecot-2.2: fts: Added unload() method to fts-parsers to allow ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e4119adae01d changeset: 17231:e4119adae01d user: Timo Sirainen date: Thu Apr 17 10:26:11 2014 +0200 description: fts: Added unload() method to fts-parsers to allow them to cleanup before plugin unload. diffstat: src/plugins/fts/fts-parser-html.c | 3 ++- src/plugins/fts/fts-parser-script.c | 3 ++- src/plugins/fts/fts-parser.c | 10 ++++++++++ src/plugins/fts/fts-parser.h | 3 +++ src/plugins/fts/fts-plugin.c | 2 ++ 5 files changed, 19 insertions(+), 2 deletions(-) diffs (75 lines): diff -r f68ec8bd4afa -r e4119adae01d src/plugins/fts/fts-parser-html.c --- a/src/plugins/fts/fts-parser-html.c Thu Apr 17 10:17:09 2014 +0200 +++ b/src/plugins/fts/fts-parser-html.c Thu Apr 17 10:26:11 2014 +0200 @@ -262,5 +262,6 @@ struct fts_parser_vfuncs fts_parser_html = { fts_parser_html_try_init, fts_parser_html_more, - fts_parser_html_deinit + fts_parser_html_deinit, + NULL }; diff -r f68ec8bd4afa -r e4119adae01d src/plugins/fts/fts-parser-script.c --- a/src/plugins/fts/fts-parser-script.c Thu Apr 17 10:17:09 2014 +0200 +++ b/src/plugins/fts/fts-parser-script.c Thu Apr 17 10:26:11 2014 +0200 @@ -266,5 +266,6 @@ struct fts_parser_vfuncs fts_parser_script = { fts_parser_script_try_init, fts_parser_script_more, - fts_parser_script_deinit + fts_parser_script_deinit, + NULL }; diff -r f68ec8bd4afa -r e4119adae01d src/plugins/fts/fts-parser.c --- a/src/plugins/fts/fts-parser.c Thu Apr 17 10:17:09 2014 +0200 +++ b/src/plugins/fts/fts-parser.c Thu Apr 17 10:26:11 2014 +0200 @@ -89,3 +89,13 @@ else i_free(parser); } + +void fts_parsers_unload(void) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(parsers); i++) { + if (parsers[i]->unload != NULL) + parsers[i]->unload(); + } +} diff -r f68ec8bd4afa -r e4119adae01d src/plugins/fts/fts-parser.h --- a/src/plugins/fts/fts-parser.h Thu Apr 17 10:17:09 2014 +0200 +++ b/src/plugins/fts/fts-parser.h Thu Apr 17 10:26:11 2014 +0200 @@ -10,6 +10,7 @@ const char *content_disposition); void (*more)(struct fts_parser *parser, struct message_block *block); void (*deinit)(struct fts_parser *parser); + void (*unload)(void); }; struct fts_parser { @@ -31,4 +32,6 @@ void fts_parser_more(struct fts_parser *parser, struct message_block *block); void fts_parser_deinit(struct fts_parser **parser); +void fts_parsers_unload(void); + #endif diff -r f68ec8bd4afa -r e4119adae01d src/plugins/fts/fts-plugin.c --- a/src/plugins/fts/fts-plugin.c Thu Apr 17 10:17:09 2014 +0200 +++ b/src/plugins/fts/fts-plugin.c Thu Apr 17 10:26:11 2014 +0200 @@ -2,6 +2,7 @@ #include "lib.h" #include "mail-storage-hooks.h" +#include "fts-parser.h" #include "fts-storage.h" #include "fts-plugin.h" @@ -22,5 +23,6 @@ void fts_plugin_deinit(void) { + fts_parsers_unload(); mail_storage_hooks_remove(&fts_mail_storage_hooks); } From dovecot at dovecot.org Thu Apr 17 08:30:16 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Apr 2014 08:30:16 +0000 Subject: dovecot-2.2: fts: Never put text/plain contents through fts-pars... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/45e7980f6507 changeset: 17232:45e7980f6507 user: Timo Sirainen date: Thu Apr 17 10:27:43 2014 +0200 description: fts: Never put text/plain contents through fts-parsers. diffstat: src/plugins/fts/fts-parser.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (16 lines): diff -r e4119adae01d -r 45e7980f6507 src/plugins/fts/fts-parser.c --- a/src/plugins/fts/fts-parser.c Thu Apr 17 10:26:11 2014 +0200 +++ b/src/plugins/fts/fts-parser.c Thu Apr 17 10:27:43 2014 +0200 @@ -17,6 +17,12 @@ { unsigned int i; + if (strcmp(content_type, "text/plain") == 0) { + /* we probably don't want/need to allow parsers to handle + plaintext? */ + return NULL; + } + for (i = 0; i < N_ELEMENTS(parsers); i++) { *parser_r = parsers[i]->try_init(user, content_type, content_disposition); From dovecot at dovecot.org Thu Apr 17 08:30:16 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Apr 2014 08:30:16 +0000 Subject: dovecot-2.2: fts: Added fts-parser support for Tika Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3db2ab503759 changeset: 17233:3db2ab503759 user: Timo Sirainen date: Thu Apr 17 10:29:10 2014 +0200 description: fts: Added fts-parser support for Tika It can be enabled by setting: plugin { fts_tika = http://localhost:9998/tika/ } diffstat: src/plugins/fts/Makefile.am | 2 + src/plugins/fts/fts-parser-tika.c | 224 ++++++++++++++++++++++++++++++++++++++ src/plugins/fts/fts-parser.c | 3 +- src/plugins/fts/fts-parser.h | 1 + 4 files changed, 229 insertions(+), 1 deletions(-) diffs (271 lines): diff -r 45e7980f6507 -r 3db2ab503759 src/plugins/fts/Makefile.am --- a/src/plugins/fts/Makefile.am Thu Apr 17 10:27:43 2014 +0200 +++ b/src/plugins/fts/Makefile.am Thu Apr 17 10:29:10 2014 +0200 @@ -4,6 +4,7 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-http \ -I$(top_srcdir)/src/lib-mail \ -I$(top_srcdir)/src/lib-index \ -I$(top_srcdir)/src/lib-storage \ @@ -25,6 +26,7 @@ fts-parser.c \ fts-parser-html.c \ fts-parser-script.c \ + fts-parser-tika.c \ fts-plugin.c \ fts-search.c \ fts-search-serialize.c \ diff -r 45e7980f6507 -r 3db2ab503759 src/plugins/fts/fts-parser-tika.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/fts/fts-parser-tika.c Thu Apr 17 10:29:10 2014 +0200 @@ -0,0 +1,224 @@ +/* Copyright (c) 2014 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "module-context.h" +#include "http-url.h" +#include "http-client.h" +#include "message-parser.h" +#include "mail-user.h" +#include "fts-parser.h" + +#define TIKA_USER_CONTEXT(obj) \ + MODULE_CONTEXT(obj, fts_parser_tika_user_module) + +struct fts_parser_tika_user { + union mail_user_module_context module_ctx; + struct http_url *http_url; +}; + +struct tika_fts_parser { + struct fts_parser parser; + struct mail_user *user; + struct http_client_request *http_req; + + struct ioloop *ioloop; + struct io *io; + struct istream *payload; + + bool http_req_finished; + bool failed; +}; + +static struct http_client *tika_http_client = NULL; +static MODULE_CONTEXT_DEFINE_INIT(fts_parser_tika_user_module, + &mail_user_module_register); + +static int +tika_get_http_client_url(struct mail_user *user, struct http_url **http_url_r) +{ + struct fts_parser_tika_user *tuser = TIKA_USER_CONTEXT(user); + struct http_client_settings http_set; + const char *url, *error; + + url = mail_user_plugin_getenv(user, "fts_tika"); + if (url == NULL) { + /* fts_tika disabled */ + return -1; + } + + if (tuser != NULL) { + *http_url_r = tuser->http_url; + return *http_url_r == NULL ? -1 : 0; + } + + tuser = p_new(user->pool, struct fts_parser_tika_user, 1); + MODULE_CONTEXT_SET(user, fts_parser_tika_user_module, tuser); + + if (http_url_parse(url, NULL, 0, user->pool, + &tuser->http_url, &error) < 0) { + i_error("fts_tika: Failed to parse HTTP url %s: %s", url, error); + return -1; + } + + if (tika_http_client == NULL) { + memset(&http_set, 0, sizeof(http_set)); + http_set.max_idle_time_msecs = 100; + http_set.max_parallel_connections = 1; + http_set.max_pipelined_requests = 1; + http_set.max_redirects = 1; + http_set.max_attempts = 3; + http_set.debug = user->mail_debug; + tika_http_client = http_client_init(&http_set); + } + *http_url_r = tuser->http_url; + return 0; +} + +static void +fts_tika_parser_response(const struct http_response *response, + struct tika_fts_parser *parser) +{ + i_assert(parser->payload == NULL); + + switch (response->status) { + case 200: + /* read response */ + i_stream_ref(response->payload); + parser->payload = response->payload; + break; + case 204: /* empty response */ + case 422: /* Unprocessable Entity */ + if (parser->user->mail_debug) { + i_debug("fts_tika: PUT %s failed: %s", + mail_user_plugin_getenv(parser->user, "fts_tika"), + response->reason); + } + parser->payload = i_stream_create_from_data("", 0); + break; + default: + i_error("fts_tika: PUT %s failed: %s", + mail_user_plugin_getenv(parser->user, "fts_tika"), + response->reason); + parser->failed = TRUE; + break; + } + parser->http_req_finished = TRUE; + io_loop_stop(current_ioloop); +} + +static struct fts_parser * +fts_parser_tika_try_init(struct mail_user *user, const char *content_type, + const char *content_disposition) +{ + struct tika_fts_parser *parser; + struct http_url *http_url; + struct http_client_request *http_req; + + if (tika_get_http_client_url(user, &http_url) < 0) + return NULL; + + parser = i_new(struct tika_fts_parser, 1); + parser->parser.v = fts_parser_tika; + parser->user = user; + + http_req = http_client_request(tika_http_client, "PUT", + http_url->host_name, + t_strconcat(http_url->path, http_url->enc_query, NULL), + fts_tika_parser_response, parser); + http_client_request_set_port(http_req, http_url->port); + http_client_request_set_ssl(http_req, http_url->have_ssl); + http_client_request_add_header(http_req, "Content-Type", content_type); + http_client_request_add_header(http_req, "Content-Disposition", + content_disposition); + http_client_request_add_header(http_req, "Accept", "text/plain"); + + parser->http_req = http_req; + return &parser->parser; +} + +static void fts_parser_tika_more(struct fts_parser *_parser, + struct message_block *block) +{ + struct tika_fts_parser *parser = (struct tika_fts_parser *)_parser; + const unsigned char *data; + size_t size; + ssize_t ret; + + if (block->size > 0) { + /* first we'll send everything to Tika */ + if (!parser->failed && + http_client_request_send_payload(&parser->http_req, + block->data, + block->size) < 0) + parser->failed = TRUE; + block->size = 0; + return; + } + + if (parser->payload == NULL) { + /* read the result from Tika */ + if (!parser->failed && + http_client_request_finish_payload(&parser->http_req) < 0) + parser->failed = TRUE; + if (!parser->failed && parser->payload == NULL) + http_client_wait(tika_http_client); + if (parser->failed) + return; + i_assert(parser->payload != NULL); + } + /* continue returning data from Tika */ + while ((ret = i_stream_read_data(parser->payload, &data, &size, 0)) == 0) { + if (parser->failed) + return; + /* wait for more input from Tika */ + if (parser->ioloop == NULL) { + parser->ioloop = io_loop_create(); + parser->io = io_add_istream(parser->payload, io_loop_stop, + current_ioloop); + } else { + io_loop_set_current(parser->ioloop); + } + io_loop_run(current_ioloop); + } + if (size > 0) { + i_assert(ret > 0); + block->data = data; + block->size = size; + i_stream_skip(parser->payload, size); + } else { + /* finished */ + i_assert(ret == -1); + } +} + +static void fts_parser_tika_deinit(struct fts_parser *_parser) +{ + struct tika_fts_parser *parser = (struct tika_fts_parser *)_parser; + + if (parser->ioloop != NULL) { + io_remove(&parser->io); + io_loop_destroy(&parser->ioloop); + } + if (parser->payload != NULL) + i_stream_unref(&parser->payload); + /* FIXME: kludgy, http_req should be NULL here if we don't want to + free it. requires lib-http changes. */ + if (parser->http_req != NULL && !parser->http_req_finished) + http_client_request_abort(&parser->http_req); + i_free(parser); +} + +static void fts_parser_tika_unload(void) +{ + if (tika_http_client != NULL) + http_client_deinit(&tika_http_client); +} + +struct fts_parser_vfuncs fts_parser_tika = { + fts_parser_tika_try_init, + fts_parser_tika_more, + fts_parser_tika_deinit, + fts_parser_tika_unload +}; diff -r 45e7980f6507 -r 3db2ab503759 src/plugins/fts/fts-parser.c --- a/src/plugins/fts/fts-parser.c Thu Apr 17 10:27:43 2014 +0200 +++ b/src/plugins/fts/fts-parser.c Thu Apr 17 10:29:10 2014 +0200 @@ -8,7 +8,8 @@ const struct fts_parser_vfuncs *parsers[] = { &fts_parser_html, - &fts_parser_script + &fts_parser_script, + &fts_parser_tika }; bool fts_parser_init(struct mail_user *user, diff -r 45e7980f6507 -r 3db2ab503759 src/plugins/fts/fts-parser.h --- a/src/plugins/fts/fts-parser.h Thu Apr 17 10:27:43 2014 +0200 +++ b/src/plugins/fts/fts-parser.h Thu Apr 17 10:29:10 2014 +0200 @@ -20,6 +20,7 @@ extern struct fts_parser_vfuncs fts_parser_html; extern struct fts_parser_vfuncs fts_parser_script; +extern struct fts_parser_vfuncs fts_parser_tika; bool fts_parser_init(struct mail_user *user, const char *content_type, const char *content_disposition, From dovecot at dovecot.org Thu Apr 17 10:27:21 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Apr 2014 10:27:21 +0000 Subject: dovecot-2.2: imapc: Fixed fetching specific headers without imap... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2fcb1d28ddbc changeset: 17234:2fcb1d28ddbc user: Timo Sirainen date: Thu Apr 17 12:26:46 2014 +0200 description: imapc: Fixed fetching specific headers without imapc_features=fetch-headers diffstat: src/lib-storage/index/imapc/imapc-mail.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (22 lines): diff -r 3db2ab503759 -r 2fcb1d28ddbc src/lib-storage/index/imapc/imapc-mail.c --- a/src/lib-storage/index/imapc/imapc-mail.c Thu Apr 17 10:29:10 2014 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail.c Thu Apr 17 12:26:46 2014 +0200 @@ -194,6 +194,8 @@ { struct mailbox_header_lookup_ctx *headers; const char *header_names[2]; + const unsigned char *data; + size_t size; struct istream *input; int ret; @@ -204,6 +206,9 @@ mailbox_header_lookup_unref(&headers); if (ret < 0) return -1; + + while (i_stream_read_data(input, &data, &size, 0) > 0) + i_stream_skip(input, size); /* the header should cached now. */ return index_mail_get_headers(_mail, field, decode_to_utf8, value_r); } From dovecot at dovecot.org Thu Apr 17 12:26:09 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Apr 2014 12:26:09 +0000 Subject: dovecot-2.2: auth: Use special AUTH_SUBSYS_DB/MECH parameters as... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9b095cec9332 changeset: 17235:9b095cec9332 user: Timo Sirainen date: Thu Apr 17 14:21:55 2014 +0200 description: auth: Use special AUTH_SUBSYS_DB/MECH parameters as auth_request_log*() subsystem. This avoids hardcoded strings all over the place and also allows assigning the correct passdb/userdb name for log messages generated by generic passdb/userdb code, which doesn't know exactly where it was called from. diffstat: src/auth/auth-request-handler.c | 6 +- src/auth/auth-request.c | 79 +++++++++++++++++++++++----------------- src/auth/auth-request.h | 4 ++ src/auth/db-checkpassword.c | 53 +++++++++++++-------------- src/auth/db-dict.c | 6 +- src/auth/db-ldap.c | 28 +++++++------- src/auth/db-passwd-file.c | 10 ++-- src/auth/mech-anonymous.c | 2 +- src/auth/mech-apop.c | 9 ++-- src/auth/mech-cram-md5.c | 6 +- src/auth/mech-digest-md5.c | 6 +- src/auth/mech-dovecot-token.c | 4 +- src/auth/mech-external.c | 6 +- src/auth/mech-gssapi.c | 43 +++++++++++---------- src/auth/mech-login.c | 2 +- src/auth/mech-ntlm.c | 16 ++++---- src/auth/mech-otp.c | 16 ++++---- src/auth/mech-plain.c | 6 +- src/auth/mech-rpa.c | 8 ++-- src/auth/mech-scram-sha1.c | 7 +-- src/auth/mech-skey.c | 13 +++--- src/auth/mech-winbind.c | 14 +++--- src/auth/passdb-blocking.c | 4 +- src/auth/passdb-bsdauth.c | 8 ++-- src/auth/passdb-cache.c | 19 +++++---- src/auth/passdb-checkpassword.c | 2 +- src/auth/passdb-dict.c | 10 ++-- src/auth/passdb-imap.c | 8 ++-- src/auth/passdb-ldap.c | 28 +++++++------ src/auth/passdb-pam.c | 33 +++++++++------- src/auth/passdb-passwd-file.c | 2 +- src/auth/passdb-passwd.c | 10 ++-- src/auth/passdb-shadow.c | 8 ++-- src/auth/passdb-sia.c | 4 +- src/auth/passdb-sql.c | 22 ++++++----- src/auth/passdb-static.c | 4 +- src/auth/passdb-vpopmail.c | 6 +- src/auth/passdb.c | 12 +++--- src/auth/userdb-dict.c | 12 +++--- src/auth/userdb-ldap.c | 6 +- src/auth/userdb-nss.c | 10 ++-- src/auth/userdb-passwd.c | 6 +- src/auth/userdb-prefetch.c | 8 ++-- src/auth/userdb-sql.c | 10 ++-- src/auth/userdb-static.c | 2 +- src/auth/userdb-vpopmail.c | 12 +++--- src/auth/userdb.c | 4 +- 47 files changed, 312 insertions(+), 282 deletions(-) diffs (truncated from 2363 to 300 lines): diff -r 2fcb1d28ddbc -r 9b095cec9332 src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Thu Apr 17 12:26:46 2014 +0200 +++ b/src/auth/auth-request-handler.c Thu Apr 17 14:21:55 2014 +0200 @@ -391,7 +391,7 @@ { string_t *str = t_str_new(128); - auth_request_log_info(request, request->mech->mech_name, "%s", reason); + auth_request_log_info(request, AUTH_SUBSYS_MECH, "%s", reason); str_printfa(str, "FAIL\t%u\treason=", request->id); str_append_tabescaped(str, reason); @@ -406,12 +406,12 @@ if (request->state != AUTH_REQUEST_STATE_MECH_CONTINUE) { /* client's fault */ - auth_request_log_error(request, request->mech->mech_name, + auth_request_log_error(request, AUTH_SUBSYS_MECH, "Request %u.%u timed out after %u secs, state=%d", request->handler->client_pid, request->id, secs, request->state); } else if (request->set->verbose) { - auth_request_log_info(request, request->mech->mech_name, + auth_request_log_info(request, AUTH_SUBSYS_MECH, "Request timed out waiting for client to continue authentication " "(%u secs)", secs); } diff -r 2fcb1d28ddbc -r 9b095cec9332 src/auth/auth-request.c --- a/src/auth/auth-request.c Thu Apr 17 12:26:46 2014 +0200 +++ b/src/auth/auth-request.c Thu Apr 17 14:21:55 2014 +0200 @@ -28,6 +28,7 @@ #include #include +#define AUTH_SUBSYS_PROXY "proxy" #define AUTH_DNS_SOCKET_PATH "dns-client" #define AUTH_DNS_DEFAULT_TIMEOUT_MSECS (1000*10) #define AUTH_DNS_WARN_MSECS 500 @@ -39,6 +40,8 @@ struct dns_lookup *dns_lookup; }; +const char auth_default_subsystems[2]; + unsigned int auth_request_state_count[AUTH_REQUEST_STATE_MAX]; static void get_log_prefix(string_t *str, struct auth_request *auth_request, @@ -479,7 +482,8 @@ return; /* master login successful. update user and master_user variables. */ - auth_request_log_info(request, "passdb", "Master user logging in as %s", + auth_request_log_info(request, AUTH_SUBSYS_DB, + "Master user logging in as %s", request->requested_login_user); request->master_user = request->user; @@ -540,7 +544,7 @@ lookup returned that user doesn't exist in it. internal errors are fatal here. */ if (*result != PASSDB_RESULT_INTERNAL_FAILURE) { - auth_request_log_info(request, "passdb", + auth_request_log_info(request, AUTH_SUBSYS_DB, "User found from deny passdb"); *result = PASSDB_RESULT_USER_DISABLED; } @@ -699,7 +703,7 @@ if (passdb_cache_verify_plain(request, cache_key, request->mech_password, &result, TRUE)) { - auth_request_log_info(request, "passdb", + auth_request_log_info(request, AUTH_SUBSYS_DB, "Falling back to expired data from cache"); } } @@ -731,7 +735,7 @@ /* no masterdbs, master logins not supported */ i_assert(request->requested_login_user != NULL); - auth_request_log_info(request, "passdb", + auth_request_log_info(request, AUTH_SUBSYS_MECH, "Attempted master login with no master passdbs " "(trying to log in as user: %s)", request->requested_login_user); @@ -754,7 +758,7 @@ } if (password_has_illegal_chars(password)) { - auth_request_log_info(request, "passdb", + auth_request_log_info(request, AUTH_SUBSYS_DB, "Attempted login with password having illegal chars"); callback(PASSDB_RESULT_USER_UNKNOWN, request); return; @@ -805,7 +809,7 @@ } else { if (request->set->debug_passwords && result == PASSDB_RESULT_OK) { - auth_request_log_debug(request, "password", + auth_request_log_debug(request, AUTH_SUBSYS_DB, "Credentials: %s", binary_to_hex(credentials, size)); } @@ -845,7 +849,7 @@ if (passdb_cache_lookup_credentials(request, cache_key, &cache_cred, &cache_scheme, &result, TRUE)) { - auth_request_log_info(request, "passdb", + auth_request_log_info(request, AUTH_SUBSYS_DB, "Falling back to expired data from cache"); passdb_handle_credentials( result, cache_cred, cache_scheme, @@ -895,7 +899,7 @@ if (passdb->iface.lookup_credentials == NULL) { /* this passdb doesn't support credentials */ - auth_request_log_debug(request, "password", + auth_request_log_debug(request, AUTH_SUBSYS_DB, "passdb doesn't support credential lookups"); auth_request_lookup_credentials_callback( PASSDB_RESULT_SCHEME_NOT_AVAILABLE, @@ -976,11 +980,13 @@ value = auth_cache_lookup(passdb_cache, request, key, &node, &expired, &neg_expired); if (value == NULL || (expired && !use_expired)) { - auth_request_log_debug(request, "userdb-cache", - value == NULL ? "miss" : "expired"); + auth_request_log_debug(request, AUTH_SUBSYS_DB, + value == NULL ? "userdb cache miss" : + "userdb cache expired"); return FALSE; } - auth_request_log_debug(request, "userdb-cache", "hit: %s", value); + auth_request_log_debug(request, AUTH_SUBSYS_DB, + "userdb cache hit: %s", value); if (*value == '\0') { /* negative cache entry */ @@ -1075,11 +1081,10 @@ /* this was an actual login attempt, the user should have been found. */ if (auth_request_get_auth(request)->userdbs->next == NULL) { - auth_request_log_error(request, "userdb", - "user not found from userdb %s", - request->userdb->userdb->iface->name); + auth_request_log_error(request, AUTH_SUBSYS_DB, + "user not found from userdb"); } else { - auth_request_log_error(request, "userdb", + auth_request_log_error(request, AUTH_SUBSYS_MECH, "user not found from any userdbs"); } } @@ -1098,7 +1103,7 @@ if (auth_request_lookup_user_cache(request, cache_key, &reply, &result, TRUE)) { request->userdb_reply = reply; - auth_request_log_info(request, "userdb", + auth_request_log_info(request, AUTH_SUBSYS_DB, "Falling back to expired data from cache"); } } @@ -1269,7 +1274,7 @@ if (request->requested_login_user == NULL) return FALSE; - auth_request_log_debug(request, "auth", + auth_request_log_debug(request, AUTH_SUBSYS_DB, "Master user lookup for login: %s", request->requested_login_user); return TRUE; @@ -1285,18 +1290,18 @@ if (request->remote_ip.family == 0) { /* IP not known */ - auth_request_log_info(request, "passdb", + auth_request_log_info(request, AUTH_SUBSYS_DB, "allow_nets check failed: Remote IP not known"); request->failed = TRUE; return; } for (net = t_strsplit_spaces(networks, ", "); *net != NULL; net++) { - auth_request_log_debug(request, "auth", + auth_request_log_debug(request, AUTH_SUBSYS_DB, "allow_nets: Matching for network %s", *net); if (net_parse_range(*net, &net_ip, &bits) < 0) { - auth_request_log_info(request, "passdb", + auth_request_log_info(request, AUTH_SUBSYS_DB, "allow_nets: Invalid network '%s'", *net); } @@ -1307,7 +1312,7 @@ } if (!found) { - auth_request_log_info(request, "passdb", + auth_request_log_info(request, AUTH_SUBSYS_DB, "allow_nets check failed: IP not in allowed networks"); } request->failed = !found; @@ -1318,8 +1323,7 @@ const char *default_scheme, bool noscheme) { if (request->passdb_password != NULL) { - auth_request_log_error(request, - request->passdb->passdb->iface.name, + auth_request_log_error(request, AUTH_SUBSYS_DB, "Multiple password values not supported"); return; } @@ -1382,7 +1386,7 @@ return FALSE; if (strcmp(request->user, new_value) != 0) { - auth_request_log_debug(request, "auth", + auth_request_log_debug(request, AUTH_SUBSYS_DB, "username changed %s -> %s", request->user, new_value); request->user = p_strdup(request->pool, new_value); @@ -1441,8 +1445,7 @@ if (password != NULL) { (void)password_get_scheme(&password); if (*password != '\0') { - auth_request_log_error(request, - request->passdb->passdb->iface.name, + auth_request_log_error(request, AUTH_SUBSYS_DB, "nopassword set but password is " "non-empty"); return; @@ -1533,7 +1536,7 @@ var_expand(path, path_template, auth_request_get_var_expand_table(request, NULL)); if (stat(str_c(path), &st) < 0) { - auth_request_log_error(request, "uidgid_file", + auth_request_log_error(request, AUTH_SUBSYS_DB, "stat(%s) failed: %m", str_c(path)); } else { auth_fields_add(request->userdb_reply, @@ -1635,7 +1638,7 @@ } else { /* add only one */ if (values[1] != NULL) { - auth_request_log_warning(request, "userdb", + auth_request_log_warning(request, AUTH_SUBSYS_DB, "Multiple values found for '%s', " "using value '%s'", name, *values); } @@ -1723,13 +1726,13 @@ i_assert(host != NULL); if (result->ret != 0) { - auth_request_log_error(request, "proxy", + auth_request_log_error(request, AUTH_SUBSYS_PROXY, "DNS lookup for %s failed: %s", host, result->error); request->internal_failure = TRUE; auth_request_proxy_finish_failure(request); } else { if (result->msecs > AUTH_DNS_WARN_MSECS) { - auth_request_log_warning(request, "proxy", + auth_request_log_warning(request, AUTH_SUBSYS_PROXY, "DNS lookup for %s took %u.%03u s", host, result->msecs/1000, result->msecs % 1000); } @@ -1766,7 +1769,7 @@ value = auth_fields_find(request->extra_fields, "proxy_timeout"); if (value != NULL) { if (str_to_uint(value, &secs) < 0) { - auth_request_log_error(request, "proxy", + auth_request_log_error(request, AUTH_SUBSYS_PROXY, "Invalid proxy_timeout value: %s", value); } else { dns_set.timeout_msecs = secs*1000; @@ -2133,9 +2136,19 @@ const char *subsystem) { #define MAX_LOG_USERNAME_LEN 64 - const char *ip; + const char *ip, *name; - str_append(str, subsystem); + if (subsystem == AUTH_SUBSYS_DB) { + if (!auth_request->userdb_lookup) + name = auth_request->passdb->passdb->iface.name; + else + name = auth_request->userdb->userdb->iface->name; + } else if (subsystem == AUTH_SUBSYS_MECH) { + name = t_str_lcase(auth_request->mech->mech_name); + } else { + name = subsystem; + } + str_append(str, name); str_append_c(str, '('); if (auth_request->user == NULL) diff -r 2fcb1d28ddbc -r 9b095cec9332 src/auth/auth-request.h --- a/src/auth/auth-request.h Thu Apr 17 12:26:46 2014 +0200 +++ b/src/auth/auth-request.h Thu Apr 17 14:21:55 2014 +0200 @@ -153,6 +153,10 @@ extern const struct var_expand_table From dovecot at dovecot.org Thu Apr 17 12:26:09 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Apr 2014 12:26:09 +0000 Subject: dovecot-2.2: auth: If passdb/userdb has a name, use it for the l... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/917946c08f83 changeset: 17236:917946c08f83 user: Timo Sirainen date: Thu Apr 17 14:24:18 2014 +0200 description: auth: If passdb/userdb has a name, use it for the log prefix instead of the driver name. diffstat: src/auth/auth-request.c | 13 +++++++++---- src/auth/auth-settings.c | 8 ++++++-- src/auth/auth-settings.h | 2 ++ src/auth/auth.c | 1 + 4 files changed, 18 insertions(+), 6 deletions(-) diffs (106 lines): diff -r 9b095cec9332 -r 917946c08f83 src/auth/auth-request.c --- a/src/auth/auth-request.c Thu Apr 17 14:21:55 2014 +0200 +++ b/src/auth/auth-request.c Thu Apr 17 14:24:18 2014 +0200 @@ -2139,10 +2139,15 @@ const char *ip, *name; if (subsystem == AUTH_SUBSYS_DB) { - if (!auth_request->userdb_lookup) - name = auth_request->passdb->passdb->iface.name; - else - name = auth_request->userdb->userdb->iface->name; + if (!auth_request->userdb_lookup) { + name = auth_request->passdb->set->name[0] != '\0' ? + auth_request->passdb->set->name : + auth_request->passdb->passdb->iface.name; + } else { + name = auth_request->userdb->set->name[0] != '\0' ? + auth_request->userdb->set->name : + auth_request->userdb->userdb->iface->name; + } } else if (subsystem == AUTH_SUBSYS_MECH) { name = t_str_lcase(auth_request->mech->mech_name); } else { diff -r 9b095cec9332 -r 917946c08f83 src/auth/auth-settings.c --- a/src/auth/auth-settings.c Thu Apr 17 14:21:55 2014 +0200 +++ b/src/auth/auth-settings.c Thu Apr 17 14:24:18 2014 +0200 @@ -107,6 +107,7 @@ { type, #name, offsetof(struct auth_passdb_settings, name), NULL } static const struct setting_define auth_passdb_setting_defines[] = { + DEF(SET_STR, name), DEF(SET_STR, driver), DEF(SET_STR, args), DEF(SET_STR, default_fields), @@ -125,6 +126,7 @@ }; static const struct auth_passdb_settings auth_passdb_default_settings = { + .name = "", .driver = "", .args = "", .default_fields = "", @@ -144,7 +146,7 @@ .defines = auth_passdb_setting_defines, .defaults = &auth_passdb_default_settings, - .type_offset = (size_t)-1, + .type_offset = offsetof(struct auth_passdb_settings, name), .struct_size = sizeof(struct auth_passdb_settings), .parent_offset = (size_t)-1, @@ -158,6 +160,7 @@ { type, #name, offsetof(struct auth_userdb_settings, name), NULL } static const struct setting_define auth_userdb_setting_defines[] = { + DEF(SET_STR, name), DEF(SET_STR, driver), DEF(SET_STR, args), DEF(SET_STR, default_fields), @@ -173,6 +176,7 @@ static const struct auth_userdb_settings auth_userdb_default_settings = { /* NOTE: when adding fields, update also auth.c:userdb_dummy_set */ + .name = "", .driver = "", .args = "", .default_fields = "", @@ -188,7 +192,7 @@ .defines = auth_userdb_setting_defines, .defaults = &auth_userdb_default_settings, - .type_offset = (size_t)-1, + .type_offset = offsetof(struct auth_userdb_settings, name), .struct_size = sizeof(struct auth_userdb_settings), .parent_offset = (size_t)-1, diff -r 9b095cec9332 -r 917946c08f83 src/auth/auth-settings.h --- a/src/auth/auth-settings.h Thu Apr 17 14:21:55 2014 +0200 +++ b/src/auth/auth-settings.h Thu Apr 17 14:24:18 2014 +0200 @@ -5,6 +5,7 @@ struct master_service_settings_output; struct auth_passdb_settings { + const char *name; const char *driver; const char *args; const char *default_fields; @@ -20,6 +21,7 @@ }; struct auth_userdb_settings { + const char *name; const char *driver; const char *args; const char *default_fields; diff -r 9b095cec9332 -r 917946c08f83 src/auth/auth.c --- a/src/auth/auth.c Thu Apr 17 14:21:55 2014 +0200 +++ b/src/auth/auth.c Thu Apr 17 14:24:18 2014 +0200 @@ -10,6 +10,7 @@ #include "auth.h" struct auth_userdb_settings userdb_dummy_set = { + .name = "", .driver = "static", .args = "", .default_fields = "", From dovecot at dovecot.org Thu Apr 17 14:13:34 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Apr 2014 14:13:34 +0000 Subject: dovecot-2.2: lib-mail: Moved struct message_part to a separate m... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/63f12bb366b0 changeset: 17237:63f12bb366b0 user: Timo Sirainen date: Thu Apr 17 16:04:13 2014 +0200 description: lib-mail: Moved struct message_part to a separate message-part.h diffstat: src/lib-mail/Makefile.am | 1 + src/lib-mail/message-parser.h | 34 +--------------------------------- src/lib-mail/message-part.h | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 33 deletions(-) diffs (104 lines): diff -r 917946c08f83 -r 63f12bb366b0 src/lib-mail/Makefile.am --- a/src/lib-mail/Makefile.am Thu Apr 17 14:24:18 2014 +0200 +++ b/src/lib-mail/Makefile.am Thu Apr 17 16:04:13 2014 +0200 @@ -51,6 +51,7 @@ message-header-parser.h \ message-id.h \ message-parser.h \ + message-part.h \ message-part-serialize.h \ message-search.h \ message-size.h \ diff -r 917946c08f83 -r 63f12bb366b0 src/lib-mail/message-parser.h --- a/src/lib-mail/message-parser.h Thu Apr 17 14:24:18 2014 +0200 +++ b/src/lib-mail/message-parser.h Thu Apr 17 16:04:13 2014 +0200 @@ -2,7 +2,7 @@ #define MESSAGE_PARSER_H #include "message-header-parser.h" -#include "message-size.h" +#include "message-part.h" enum message_parser_flags { /* Don't return message bodies in message_blocks. */ @@ -17,38 +17,6 @@ MESSAGE_PARSER_FLAG_INCLUDE_BOUNDARIES = 0x08 }; -/* Note that these flags are used directly by message-parser-serialize, so - existing flags can't be changed without breaking backwards compatibility */ -enum message_part_flags { - MESSAGE_PART_FLAG_MULTIPART = 0x01, - MESSAGE_PART_FLAG_MULTIPART_DIGEST = 0x02, - MESSAGE_PART_FLAG_MESSAGE_RFC822 = 0x04, - - /* content-type: text/... */ - MESSAGE_PART_FLAG_TEXT = 0x08, - - MESSAGE_PART_FLAG_UNUSED = 0x10, - - /* message part header or body contains NULs */ - MESSAGE_PART_FLAG_HAS_NULS = 0x20, - - /* Mime-Version header exists. */ - MESSAGE_PART_FLAG_IS_MIME = 0x40 -}; - -struct message_part { - struct message_part *parent; - struct message_part *next; - struct message_part *children; - - uoff_t physical_pos; /* absolute position from beginning of message */ - struct message_size header_size; - struct message_size body_size; - - enum message_part_flags flags; - void *context; -}; - struct message_parser_ctx; struct message_block { diff -r 917946c08f83 -r 63f12bb366b0 src/lib-mail/message-part.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-mail/message-part.h Thu Apr 17 16:04:13 2014 +0200 @@ -0,0 +1,38 @@ +#ifndef MESSAGE_PART_H +#define MESSAGE_PART_H + +#include "message-size.h" + +/* Note that these flags are used directly by message-parser-serialize, so + existing flags can't be changed without breaking backwards compatibility */ +enum message_part_flags { + MESSAGE_PART_FLAG_MULTIPART = 0x01, + MESSAGE_PART_FLAG_MULTIPART_DIGEST = 0x02, + MESSAGE_PART_FLAG_MESSAGE_RFC822 = 0x04, + + /* content-type: text/... */ + MESSAGE_PART_FLAG_TEXT = 0x08, + + MESSAGE_PART_FLAG_UNUSED = 0x10, + + /* message part header or body contains NULs */ + MESSAGE_PART_FLAG_HAS_NULS = 0x20, + + /* Mime-Version header exists. */ + MESSAGE_PART_FLAG_IS_MIME = 0x40 +}; + +struct message_part { + struct message_part *parent; + struct message_part *next; + struct message_part *children; + + uoff_t physical_pos; /* absolute position from beginning of message */ + struct message_size header_size; + struct message_size body_size; + + enum message_part_flags flags; + void *context; +}; + +#endif From dovecot at dovecot.org Thu Apr 17 14:13:34 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Apr 2014 14:13:34 +0000 Subject: dovecot-2.2: lib-mail: Added support for struct message_part <->... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e5635fc0710f changeset: 17238:e5635fc0710f user: Timo Sirainen date: Thu Apr 17 16:09:52 2014 +0200 description: lib-mail: Added support for struct message_part <-> index number translations. diffstat: src/lib-mail/Makefile.am | 6 ++ src/lib-mail/message-parser.c | 4 +- src/lib-mail/message-part-serialize.c | 4 +- src/lib-mail/message-part.c | 40 +++++++++++++++ src/lib-mail/message-part.h | 11 ++++ src/lib-mail/test-message-part.c | 93 +++++++++++++++++++++++++++++++++++ 6 files changed, 156 insertions(+), 2 deletions(-) diffs (232 lines): diff -r 63f12bb366b0 -r e5635fc0710f src/lib-mail/Makefile.am --- a/src/lib-mail/Makefile.am Thu Apr 17 16:04:13 2014 +0200 +++ b/src/lib-mail/Makefile.am Thu Apr 17 16:09:52 2014 +0200 @@ -24,6 +24,7 @@ message-header-parser.c \ message-id.c \ message-parser.c \ + message-part.c \ message-part-serialize.c \ message-search.c \ message-size.c \ @@ -77,6 +78,7 @@ test-message-header-parser \ test-message-id \ test-message-parser \ + test-message-part \ test-quoted-printable \ test-rfc2231-parser @@ -151,6 +153,10 @@ test_message_parser_LDADD = message-parser.lo message-header-parser.lo message-size.lo rfc822-parser.lo rfc2231-parser.lo $(test_libs) test_message_parser_DEPENDENCIES = $(test_deps) +test_message_part_SOURCES = test-message-part.c +test_message_part_LDADD = message-part.lo message-parser.lo message-header-parser.lo message-size.lo rfc822-parser.lo rfc2231-parser.lo $(test_libs) +test_message_part_DEPENDENCIES = $(test_deps) + test_quoted_printable_SOURCES = test-quoted-printable.c test_quoted_printable_LDADD = quoted-printable.lo $(test_libs) test_quoted_printable_DEPENDENCIES = $(test_deps) diff -r 63f12bb366b0 -r e5635fc0710f src/lib-mail/message-parser.c --- a/src/lib-mail/message-parser.c Thu Apr 17 16:04:13 2014 +0200 +++ b/src/lib-mail/message-parser.c Thu Apr 17 16:09:52 2014 +0200 @@ -163,10 +163,12 @@ static struct message_part * message_part_append(pool_t pool, struct message_part *parent) { - struct message_part *part, **list; + struct message_part *p, *part, **list; part = p_new(pool, struct message_part, 1); part->parent = parent; + for (p = parent; p != NULL; p = p->parent) + p->children_count++; /* set child position */ part->physical_pos = diff -r 63f12bb366b0 -r e5635fc0710f src/lib-mail/message-part-serialize.c --- a/src/lib-mail/message-part-serialize.c Thu Apr 17 16:04:13 2014 +0200 +++ b/src/lib-mail/message-part-serialize.c Thu Apr 17 16:09:52 2014 +0200 @@ -123,7 +123,7 @@ unsigned int siblings, struct message_part **part_r) { - struct message_part *part, *first_part, **next_part; + struct message_part *p, *part, *first_part, **next_part; unsigned int children_count; uoff_t pos; bool root = parent == NULL; @@ -135,6 +135,8 @@ part = p_new(ctx->pool, struct message_part, 1); part->parent = parent; + for (p = parent; p != NULL; p = p->parent) + p->children_count++; if (!read_next(ctx, &part->flags, sizeof(part->flags))) return FALSE; diff -r 63f12bb366b0 -r e5635fc0710f src/lib-mail/message-part.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-mail/message-part.c Thu Apr 17 16:09:52 2014 +0200 @@ -0,0 +1,40 @@ +/* Copyright (c) 2014 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "message-part.h" + +unsigned int message_part_to_idx(const struct message_part *part) +{ + const struct message_part *p; + unsigned int n; + + if (part->parent == NULL) { + /* root */ + return 0; + } + for (n = 0, p = part->parent->children; p != part; p = p->next, n++) + ; + return n + 1 + message_part_to_idx(part->parent); +} + +static struct message_part * +message_sub_part_by_idx(struct message_part *parts, + unsigned int idx) +{ + struct message_part *part = parts; + + for (; part != NULL && idx > 0; part = part->next) { + if (part->children_count >= idx) + return message_sub_part_by_idx(part->children, idx-1); + idx -= part->children_count + 1; + } + return part; +} + +struct message_part * +message_part_by_idx(struct message_part *parts, unsigned int idx); +{ + i_assert(parts->parent == NULL); + + return message_sub_part_by_idx(parts, idx); +} diff -r 63f12bb366b0 -r e5635fc0710f src/lib-mail/message-part.h --- a/src/lib-mail/message-part.h Thu Apr 17 16:04:13 2014 +0200 +++ b/src/lib-mail/message-part.h Thu Apr 17 16:09:52 2014 +0200 @@ -31,8 +31,19 @@ struct message_size header_size; struct message_size body_size; + /* total number of message_parts under children */ + unsigned int children_count; enum message_part_flags flags; void *context; }; +/* Return index number for the message part. The indexes are in the same order + as they exist in the flat RFC822 message. The root part is 0, its first + child is 1 and so on. */ +unsigned int message_part_to_idx(const struct message_part *part); +/* Find message part by its index number, or return NULL if the index + doesn't exist. */ +struct message_part * +message_part_by_idx(struct message_part *parts, unsigned int idx); + #endif diff -r 63f12bb366b0 -r e5635fc0710f src/lib-mail/test-message-part.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-mail/test-message-part.c Thu Apr 17 16:09:52 2014 +0200 @@ -0,0 +1,93 @@ +/* Copyright (c) 2014 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "message-parser.h" +#include "test-common.h" + +static const char test_msg[] = +"From user at domain Fri Feb 22 17:06:23 2008\n" +"From: user at domain.org\n" +"Date: Sat, 24 Mar 2007 23:00:00 +0200\n" +"Mime-Version: 1.0\n" +"Content-Type: multipart/mixed; boundary=\"foo bar\"\n" +"\n" +"Root MIME prologue\n" +"\n" +"--foo bar\n" +"Content-Type: text/x-myown; charset=us-ascii\n" +"\n" +"hello\n" +"\n" +"--foo bar\n" +"Content-Type: message/rfc822\n" +"\n" +"From: sub at domain.org\n" +"Date: Sun, 12 Aug 2012 12:34:56 +0300\n" +"Subject: submsg\n" +"Content-Type: multipart/alternative; boundary=\"sub1\"\n" +"\n" +"Sub MIME prologue\n" +"--sub1\n" +"Content-Type: text/html\n" +"\n" +"

Hello world

\n" +"\n" +"--sub1\n" +"Content-Type: text/plain\n" +"\n" +"Hello another world\n" +"\n" +"--sub1--\n" +"Sub MIME epilogue\n" +"\n" +"--foo bar--\n" +"Root MIME epilogue\n" +"\n"; +#define TEST_MSG_LEN (sizeof(test_msg)-1) + +static void test_message_part_idx(void) +{ + struct message_parser_ctx *parser; + struct istream *input; + struct message_part *parts, *part, *prev_part; + struct message_block block; + unsigned int i; + pool_t pool; + int ret; + + test_begin("message part indexes"); + pool = pool_alloconly_create("message parser", 10240); + input = i_stream_create_from_data(test_msg, TEST_MSG_LEN); + + parser = message_parser_init(pool, input, 0, 0); + while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ; + test_assert(ret < 0); + test_assert(message_parser_deinit(&parser, &parts) == 0); + + part = message_part_by_idx(parts, 0); + test_assert(part == parts); + test_assert(message_part_by_idx(parts, 1) == parts->children); + + for (i = 1; i < 6; i++) { + prev_part = part; + part = message_part_by_idx(parts, i); + test_assert(part != NULL); + test_assert(message_part_to_idx(part) == i); + test_assert(prev_part->physical_pos < part->physical_pos); + } + test_assert(message_part_by_idx(parts, i) == NULL); + + i_stream_unref(&input); + pool_unref(&pool); + test_end(); +} + +int main(void) +{ + static void (*test_functions[])(void) = { + test_message_part_idx, + NULL + }; + return test_run(test_functions); +} From dovecot at dovecot.org Thu Apr 17 14:13:34 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Apr 2014 14:13:34 +0000 Subject: dovecot-2.2: fts: Added message_part to fts_backend_build_key. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1b02ece5298c changeset: 17239:1b02ece5298c user: Timo Sirainen date: Thu Apr 17 16:12:00 2014 +0200 description: fts: Added message_part to fts_backend_build_key. This allows FTS backends to index MIME parts separately from each others. diffstat: src/plugins/fts/fts-api.h | 1 + src/plugins/fts/fts-build-mail.c | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diffs (50 lines): diff -r e5635fc0710f -r 1b02ece5298c src/plugins/fts/fts-api.h --- a/src/plugins/fts/fts-api.h Thu Apr 17 16:09:52 2014 +0200 +++ b/src/plugins/fts/fts-api.h Thu Apr 17 16:12:00 2014 +0200 @@ -24,6 +24,7 @@ struct fts_backend_build_key { uint32_t uid; enum fts_backend_build_key_type type; + struct message_part *part; /* for _KEY_HDR: */ const char *hdr_name; diff -r e5635fc0710f -r 1b02ece5298c src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Thu Apr 17 16:09:52 2014 +0200 +++ b/src/plugins/fts/fts-build-mail.c Thu Apr 17 16:12:00 2014 +0200 @@ -111,6 +111,7 @@ key.uid = ctx->mail->uid; key.type = block->part->physical_pos == 0 ? FTS_BACKEND_BUILD_KEY_HDR : FTS_BACKEND_BUILD_KEY_MIME_HDR; + key.part = block->part; key.hdr_name = hdr->name; if (!fts_backend_update_set_build_key(ctx->update_ctx, &key)) @@ -139,7 +140,8 @@ } static bool -fts_build_body_begin(struct fts_mail_build_context *ctx, bool *binary_body_r) +fts_build_body_begin(struct fts_mail_build_context *ctx, + struct message_part *part, bool *binary_body_r) { struct mail_storage *storage; const char *content_type; @@ -150,6 +152,7 @@ *binary_body_r = FALSE; memset(&key, 0, sizeof(key)); key.uid = ctx->mail->uid; + key.part = part; content_type = ctx->content_type != NULL ? ctx->content_type : "text/plain"; @@ -323,7 +326,8 @@ /* always handle headers */ } else if (raw_block.size == 0) { /* end of headers */ - skip_body = !fts_build_body_begin(&ctx, &binary_body); + skip_body = !fts_build_body_begin(&ctx, raw_block.part, + &binary_body); if (binary_body) message_decoder_set_return_binary(decoder, TRUE); body_part = TRUE; From dovecot at dovecot.org Thu Apr 17 14:18:23 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Apr 2014 14:18:23 +0000 Subject: dovecot-2.2: lib-mail: Compile fix for last change.. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7c12201decaf changeset: 17240:7c12201decaf user: Timo Sirainen date: Thu Apr 17 16:18:02 2014 +0200 description: lib-mail: Compile fix for last change.. diffstat: src/lib-mail/message-part.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 1b02ece5298c -r 7c12201decaf src/lib-mail/message-part.c --- a/src/lib-mail/message-part.c Thu Apr 17 16:12:00 2014 +0200 +++ b/src/lib-mail/message-part.c Thu Apr 17 16:18:02 2014 +0200 @@ -32,7 +32,7 @@ } struct message_part * -message_part_by_idx(struct message_part *parts, unsigned int idx); +message_part_by_idx(struct message_part *parts, unsigned int idx) { i_assert(parts->parent == NULL); From dovecot at dovecot.org Thu Apr 17 14:35:37 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Apr 2014 14:35:37 +0000 Subject: dovecot-2.2: fts-lucene: Fixed "doveadm dump" crash when built w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7809bc519633 changeset: 17241:7809bc519633 user: Timo Sirainen date: Thu Apr 17 16:35:05 2014 +0200 description: fts-lucene: Fixed "doveadm dump" crash when built with stemmer diffstat: src/plugins/fts-lucene/lucene-wrapper.cc | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 7c12201decaf -r 7809bc519633 src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Thu Apr 17 16:18:02 2014 +0200 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Thu Apr 17 16:35:05 2014 +0200 @@ -119,7 +119,7 @@ index->set.default_language = ""; } #ifdef HAVE_LUCENE_STEMMER - if (!set->no_snowball) { + if (set == NULL || !set->no_snowball) { index->default_analyzer = _CLNEW snowball::SnowballAnalyzer(index->normalizer, index->set.default_language); From dovecot at dovecot.org Thu Apr 17 14:42:26 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Apr 2014 14:42:26 +0000 Subject: dovecot-2.2: fts-lucene: Added mime_parts parameter to index MIM... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cba90b79fa48 changeset: 17242:cba90b79fa48 user: Timo Sirainen date: Thu Apr 17 16:42:02 2014 +0200 description: fts-lucene: Added mime_parts parameter to index MIME parts as separate documents. This is in preparation for being able to actually read such information and show it to clients via some new IMAP extension. Also it might be better to index some MIME parts together to avoid wasting disk space, but for now this is all or nothing. For example anything that is "message body" could probably be indexed without the part number at all. diffstat: src/plugins/fts-lucene/doveadm-fts-lucene.c | 5 +++- src/plugins/fts-lucene/fts-backend-lucene.c | 13 ++++++++++- src/plugins/fts-lucene/fts-lucene-plugin.c | 4 +++ src/plugins/fts-lucene/fts-lucene-plugin.h | 1 + src/plugins/fts-lucene/lucene-wrapper.cc | 31 +++++++++++++++++++++++++--- src/plugins/fts-lucene/lucene-wrapper.h | 6 ++-- 6 files changed, 50 insertions(+), 10 deletions(-) diffs (220 lines): diff -r 7809bc519633 -r cba90b79fa48 src/plugins/fts-lucene/doveadm-fts-lucene.c --- a/src/plugins/fts-lucene/doveadm-fts-lucene.c Thu Apr 17 16:35:05 2014 +0200 +++ b/src/plugins/fts-lucene/doveadm-fts-lucene.c Thu Apr 17 16:42:02 2014 +0200 @@ -34,7 +34,10 @@ memcpy(prev_guid, rec->mailbox_guid, sizeof(prev_guid)); printf("%s: ", guid_128_to_string(prev_guid)); } - printf("%u,", rec->uid); + printf("%u", rec->uid); + if (rec->part_num != 0) + printf("[%u]", rec->part_num); + printf("\n"); } printf("\n"); if (lucene_index_iter_deinit(&iter) < 0) diff -r 7809bc519633 -r cba90b79fa48 src/plugins/fts-lucene/fts-backend-lucene.c --- a/src/plugins/fts-lucene/fts-backend-lucene.c Thu Apr 17 16:35:05 2014 +0200 +++ b/src/plugins/fts-lucene/fts-backend-lucene.c Thu Apr 17 16:42:02 2014 +0200 @@ -5,6 +5,7 @@ #include "hash.h" #include "hex-binary.h" #include "strescape.h" +#include "message-part.h" #include "mail-namespace.h" #include "mail-storage-private.h" #include "fts-expunge-log.h" @@ -41,7 +42,7 @@ uint32_t last_indexed_uid; char *first_box_vname; - uint32_t uid; + uint32_t uid, part_num; char *hdr_name; unsigned int added_msgs; @@ -49,6 +50,7 @@ bool lucene_opened; bool last_indexed_uid_set; + bool mime_parts; }; static int fts_backend_lucene_mkdir(struct lucene_fts_backend *backend) @@ -203,11 +205,14 @@ struct lucene_fts_backend *backend = (struct lucene_fts_backend *)_backend; struct lucene_fts_backend_update_context *ctx; + struct fts_lucene_user *fuser = + FTS_LUCENE_USER_CONTEXT(_backend->ns->user); i_assert(!backend->updating); ctx = i_new(struct lucene_fts_backend_update_context, 1); ctx->ctx.backend = _backend; + ctx->mime_parts = fuser->set.mime_parts; backend->updating = TRUE; return &ctx->ctx; } @@ -375,6 +380,8 @@ } ctx->uid = key->uid; + if (ctx->mime_parts) + ctx->part_num = message_part_to_idx(key->part); return TRUE; } @@ -385,6 +392,7 @@ (struct lucene_fts_backend_update_context *)_ctx; ctx->uid = 0; + ctx->part_num = 0; i_free_and_null(ctx->hdr_name); } @@ -405,7 +413,8 @@ T_BEGIN { ret = lucene_index_build_more(backend->index, ctx->uid, - data, size, ctx->hdr_name); + ctx->part_num, data, size, + ctx->hdr_name); } T_END; return ret; } diff -r 7809bc519633 -r cba90b79fa48 src/plugins/fts-lucene/fts-lucene-plugin.c --- a/src/plugins/fts-lucene/fts-lucene-plugin.c Thu Apr 17 16:35:05 2014 +0200 +++ b/src/plugins/fts-lucene/fts-lucene-plugin.c Thu Apr 17 16:42:02 2014 +0200 @@ -32,6 +32,8 @@ set->normalize = TRUE; } else if (strcmp(*tmp, "no_snowball") == 0) { set->no_snowball = TRUE; + } else if (strcmp(*tmp, "mime_parts") == 0) { + set->mime_parts = TRUE; } else { i_error("fts_lucene: Invalid setting: %s", *tmp); return -1; @@ -79,6 +81,8 @@ crc = crc32_str_more(crc, "n"); if (set->no_snowball) crc = crc32_str_more(crc, "s"); + /* don't include mime_parts here, since changing it doesn't + necessarily need the index to be rebuilt */ return crc; } diff -r 7809bc519633 -r cba90b79fa48 src/plugins/fts-lucene/fts-lucene-plugin.h --- a/src/plugins/fts-lucene/fts-lucene-plugin.h Thu Apr 17 16:35:05 2014 +0200 +++ b/src/plugins/fts-lucene/fts-lucene-plugin.h Thu Apr 17 16:42:02 2014 +0200 @@ -14,6 +14,7 @@ const char *whitespace_chars; bool normalize; bool no_snowball; + bool mime_parts; }; struct fts_lucene_user { diff -r 7809bc519633 -r cba90b79fa48 src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Thu Apr 17 16:35:05 2014 +0200 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Thu Apr 17 16:42:02 2014 +0200 @@ -72,7 +72,7 @@ ARRAY(struct lucene_analyzer) analyzers; Document *doc; - uint32_t prev_uid; + uint32_t prev_uid, prev_part_idx; }; struct rescan_context { @@ -307,6 +307,22 @@ return 0; } +static uint32_t +lucene_doc_get_part(struct lucene_index *index, Document *doc) +{ + Field *field = doc->getField(_T("part")); + const TCHAR *part = field == NULL ? NULL : field->stringValue(); + if (part == NULL) + return 0; + + uint32_t num = 0; + while (*part != 0) { + num = num*10 + (*part - '0'); + part++; + } + return num; +} + int lucene_index_get_last_uid(struct lucene_index *index, uint32_t *last_uid_r) { int ret = 0; @@ -510,20 +526,25 @@ } int lucene_index_build_more(struct lucene_index *index, uint32_t uid, - const unsigned char *data, size_t size, - const char *hdr_name) + uint32_t part_idx, const unsigned char *data, + size_t size, const char *hdr_name) { wchar_t id[MAX_INT_STRLEN]; size_t namesize, datasize; - if (uid != index->prev_uid) { + if (uid != index->prev_uid || part_idx != index->prev_part_idx) { if (lucene_index_build_flush(index) < 0) return -1; index->prev_uid = uid; + index->prev_part_idx = part_idx; index->doc = _CLNEW Document(); swprintf(id, N_ELEMENTS(id), L"%u", uid); index->doc->add(*_CLNEW Field(_T("uid"), id, Field::STORE_YES | Field::INDEX_UNTOKENIZED)); + if (part_idx != 0) { + swprintf(id, N_ELEMENTS(id), L"%u", part_idx); + index->doc->add(*_CLNEW Field(_T("part"), id, Field::STORE_YES | Field::INDEX_UNTOKENIZED)); + } index->doc->add(*_CLNEW Field(_T("box"), index->mailbox_guid, Field::STORE_YES | Field::INDEX_UNTOKENIZED)); } @@ -573,6 +594,7 @@ return 0; } index->prev_uid = 0; + index->prev_part_idx = 0; if (index->writer == NULL) { lucene_index_close(index); @@ -1497,6 +1519,7 @@ (void)fts_lucene_get_mailbox_guid(iter->index, doc, iter->rec.mailbox_guid); (void)lucene_doc_get_uid(iter->index, doc, &iter->rec.uid); + iter->rec.part_num = lucene_doc_get_part(iter->index, doc); return &iter->rec; } diff -r 7809bc519633 -r cba90b79fa48 src/plugins/fts-lucene/lucene-wrapper.h --- a/src/plugins/fts-lucene/lucene-wrapper.h Thu Apr 17 16:35:05 2014 +0200 +++ b/src/plugins/fts-lucene/lucene-wrapper.h Thu Apr 17 16:42:02 2014 +0200 @@ -12,7 +12,7 @@ struct lucene_index_record { guid_128_t mailbox_guid; - uint32_t uid; + uint32_t uid, part_num; }; HASH_TABLE_DEFINE_TYPE(wguid_result, wchar_t *, struct fts_result *); @@ -31,8 +31,8 @@ int lucene_index_build_init(struct lucene_index *index); int lucene_index_build_more(struct lucene_index *index, uint32_t uid, - const unsigned char *data, size_t size, - const char *hdr_name); + uint32_t part_num, const unsigned char *data, + size_t size, const char *hdr_name); int lucene_index_build_deinit(struct lucene_index *index); void lucene_index_close(struct lucene_index *index); From dovecot at dovecot.org Sat Apr 19 09:07:59 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 19 Apr 2014 09:07:59 +0000 Subject: dovecot-2.2: ioloop: Fixed segfault crash in io_loop_call_pendin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7540a5cdbdf1 changeset: 17243:7540a5cdbdf1 user: Timo Sirainen date: Sat Apr 19 11:07:45 2014 +0200 description: ioloop: Fixed segfault crash in io_loop_call_pending(). When a callback called io_set_pending(), the io_loop_call_pending() loop crossed the end of the ioloop->io_files list. Based on patch by Stephan Bosch diffstat: src/lib/ioloop.c | 15 ++++++++++----- 1 files changed, 10 insertions(+), 5 deletions(-) diffs (25 lines): diff -r cba90b79fa48 -r 7540a5cdbdf1 src/lib/ioloop.c --- a/src/lib/ioloop.c Thu Apr 17 16:42:02 2014 +0200 +++ b/src/lib/ioloop.c Sat Apr 19 11:07:45 2014 +0200 @@ -470,11 +470,16 @@ { struct io_file *io; - for (io = ioloop->io_files; ioloop->io_pending_count > 0; ) { - ioloop->next_io_file = io->next; - if (io->io.pending) - io_loop_call_io(&io->io); - io = ioloop->next_io_file; + while (ioloop->io_pending_count > 0) { + io = ioloop->io_files; + do { + ioloop->next_io_file = io->next; + if (io->io.pending) + io_loop_call_io(&io->io); + if (ioloop->io_pending_count == 0) + break; + io = ioloop->next_io_file; + } while (io != NULL); } } From dovecot at dovecot.org Sat Apr 19 09:28:40 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 19 Apr 2014 09:28:40 +0000 Subject: dovecot-2.2: connection: Added support for creating a stream fro... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/54178ae78835 changeset: 17246:54178ae78835 user: Stephan Bosch date: Sat Apr 19 11:23:18 2014 +0200 description: connection: Added support for creating a stream from input/output stream pair. diffstat: src/lib/connection.c | 36 ++++++++++++++++++++++++++++++++++++ src/lib/connection.h | 3 +++ 2 files changed, 39 insertions(+), 0 deletions(-) diffs (59 lines): diff -r 6c3bd941e153 -r 54178ae78835 src/lib/connection.c --- a/src/lib/connection.c Sat Apr 19 11:21:47 2014 +0200 +++ b/src/lib/connection.c Sat Apr 19 11:23:18 2014 +0200 @@ -198,6 +198,42 @@ list->connections_count++; } +void connection_init_from_streams(struct connection_list *list, + struct connection *conn, const char *name, + struct istream *input, struct ostream *output) +{ + i_assert(name != NULL); + + conn->list = list; + conn->name = i_strdup(name); + conn->fd_in = i_stream_get_fd(input); + conn->fd_out = o_stream_get_fd(output); + + i_assert(conn->fd_in >= 0); + i_assert(conn->fd_out >= 0); + i_assert(conn->io == NULL); + i_assert(conn->input == NULL); + i_assert(conn->output == NULL); + i_assert(conn->to == NULL); + + conn->input = input; + i_stream_ref(conn->input); + i_stream_set_name(conn->input, conn->name); + + conn->output = output; + o_stream_ref(conn->output); + o_stream_set_no_error_handling(conn->output, TRUE); + o_stream_set_name(conn->output, conn->name); + + conn->io = io_add(conn->fd_in, IO_READ, *list->v.input, conn); + + DLLIST_PREPEND(&list->connections, conn); + list->connections_count++; + + if (list->v.client_connected != NULL) + list->v.client_connected(conn, TRUE); +} + static void connection_ip_connected(struct connection *conn) { io_remove(&conn->io); diff -r 6c3bd941e153 -r 54178ae78835 src/lib/connection.h --- a/src/lib/connection.h Sat Apr 19 11:21:47 2014 +0200 +++ b/src/lib/connection.h Sat Apr 19 11:23:18 2014 +0200 @@ -100,6 +100,9 @@ const struct ip_addr *ip, unsigned int port); void connection_init_client_unix(struct connection_list *list, struct connection *conn, const char *path); +void connection_init_from_streams(struct connection_list *list, + struct connection *conn, const char *name, + struct istream *input, struct ostream *output); int connection_client_connect(struct connection *conn); From dovecot at dovecot.org Tue Apr 22 10:17:04 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Apr 2014 10:17:04 +0000 Subject: dovecot-2.2: safe_memset() wasn't safe with modern compilers any... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/09c114091c88 changeset: 17247:09c114091c88 user: Timo Sirainen date: Tue Apr 22 13:15:21 2014 +0300 description: safe_memset() wasn't safe with modern compilers anymore. Another try based on David Jacobson's code in https://www.mail-archive.com/openssl-dev at openssl.org/msg34134.html : """ Since vs points to a volatile, the load in the while clause actually has to be done. That forces the compiler to actually store c into at least the byte that is tested, in practice byte zero. But the fact that the index is volatile zero, and since it is volatile it could spontaneously change to anything, the compiler has to store c into all bytes. The key observation is that while you can't pass a volatile to memset (you get a warning and the volatile gets stripped away), you can use a volatile in a test that could go the wrong way if the memset were elided. """ diffstat: src/lib/safe-memset.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 54178ae78835 -r 09c114091c88 src/lib/safe-memset.c --- a/src/lib/safe-memset.c Sat Apr 19 11:23:18 2014 +0200 +++ b/src/lib/safe-memset.c Tue Apr 22 13:15:21 2014 +0300 @@ -5,8 +5,10 @@ void safe_memset(void *data, int c, size_t size) { + volatile unsigned int volatile_zero_idx = 0; volatile unsigned char *p = data; - for (; size > 0; size--) - *p++ = (unsigned char)c; + do { + memset(data, c, size); + } while (p[volatile_zero_idx] != c); } From dovecot at dovecot.org Wed Apr 23 13:32:04 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Apr 2014 13:32:04 +0000 Subject: dovecot-2.2: virtual: Delay initializing search args until the m... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0aa159d6ef30 changeset: 17248:0aa159d6ef30 user: Timo Sirainen date: Wed Apr 23 16:31:36 2014 +0300 description: virtual: Delay initializing search args until the mailbox is actually opened. Fixes assert-crash caused by recent changes when virtual mailbox search args contained e.g. keywords. diffstat: src/plugins/virtual/virtual-storage.c | 5 ++--- src/plugins/virtual/virtual-storage.h | 1 + src/plugins/virtual/virtual-sync.c | 8 ++++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diffs (65 lines): diff -r 09c114091c88 -r 0aa159d6ef30 src/plugins/virtual/virtual-storage.c --- a/src/plugins/virtual/virtual-storage.c Tue Apr 22 13:15:21 2014 +0300 +++ b/src/plugins/virtual/virtual-storage.c Wed Apr 23 16:31:36 2014 +0300 @@ -178,7 +178,6 @@ i_array_init(&bbox->uids, 64); i_array_init(&bbox->sync_pending_removes, 64); - mail_search_args_init(bbox->search_args, bbox->box, FALSE, NULL); return 1; } @@ -195,7 +194,6 @@ if (ret <= 0) { if (ret < 0) break; - mail_search_args_unref(&bboxes[i]->search_args); array_delete(&mbox->backend_boxes, i, 1); bboxes = array_get(&mbox->backend_boxes, &count); } else { @@ -251,7 +249,8 @@ if (bboxes[i]->box == NULL) continue; - mail_search_args_deinit(bboxes[i]->search_args); + if (bboxes[i]->search_args != NULL) + mail_search_args_deinit(bboxes[i]->search_args); mailbox_free(&bboxes[i]->box); if (array_is_created(&bboxes[i]->sync_outside_expunges)) array_free(&bboxes[i]->sync_outside_expunges); diff -r 09c114091c88 -r 0aa159d6ef30 src/plugins/virtual/virtual-storage.h --- a/src/plugins/virtual/virtual-storage.h Tue Apr 22 13:15:21 2014 +0300 +++ b/src/plugins/virtual/virtual-storage.h Wed Apr 23 16:31:36 2014 +0300 @@ -96,6 +96,7 @@ unsigned int wildcard:1; unsigned int clear_recent:1; unsigned int uids_nonsorted:1; + unsigned int search_args_initialized:1; }; ARRAY_DEFINE_TYPE(virtual_backend_box, struct virtual_backend_box *); diff -r 09c114091c88 -r 0aa159d6ef30 src/plugins/virtual/virtual-sync.c --- a/src/plugins/virtual/virtual-sync.c Tue Apr 22 13:15:21 2014 +0300 +++ b/src/plugins/virtual/virtual-sync.c Wed Apr 23 16:31:36 2014 +0300 @@ -460,6 +460,10 @@ trans = mailbox_transaction_begin(bbox->box, 0); + if (!bbox->search_args_initialized) { + mail_search_args_init(bbox->search_args, bbox->box, FALSE, NULL); + bbox->search_args_initialized = TRUE; + } search_ctx = mailbox_search_init(trans, bbox->search_args, NULL, 0, NULL); @@ -677,6 +681,10 @@ /* initialize the search result from all the existing messages in virtual index. */ + if (!bbox->search_args_initialized) { + mail_search_args_init(bbox->search_args, bbox->box, FALSE, NULL); + bbox->search_args_initialized = TRUE; + } result = mailbox_search_result_alloc(bbox->box, bbox->search_args, result_flags); mailbox_search_result_initial_done(result); From dovecot at dovecot.org Wed Apr 23 13:45:41 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Apr 2014 13:45:41 +0000 Subject: dovecot-2.2: virtual: Fixed assert-crash in IDLE caused by earli... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/79bfc730a8b0 changeset: 17249:79bfc730a8b0 user: Timo Sirainen date: Wed Apr 23 16:45:10 2014 +0300 description: virtual: Fixed assert-crash in IDLE caused by earlier changes. This change negates all the improvements of the earlier changes when IDLE is used. This could be fixed by using mailbox-list-notify.h API. diffstat: src/plugins/virtual/virtual-storage.c | 26 +++++++++++++++++--------- src/plugins/virtual/virtual-storage.h | 1 + src/plugins/virtual/virtual-sync.c | 7 +++++-- 3 files changed, 23 insertions(+), 11 deletions(-) diffs (68 lines): diff -r 0aa159d6ef30 -r 79bfc730a8b0 src/plugins/virtual/virtual-storage.c --- a/src/plugins/virtual/virtual-storage.c Wed Apr 23 16:31:36 2014 +0300 +++ b/src/plugins/virtual/virtual-storage.c Wed Apr 23 16:45:10 2014 +0300 @@ -417,17 +417,25 @@ static void virtual_notify_changes(struct mailbox *box) { struct virtual_mailbox *mbox = (struct virtual_mailbox *)box; - struct virtual_backend_box *const *bboxes; - unsigned int i, count; + struct virtual_backend_box *const *bboxp; - bboxes = array_get(&mbox->backend_boxes, &count); - for (i = 0; i < count; i++) { - struct mailbox *bbox = bboxes[i]->box; + if (box->notify_callback == NULL) { + array_foreach(&mbox->backend_boxes, bboxp) + mailbox_notify_changes_stop((*bboxp)->box); + return; + } - if (box->notify_callback == NULL) - mailbox_notify_changes_stop(bbox); - else - mailbox_notify_changes(bbox, virtual_notify_callback, box); + /* FIXME: if mailbox_list_index=yes, use mailbox-list-notify.h API + to wait for changes and avoid opening all mailboxes here. */ + + array_foreach(&mbox->backend_boxes, bboxp) { + if (mailbox_open((*bboxp)->box) < 0) { + /* we can't report error in here, so do it later */ + (*bboxp)->open_failed = TRUE; + continue; + } + mailbox_notify_changes((*bboxp)->box, + virtual_notify_callback, box); } } diff -r 0aa159d6ef30 -r 79bfc730a8b0 src/plugins/virtual/virtual-storage.h --- a/src/plugins/virtual/virtual-storage.h Wed Apr 23 16:31:36 2014 +0300 +++ b/src/plugins/virtual/virtual-storage.h Wed Apr 23 16:45:10 2014 +0300 @@ -92,6 +92,7 @@ struct imap_match_glob *glob; struct mail_namespace *ns; + unsigned int open_failed:1; unsigned int sync_seen:1; unsigned int wildcard:1; unsigned int clear_recent:1; diff -r 0aa159d6ef30 -r 79bfc730a8b0 src/plugins/virtual/virtual-sync.c --- a/src/plugins/virtual/virtual-sync.c Wed Apr 23 16:31:36 2014 +0300 +++ b/src/plugins/virtual/virtual-sync.c Wed Apr 23 16:45:10 2014 +0300 @@ -1041,10 +1041,13 @@ mailbox. */ i_assert(array_count(&bbox->sync_pending_removes) == 0); - if (bbox_index_opened) { - /* index already opened, refresh it */ + if (bbox_index_opened || bbox->open_failed) { + /* a) index already opened, refresh it + b) delayed error handling for mailbox_open() + that failed in virtual_notify_changes() */ if (mailbox_sync(bbox->box, sync_flags) < 0) return -1; + bbox->open_failed = FALSE; } if (mailbox_get_status(bbox->box, STATUS_UIDVALIDITY | From dovecot at dovecot.org Wed Apr 23 16:19:49 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Apr 2014 16:19:49 +0000 Subject: dovecot-2.2: Added o_stream_get_max_buffer_size() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6abe16ff8108 changeset: 17250:6abe16ff8108 user: Timo Sirainen date: Wed Apr 23 19:19:24 2014 +0300 description: Added o_stream_get_max_buffer_size() diffstat: src/lib/ostream.c | 5 +++++ src/lib/ostream.h | 2 ++ 2 files changed, 7 insertions(+), 0 deletions(-) diffs (27 lines): diff -r 79bfc730a8b0 -r 6abe16ff8108 src/lib/ostream.c --- a/src/lib/ostream.c Wed Apr 23 16:45:10 2014 +0300 +++ b/src/lib/ostream.c Wed Apr 23 19:19:24 2014 +0300 @@ -112,6 +112,11 @@ io_stream_set_max_buffer_size(&stream->real_stream->iostream, max_size); } +size_t o_stream_get_max_buffer_size(struct ostream *stream) +{ + return stream->real_stream->max_buffer_size; +} + void o_stream_cork(struct ostream *stream) { struct ostream_private *_stream = stream->real_stream; diff -r 79bfc730a8b0 -r 6abe16ff8108 src/lib/ostream.h --- a/src/lib/ostream.h Wed Apr 23 16:45:10 2014 +0300 +++ b/src/lib/ostream.h Wed Apr 23 19:19:24 2014 +0300 @@ -77,6 +77,8 @@ void o_stream_unset_flush_callback(struct ostream *stream); /* Change the maximum size for stream's output buffer to grow. */ void o_stream_set_max_buffer_size(struct ostream *stream, size_t max_size); +/* Returns the current max. buffer size. */ +size_t o_stream_get_max_buffer_size(struct ostream *stream); /* Delays sending as far as possible, writing only full buffers. Also sets TCP_CORK on if supported. */ From dovecot at dovecot.org Wed Apr 23 16:20:56 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Apr 2014 16:20:56 +0000 Subject: dovecot-2.2: dsync: When initializing via doveadm-server, change... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/98d3020ea144 changeset: 17251:98d3020ea144 user: Timo Sirainen date: Wed Apr 23 19:20:23 2014 +0300 description: dsync: When initializing via doveadm-server, change the iostream buffer sizes to unlimited. diffstat: src/doveadm/dsync/doveadm-dsync.c | 14 ++++++++++++-- 1 files changed, 12 insertions(+), 2 deletions(-) diffs (40 lines): diff -r 6abe16ff8108 -r 98d3020ea144 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Wed Apr 23 19:19:24 2014 +0300 +++ b/src/doveadm/dsync/doveadm-dsync.c Wed Apr 23 19:20:23 2014 +0300 @@ -69,6 +69,7 @@ struct io *io_err; struct istream *input, *err_stream; struct ostream *output; + size_t input_orig_bufsize, output_orig_bufsize; struct ssl_iostream_context *ssl_ctx; struct ssl_iostream *ssl_iostream; @@ -462,6 +463,11 @@ fd_set_nonblock(ctx->fd_out, TRUE); ctx->input = i_stream_create_fd(ctx->fd_in, (size_t)-1, FALSE); ctx->output = o_stream_create_fd(ctx->fd_out, (size_t)-1, FALSE); + } else { + ctx->input_orig_bufsize = i_stream_get_max_buffer_size(ctx->input); + ctx->output_orig_bufsize = o_stream_get_max_buffer_size(ctx->output); + i_stream_set_max_buffer_size(ctx->input, (size_t)-1); + o_stream_set_max_buffer_size(ctx->output, (size_t)-1); } if (ctx->rawlog_path != NULL) { iostream_rawlog_create_path(ctx->rawlog_path, @@ -615,10 +621,14 @@ ssl_iostream_destroy(&ctx->ssl_iostream); if (ctx->ssl_ctx != NULL) ssl_iostream_context_deinit(&ctx->ssl_ctx); - if (ctx->input != NULL) + if (ctx->input != NULL) { + i_stream_set_max_buffer_size(ctx->input, ctx->input_orig_bufsize); i_stream_unref(&ctx->input); - if (ctx->output != NULL) + } + if (ctx->output != NULL) { + o_stream_set_max_buffer_size(ctx->output, ctx->output_orig_bufsize); o_stream_unref(&ctx->output); + } if (ctx->fd_in != -1) { if (ctx->fd_out != ctx->fd_in) i_close_fd(&ctx->fd_out); From dovecot at dovecot.org Wed Apr 23 16:21:58 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Apr 2014 16:21:58 +0000 Subject: dovecot-2.2: dsync: Added assert Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e20a6192ddcf changeset: 17252:e20a6192ddcf user: Timo Sirainen date: Wed Apr 23 19:21:40 2014 +0300 description: dsync: Added assert diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (28 lines): diff -r 98d3020ea144 -r e20a6192ddcf src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Wed Apr 23 19:20:23 2014 +0300 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Wed Apr 23 19:21:40 2014 +0300 @@ -359,6 +359,7 @@ { string_t *error; const char *line; + ssize_t ret; line = i_stream_next_line(ibc->input); if (line != NULL) { @@ -366,7 +367,7 @@ return 1; } /* try reading some */ - if (i_stream_read(ibc->input) == -1) { + if ((ret = i_stream_read(ibc->input)) == -1) { if (ibc->stopped) return -1; error = t_str_new(128); @@ -385,6 +386,7 @@ dsync_ibc_stream_stop(ibc); return -1; } + i_assert(ret >= 0); *line_r = i_stream_next_line(ibc->input); if (*line_r == NULL) { ibc->has_pending_data = FALSE; From dovecot at dovecot.org Wed Apr 23 17:54:27 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Apr 2014 17:54:27 +0000 Subject: dovecot-2.2: doveadm replicator replicate: -p parameter wasn't w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b4e2048e5312 changeset: 17253:b4e2048e5312 user: Timo Sirainen date: Wed Apr 23 20:53:55 2014 +0300 description: doveadm replicator replicate: -p parameter wasn't working correctly. We were just replicating the "-p" as the username. diffstat: src/doveadm/doveadm-replicator.c | 14 ++++++++------ 1 files changed, 8 insertions(+), 6 deletions(-) diffs (66 lines): diff -r e20a6192ddcf -r b4e2048e5312 src/doveadm/doveadm-replicator.c --- a/src/doveadm/doveadm-replicator.c Wed Apr 23 19:21:40 2014 +0300 +++ b/src/doveadm/doveadm-replicator.c Wed Apr 23 20:53:55 2014 +0300 @@ -70,7 +70,7 @@ } static struct replicator_context * -cmd_replicator_init(int argc, char *argv[], const char *getopt_args, +cmd_replicator_init(int *argc, char **argv[], const char *getopt_args, doveadm_command_t *cmd) { struct replicator_context *ctx; @@ -80,7 +80,7 @@ ctx->socket_path = t_strconcat(doveadm_settings->base_dir, "/replicator-doveadm", NULL); - while ((c = getopt(argc, argv, getopt_args)) > 0) { + while ((c = getopt(*argc, *argv, getopt_args)) > 0) { switch (c) { case 'a': ctx->socket_path = optarg; @@ -92,6 +92,8 @@ replicator_cmd_help(cmd); } } + *argc -= optind-1; + *argv += optind-1; replicator_connect(ctx); return ctx; } @@ -137,7 +139,7 @@ const char *line, *const *args; time_t last_fast, last_full; - ctx = cmd_replicator_init(argc, argv, "a:", cmd_replicator_status); + ctx = cmd_replicator_init(&argc, &argv, "a:", cmd_replicator_status); if (argv[1] == NULL) { cmd_replicator_status_overview(ctx); @@ -183,7 +185,7 @@ const char *line; unsigned int i; - ctx = cmd_replicator_init(argc, argv, "a:", cmd_replicator_dsync_status); + ctx = cmd_replicator_init(&argc, &argv, "a:", cmd_replicator_dsync_status); doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); doveadm_print_header("username", "username", @@ -219,7 +221,7 @@ if (argv[1] == NULL) replicator_cmd_help(cmd_replicator_replicate); - ctx = cmd_replicator_init(argc, argv, "a:p:", cmd_replicator_replicate); + ctx = cmd_replicator_init(&argc, &argv, "a:p:", cmd_replicator_replicate); str = t_str_new(128); str_append(str, "REPLICATE\t"); @@ -258,7 +260,7 @@ if (argv[1] == NULL) replicator_cmd_help(cmd_replicator_remove); - ctx = cmd_replicator_init(argc, argv, "a:", cmd_replicator_remove); + ctx = cmd_replicator_init(&argc, &argv, "a:", cmd_replicator_remove); str = t_str_new(128); str_append(str, "REMOVE\t"); From dovecot at dovecot.org Wed Apr 23 17:55:27 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Apr 2014 17:55:27 +0000 Subject: dovecot-2.2: doveadm replicator replicate: Added -f parameter to... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/df172c4de57e changeset: 17254:df172c4de57e user: Timo Sirainen date: Wed Apr 23 20:55:04 2014 +0300 description: doveadm replicator replicate: Added -f parameter to force a full sync for user. diffstat: src/doveadm/doveadm-replicator.c | 11 +++++++++-- src/replication/replicator/doveadm-connection.c | 16 +++++++++++----- src/replication/replicator/replicator-brain.c | 4 +++- src/replication/replicator/replicator-queue.h | 2 ++ 4 files changed, 25 insertions(+), 8 deletions(-) diffs (118 lines): diff -r b4e2048e5312 -r df172c4de57e src/doveadm/doveadm-replicator.c --- a/src/doveadm/doveadm-replicator.c Wed Apr 23 20:53:55 2014 +0300 +++ b/src/doveadm/doveadm-replicator.c Wed Apr 23 20:55:04 2014 +0300 @@ -19,6 +19,7 @@ const char *socket_path; const char *priority; struct istream *input; + bool full_sync; }; extern struct doveadm_cmd doveadm_cmd_replicator[]; @@ -85,6 +86,9 @@ case 'a': ctx->socket_path = optarg; break; + case 'f': + ctx->full_sync = TRUE; + break; case 'p': ctx->priority = optarg; break; @@ -221,7 +225,7 @@ if (argv[1] == NULL) replicator_cmd_help(cmd_replicator_replicate); - ctx = cmd_replicator_init(&argc, &argv, "a:p:", cmd_replicator_replicate); + ctx = cmd_replicator_init(&argc, &argv, "a:fp:", cmd_replicator_replicate); str = t_str_new(128); str_append(str, "REPLICATE\t"); @@ -230,6 +234,9 @@ else str_append_tabescaped(str, ctx->priority); str_append_c(str, '\t'); + if (ctx->full_sync) + str_append_c(str, 'f'); + str_append_c(str, '\t'); str_append_tabescaped(str, argv[1]); str_append_c(str, '\n'); replicator_send(ctx, str_c(str)); @@ -285,7 +292,7 @@ { cmd_replicator_dsync_status, "replicator dsync-status", "[-a ]" }, { cmd_replicator_replicate, "replicator replicate", - "[-a ] [-p ] " }, + "[-a ] [-f] [-p ] " }, { cmd_replicator_remove, "replicator remove", "[-a ] " }, }; diff -r b4e2048e5312 -r df172c4de57e src/replication/replicator/doveadm-connection.c --- a/src/replication/replicator/doveadm-connection.c Wed Apr 23 20:53:55 2014 +0300 +++ b/src/replication/replicator/doveadm-connection.c Wed Apr 23 20:55:04 2014 +0300 @@ -156,9 +156,10 @@ const char *usermask; enum replication_priority priority; unsigned int match_count; + bool full; - /* | */ - if (str_array_length(args) != 2) { + /* | */ + if (str_array_length(args) != 3) { i_error("%s: REPLICATE: Invalid parameters", client->conn.name); return -1; } @@ -166,9 +167,12 @@ o_stream_send_str(client->conn.output, "-Invalid priority\n"); return 0; } - usermask = args[1]; + full = strchr(args[1], 'f') != NULL; + usermask = args[2]; if (strchr(usermask, '*') == NULL && strchr(usermask, '?') == NULL) { - replicator_queue_add(queue, usermask, priority); + user = replicator_queue_add(queue, usermask, priority); + if (full) + user->force_full_sync = TRUE; o_stream_send_str(client->conn.output, "+1\n"); return 0; } @@ -178,7 +182,9 @@ while ((user = replicator_queue_iter_next(iter)) != NULL) { if (!wildcard_match(user->username, usermask)) continue; - replicator_queue_add(queue, user->username, priority); + user = replicator_queue_add(queue, user->username, priority); + if (full) + user->force_full_sync = TRUE; match_count++; } replicator_queue_iter_deinit(&iter); diff -r b4e2048e5312 -r df172c4de57e src/replication/replicator/replicator-brain.c --- a/src/replication/replicator/replicator-brain.c Wed Apr 23 20:53:55 2014 +0300 +++ b/src/replication/replicator/replicator-brain.c Wed Apr 23 20:55:04 2014 +0300 @@ -141,8 +141,10 @@ /* update the sync times immediately. if the replication fails we still wouldn't want it to be retried immediately. */ user->last_fast_sync = ioloop_time; - if (full) + if (full || user->force_full_sync) { user->last_full_sync = ioloop_time; + user->force_full_sync = FALSE; + } /* reset priority also. if more updates arrive during replication we'll do another replication to make sure nothing gets lost */ user->priority = REPLICATION_PRIORITY_NONE; diff -r b4e2048e5312 -r df172c4de57e src/replication/replicator/replicator-queue.h --- a/src/replication/replicator/replicator-queue.h Wed Apr 23 20:53:55 2014 +0300 +++ b/src/replication/replicator/replicator-queue.h Wed Apr 23 20:55:04 2014 +0300 @@ -20,6 +20,8 @@ unsigned int popped:1; /* Last replication sync failed */ unsigned int last_sync_failed:1; + /* Force a full sync on the next replication */ + unsigned int force_full_sync:1; }; typedef void replicator_sync_callback_t(bool success, void *context); From dovecot at dovecot.org Wed Apr 23 19:15:54 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Apr 2014 19:15:54 +0000 Subject: dovecot-2.2: director: When logging about request timeout, log a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fb31a88786df changeset: 17255:fb31a88786df user: Timo Sirainen date: Wed Apr 23 22:13:08 2014 +0300 description: director: When logging about request timeout, log also the username hash. diffstat: src/director/director-request.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r df172c4de57e -r fb31a88786df src/director/director-request.c --- a/src/director/director-request.c Wed Apr 23 20:55:04 2014 +0300 +++ b/src/director/director-request.c Wed Apr 23 22:13:08 2014 +0300 @@ -68,7 +68,7 @@ str_printfa(str, ", user refreshed %u secs ago", (unsigned int)(ioloop_time - user->timestamp)); } - str_append_c(str, ')'); + str_printfa(str, "hash=%u)", request->username_hash); return str_c(str); } From dovecot at dovecot.org Wed Apr 23 19:15:55 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Apr 2014 19:15:55 +0000 Subject: dovecot-2.2: doveadm director map: Show also the username hash. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/14752b3cd609 changeset: 17256:14752b3cd609 user: Timo Sirainen date: Wed Apr 23 22:15:28 2014 +0300 description: doveadm director map: Show also the username hash. diffstat: src/doveadm/doveadm-director.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (25 lines): diff -r fb31a88786df -r 14752b3cd609 src/doveadm/doveadm-director.c --- a/src/doveadm/doveadm-director.c Wed Apr 23 22:13:08 2014 +0300 +++ b/src/doveadm/doveadm-director.c Wed Apr 23 22:15:28 2014 +0300 @@ -297,6 +297,7 @@ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); doveadm_print_header("user", "user", DOVEADM_PRINT_HEADER_FLAG_EXPAND); + doveadm_print_header_simple("hash"); doveadm_print_header_simple("mail server ip"); doveadm_print_header_simple("expire time"); @@ -323,11 +324,13 @@ POINTER_CAST(user_hash)); if (user == NULL) { doveadm_print(""); + doveadm_print(args[0]); doveadm_print(args[2]); doveadm_print(unixdate2str(expires)); } for (; user != NULL; user = user->next) { doveadm_print(user->name); + doveadm_print(args[0]); doveadm_print(args[2]); doveadm_print(unixdate2str(expires)); } From dovecot at dovecot.org Wed Apr 23 19:30:51 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Apr 2014 19:30:51 +0000 Subject: dovecot-2.2: doveadm director map: Added -h and -u parameters to... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bc9863fa07af changeset: 17257:bc9863fa07af user: Timo Sirainen date: Wed Apr 23 22:30:28 2014 +0300 description: doveadm director map: Added -h and -u parameters to quickly map between username and hash. diffstat: src/doveadm/doveadm-director.c | 48 ++++++++++++++++++++++++++++++++++++----- 1 files changed, 42 insertions(+), 6 deletions(-) diffs (95 lines): diff -r 14752b3cd609 -r bc9863fa07af src/doveadm/doveadm-director.c --- a/src/doveadm/doveadm-director.c Wed Apr 23 22:15:28 2014 +0300 +++ b/src/doveadm/doveadm-director.c Wed Apr 23 22:30:28 2014 +0300 @@ -23,6 +23,7 @@ const char *users_path; struct istream *input; bool explicit_socket_path; + bool hash_map, user_map; }; struct user_list { @@ -102,6 +103,12 @@ case 'f': ctx->users_path = optarg; break; + case 'h': + ctx->hash_map = TRUE; + break; + case 'u': + ctx->user_map = TRUE; + break; default: director_cmd_help(cmd); } @@ -280,13 +287,28 @@ struct user_list *user; unsigned int ips_count, user_hash, expires; - ctx = cmd_director_init(argc, argv, "a:f:", cmd_director_map); - if (argv[optind] == NULL) + ctx = cmd_director_init(argc, argv, "a:f:hu", cmd_director_map); + argc -= optind; + argv += optind; + if (argc > 1 || + (ctx->hash_map && ctx->user_map) || + ((ctx->hash_map || ctx->user_map) && argc == 0)) + director_cmd_help(cmd_director_map); + + if (ctx->user_map) { + /* user -> hash mapping */ + user_hash = mail_user_hash(argv[0], doveadm_settings->director_username_hash); + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header("hash", "hash", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + doveadm_print(t_strdup_printf("%u", user_hash)); + director_disconnect(ctx); + return; + } + + if (argv[0] == NULL || ctx->hash_map) ips_count = 0; - else if (argv[optind+1] != NULL) - director_cmd_help(cmd_director_map); else - director_get_host(argv[optind], &ips, &ips_count); + director_get_host(argv[0], &ips, &ips_count); pool = pool_alloconly_create("director map users", 1024*128); hash_table_create_direct(&users, pool, 0); @@ -295,6 +317,19 @@ else user_file_get_user_list(ctx->users_path, pool, users); + if (ctx->hash_map) { + /* hash -> usernames mapping */ + if (str_to_uint(argv[0], &user_hash) < 0) + i_fatal("Invalid username hash: %s", argv[0]); + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header("user", "user", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + user = hash_table_lookup(users, POINTER_CAST(user_hash)); + for (; user != NULL; user = user->next) + doveadm_print(user->name); + goto deinit; + } + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); doveadm_print_header("user", "user", DOVEADM_PRINT_HEADER_FLAG_EXPAND); doveadm_print_header_simple("hash"); @@ -341,6 +376,7 @@ i_error("Director disconnected unexpectedly"); doveadm_exit_code = EX_TEMPFAIL; } +deinit: director_disconnect(ctx); hash_table_destroy(&users); pool_unref(&pool); @@ -683,7 +719,7 @@ { cmd_director_status, "director status", "[-a ] []" }, { cmd_director_map, "director map", - "[-a ] [-f ] []" }, + "[-a ] [-f ] [-h | -u] []" }, { cmd_director_add, "director add", "[-a ] []" }, { cmd_director_remove, "director remove", From dovecot at dovecot.org Wed Apr 23 19:34:22 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Apr 2014 19:34:22 +0000 Subject: dovecot-2.2: doveadm director map -u: Don't connect to director ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/45bedf335e92 changeset: 17258:45bedf335e92 user: Timo Sirainen date: Wed Apr 23 22:33:48 2014 +0300 description: doveadm director map -u: Don't connect to director socket, it's not needed. diffstat: src/doveadm/doveadm-director.c | 11 +++++++---- 1 files changed, 7 insertions(+), 4 deletions(-) diffs (28 lines): diff -r bc9863fa07af -r 45bedf335e92 src/doveadm/doveadm-director.c --- a/src/doveadm/doveadm-director.c Wed Apr 23 22:30:28 2014 +0300 +++ b/src/doveadm/doveadm-director.c Wed Apr 23 22:33:48 2014 +0300 @@ -78,9 +78,11 @@ static void director_disconnect(struct director_context *ctx) { - if (ctx->input->stream_errno != 0) - i_fatal("read(%s) failed: %m", ctx->socket_path); - i_stream_destroy(&ctx->input); + if (ctx->input != NULL) { + if (ctx->input->stream_errno != 0) + i_fatal("read(%s) failed: %m", ctx->socket_path); + i_stream_destroy(&ctx->input); + } } static struct director_context * @@ -113,7 +115,8 @@ director_cmd_help(cmd); } } - director_connect(ctx); + if (!ctx->user_map) + director_connect(ctx); return ctx; } From dovecot at dovecot.org Wed Apr 23 20:27:16 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Apr 2014 20:27:16 +0000 Subject: dovecot-2.2: director: Fixed handling weak users when there were... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6269063f8408 changeset: 17259:6269063f8408 user: Timo Sirainen date: Wed Apr 23 23:26:49 2014 +0300 description: director: Fixed handling weak users when there were exactly 2 director servers in the ring. diffstat: src/director/director-connection.c | 2 +- src/director/director-request.c | 2 +- src/director/director.c | 26 +++++++++++++++++++++++--- src/director/director.h | 1 + 4 files changed, 26 insertions(+), 5 deletions(-) diffs (82 lines): diff -r 45bedf335e92 -r 6269063f8408 src/director/director-connection.c --- a/src/director/director-connection.c Wed Apr 23 22:33:48 2014 +0300 +++ b/src/director/director-connection.c Wed Apr 23 23:26:49 2014 +0300 @@ -799,7 +799,7 @@ if (!user->weak) director_update_user(conn->dir, src_host, user); else { - director_update_user_weak(conn->dir, src_host, + director_update_user_weak(conn->dir, src_host, conn, dir_host, user); } } diff -r 45bedf335e92 -r 6269063f8408 src/director/director-request.c --- a/src/director/director-request.c Wed Apr 23 22:33:48 2014 +0300 +++ b/src/director/director-request.c Wed Apr 23 23:26:49 2014 +0300 @@ -241,7 +241,7 @@ return TRUE; } else { user->weak = TRUE; - director_update_user_weak(dir, dir->self_host, NULL, user); + director_update_user_weak(dir, dir->self_host, NULL, NULL, user); request->delay_reason = REQUEST_DELAY_WEAK; dir_debug("request: %u set to weak", request->username_hash); return FALSE; diff -r 45bedf335e92 -r 6269063f8408 src/director/director.c --- a/src/director/director.c Wed Apr 23 22:33:48 2014 +0300 +++ b/src/director/director.c Wed Apr 23 23:26:49 2014 +0300 @@ -566,9 +566,12 @@ } void director_update_user_weak(struct director *dir, struct director_host *src, + struct director_connection *src_conn, struct director_host *orig_src, struct user *user) { + const char *cmd; + i_assert(src != NULL); i_assert(user->weak); @@ -577,10 +580,27 @@ orig_src->last_seq++; } - director_update_send(dir, src, t_strdup_printf( - "USER-WEAK\t%s\t%u\t%u\t%u\t%s\n", + cmd = t_strdup_printf("USER-WEAK\t%s\t%u\t%u\t%u\t%s\n", net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq, - user->username_hash, net_ip2addr(&user->host->ip))); + user->username_hash, net_ip2addr(&user->host->ip)); + + if (src != dir->self_host && dir->left != NULL && dir->right != NULL && + director_connection_get_host(dir->left) == + director_connection_get_host(dir->right)) { + /* only two directors in this ring and we're forwarding + USER-WEAK from one director back to itself via another + so it sees we've received it. we can't use + director_update_send() for this, because it doesn't send + data back to the source. */ + if (dir->right == src_conn) + director_connection_send(dir->left, cmd); + else if (dir->left == src_conn) + director_connection_send(dir->right, cmd); + else + i_unreached(); + } else { + director_update_send(dir, src, cmd); + } } struct director_user_kill_finish_ctx { diff -r 45bedf335e92 -r 6269063f8408 src/director/director.h --- a/src/director/director.h Wed Apr 23 22:33:48 2014 +0300 +++ b/src/director/director.h Wed Apr 23 23:26:49 2014 +0300 @@ -124,6 +124,7 @@ void director_update_user(struct director *dir, struct director_host *src, struct user *user); void director_update_user_weak(struct director *dir, struct director_host *src, + struct director_connection *src_conn, struct director_host *orig_src, struct user *user) ATTR_NULL(3); void director_move_user(struct director *dir, struct director_host *src, From dovecot at dovecot.org Thu Apr 24 10:49:58 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Apr 2014 10:49:58 +0000 Subject: dovecot-2.2: i_stream_read_next_line(): Set also errno=ENOBUFS o... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aec6f57b8cf8 changeset: 17260:aec6f57b8cf8 user: Timo Sirainen date: Thu Apr 24 13:49:25 2014 +0300 description: i_stream_read_next_line(): Set also errno=ENOBUFS on too large line. Some callers assume that errno is set to same as istream->stream_errno diffstat: src/lib/istream.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 6269063f8408 -r aec6f57b8cf8 src/lib/istream.c --- a/src/lib/istream.c Wed Apr 23 23:26:49 2014 +0300 +++ b/src/lib/istream.c Thu Apr 24 13:49:25 2014 +0300 @@ -426,7 +426,7 @@ switch (i_stream_read(stream)) { case -2: - stream->stream_errno = ENOBUFS; + stream->stream_errno = errno = ENOBUFS; stream->eof = TRUE; return NULL; case -1: From dovecot at dovecot.org Thu Apr 24 15:19:12 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Apr 2014 15:19:12 +0000 Subject: dovecot-2.2: dsync: Fixed high data stack memory usage Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ddf374a36057 changeset: 17261:ddf374a36057 user: Timo Sirainen date: Thu Apr 24 18:18:42 2014 +0300 description: dsync: Fixed high data stack memory usage diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diffs (20 lines): diff -r aec6f57b8cf8 -r ddf374a36057 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Thu Apr 24 13:49:25 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Thu Apr 24 18:18:42 2014 +0300 @@ -557,9 +557,14 @@ { int ret; - while ((ret = importer_try_next_mail(importer, wanted_uid)) == 0 && - !importer->failed) + for (;;) { + T_BEGIN { + ret = importer_try_next_mail(importer, wanted_uid); + } T_END; + if (ret != 0 || importer->failed) + break; importer->next_local_seq = importer->cur_mail->seq + 1; + } return ret > 0; } From dovecot at dovecot.org Fri Apr 25 07:50:19 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Apr 2014 07:50:19 +0000 Subject: dovecot-2.2: dsync: If a parent \Noselect gets autocreated, don'... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8575e8f01380 changeset: 17262:8575e8f01380 user: Timo Sirainen date: Fri Apr 25 10:49:45 2014 +0300 description: dsync: If a parent \Noselect gets autocreated, don't fail later when trying to re-create it. diffstat: src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-) diffs (19 lines): diff -r ddf374a36057 -r 8575e8f01380 src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c Thu Apr 24 18:18:42 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c Fri Apr 25 10:49:45 2014 +0300 @@ -167,6 +167,15 @@ return ret; case DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR: ret = mailbox_create(box, NULL, TRUE); + if (ret < 0 && + mailbox_get_last_mail_error(box) == MAIL_ERROR_EXISTS) { + /* it doesn't matter if somebody else created this + directory or we automatically did while creating its + child mailbox. it's there now anyway and we don't + gain anything by treating this failure any + differently from success. */ + ret = 0; + } func_name = "mailbox_create"; break; case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX: From dovecot at dovecot.org Fri Apr 25 12:46:47 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Apr 2014 12:46:47 +0000 Subject: dovecot-2.2: lib-lda: Removed unused mail_send_forward() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/560793d4bf88 changeset: 17263:560793d4bf88 user: Timo Sirainen date: Fri Apr 25 15:31:08 2014 +0300 description: lib-lda: Removed unused mail_send_forward() diffstat: src/lib-lda/mail-send.c | 33 --------------------------------- src/lib-lda/mail-send.h | 1 - 2 files changed, 0 insertions(+), 34 deletions(-) diffs (50 lines): diff -r 8575e8f01380 -r 560793d4bf88 src/lib-lda/mail-send.c --- a/src/lib-lda/mail-send.c Fri Apr 25 10:49:45 2014 +0300 +++ b/src/lib-lda/mail-send.c Fri Apr 25 15:31:08 2014 +0300 @@ -182,36 +182,3 @@ o_stream_nsend(output, str_data(str), str_len(str)); return smtp_client_close(smtp_client); } - -int mail_send_forward(struct mail_deliver_context *ctx, const char *forwardto) -{ - static const char *hide_headers[] = { - "Return-Path" - }; - struct istream *input; - struct ostream *output; - struct smtp_client *smtp_client; - const char *return_path; - - if (mail_get_stream(ctx->src_mail, NULL, NULL, &input) < 0) - return -1; - - return_path = mail_deliver_get_return_address(ctx); - if (mailbox_get_settings(ctx->src_mail->box)->mail_debug) { - i_debug("Sending a forward to <%s> with return path <%s>", - forwardto, return_path); - } - - smtp_client = smtp_client_open(ctx->set, forwardto, return_path, &output); - - input = i_stream_create_header_filter(input, HEADER_FILTER_EXCLUDE | - HEADER_FILTER_NO_CR, hide_headers, - N_ELEMENTS(hide_headers), - *null_header_filter_callback, - (void *)NULL); - - (void)o_stream_send_istream(output, input); - i_stream_unref(&input); - - return smtp_client_close(smtp_client); -} diff -r 8575e8f01380 -r 560793d4bf88 src/lib-lda/mail-send.h --- a/src/lib-lda/mail-send.h Fri Apr 25 10:49:45 2014 +0300 +++ b/src/lib-lda/mail-send.h Fri Apr 25 15:31:08 2014 +0300 @@ -6,6 +6,5 @@ int mail_send_rejection(struct mail_deliver_context *ctx, const char *recipient, const char *reason); -int mail_send_forward(struct mail_deliver_context *ctx, const char *forwardto); #endif From dovecot at dovecot.org Fri Apr 25 12:46:48 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Apr 2014 12:46:48 +0000 Subject: dovecot-2.2: lib-http: Added more information to "stream input s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b433260ced36 changeset: 17264:b433260ced36 user: Timo Sirainen date: Fri Apr 25 15:46:20 2014 +0300 description: lib-http: Added more information to "stream input size changed" error diffstat: src/lib-http/http-client-request.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (17 lines): diff -r 560793d4bf88 -r b433260ced36 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Fri Apr 25 15:31:08 2014 +0300 +++ b/src/lib-http/http-client-request.c Fri Apr 25 15:46:20 2014 +0300 @@ -596,8 +596,11 @@ if (ret < 0 || i_stream_is_eof(req->payload_input)) { if (!req->payload_chunked && req->payload_input->v_offset - req->payload_offset != req->payload_size) { - *error_r = "stream input size changed [BUG]"; - i_error("stream input size changed"); //FIXME + *error_r = t_strdup_printf("BUG: stream '%s' input size changed: " + "%"PRIuUOFF_T"-%"PRIuUOFF_T" != %"PRIuUOFF_T, + i_stream_get_name(req->payload_input), + req->payload_input->v_offset, req->payload_offset, req->payload_size); + i_error("%s", *error_r); //FIXME: remove? return -1; } From dovecot at dovecot.org Fri Apr 25 12:53:00 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Apr 2014 12:53:00 +0000 Subject: dovecot-2.2: safe_memset(size=0) triggered read access to data[0... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/eae44873dbc2 changeset: 17265:eae44873dbc2 user: Timo Sirainen date: Fri Apr 25 15:52:37 2014 +0300 description: safe_memset(size=0) triggered read access to data[0], possibly crashing. diffstat: src/lib/safe-memset.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r b433260ced36 -r eae44873dbc2 src/lib/safe-memset.c --- a/src/lib/safe-memset.c Fri Apr 25 15:46:20 2014 +0300 +++ b/src/lib/safe-memset.c Fri Apr 25 15:52:37 2014 +0300 @@ -8,6 +8,9 @@ volatile unsigned int volatile_zero_idx = 0; volatile unsigned char *p = data; + if (size == 0) + return; + do { memset(data, c, size); } while (p[volatile_zero_idx] != c); From dovecot at dovecot.org Fri Apr 25 13:15:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Apr 2014 13:15:53 +0000 Subject: dovecot-2.2: lib-lda: Added new smtp_client_init/deinit() APIs Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0f9ce0073c8d changeset: 17266:0f9ce0073c8d user: Timo Sirainen date: Fri Apr 25 15:54:49 2014 +0300 description: lib-lda: Added new smtp_client_init/deinit() APIs The new API allows better error handling. diffstat: src/lib-lda/smtp-client.c | 88 ++++++++++++++++++++++++++++++++++++++-------- src/lib-lda/smtp-client.h | 8 ++++ 2 files changed, 80 insertions(+), 16 deletions(-) diffs (194 lines): diff -r eae44873dbc2 -r 0f9ce0073c8d src/lib-lda/smtp-client.c --- a/src/lib-lda/smtp-client.c Fri Apr 25 15:52:37 2014 +0300 +++ b/src/lib-lda/smtp-client.c Fri Apr 25 15:54:49 2014 +0300 @@ -34,6 +34,8 @@ char *temp_path; char *destination; char *return_path; + char *error; + bool tempfail; }; static struct smtp_client *smtp_client_devnull(struct ostream **output_r) @@ -84,7 +86,7 @@ } static struct smtp_client * -smtp_client_open_sendmail(const struct lda_settings *set, +smtp_client_init_sendmail(const struct lda_settings *set, const char *destination, const char *return_path, struct ostream **output_r) { @@ -147,7 +149,7 @@ } struct smtp_client * -smtp_client_open(const struct lda_settings *set, const char *destination, +smtp_client_init(const struct lda_settings *set, const char *destination, const char *return_path, struct ostream **output_r) { struct smtp_client *client; @@ -155,7 +157,7 @@ int fd; if (*set->submission_host == '\0') { - return smtp_client_open_sendmail(set, destination, + return smtp_client_init_sendmail(set, destination, return_path, output_r); } @@ -176,14 +178,14 @@ return client; } -static int smtp_client_close_sendmail(struct smtp_client *client) +static int smtp_client_deinit_sendmail(struct smtp_client *client) { int ret = EX_TEMPFAIL, status; o_stream_destroy(&client->output); if (client->pid == (pid_t)-1) { - /* smtp_client_open() failed already */ + /* smtp_client_init() failed already */ } else if (waitpid(client->pid, &status, 0) < 0) i_error("waitpid() failed: %m"); else if (WIFEXITED(status)) { @@ -193,8 +195,8 @@ "exit status %d", ret); } } else if (WIFSIGNALED(status)) { - i_error("Sendmail process terminated abnormally, " - "signal %d", WTERMSIG(status)); + i_error("Sendmail process terminated abnormally, signal %d", + WTERMSIG(status)); } else if (WIFSTOPPED(status)) { i_error("Sendmail process stopped, signal %d", WSTOPSIG(status)); @@ -217,13 +219,24 @@ } static void +smtp_client_error(struct smtp_client *smtp_client, const char *error) +{ + if (smtp_client->error == NULL) { + smtp_client->error = i_strdup_printf("smtp(%s): %s", + smtp_client->set->submission_host, error); + } +} + +static void rcpt_to_callback(bool success, const char *reply, void *context) { struct smtp_client *smtp_client = context; if (!success) { - i_error("smtp(%s): RCPT TO failed: %s", - smtp_client->set->submission_host, reply); + if (reply[0] != '5') + smtp_client->tempfail = TRUE; + smtp_client_error(smtp_client, t_strdup_printf( + "RCPT TO failed: %s", reply)); smtp_client_send_finished(smtp_client); } } @@ -234,8 +247,10 @@ struct smtp_client *smtp_client = context; if (!success) { - i_error("smtp(%s): DATA failed: %s", - smtp_client->set->submission_host, reply); + if (reply[0] != '5') + smtp_client->tempfail = TRUE; + smtp_client_error(smtp_client, t_strdup_printf( + "DATA failed: %s", reply)); smtp_client_send_finished(smtp_client); } else { smtp_client->success = TRUE; @@ -298,15 +313,26 @@ if (!smtp_client->finished) io_loop_run(ioloop); io_loop_destroy(&ioloop); - return smtp_client->success ? 0 : -1; + + if (smtp_client->success) + return 1; + else if (smtp_client->tempfail) + return -1; + else + return 0; } -int smtp_client_close(struct smtp_client *client) +int smtp_client_deinit(struct smtp_client *client, const char **error_r) { int ret; - if (!client->use_smtp) - return smtp_client_close_sendmail(client); + if (!client->use_smtp) { + if (smtp_client_deinit_sendmail(client) != 0) { + *error_r = "Failed to execute sendmail"; + return -1; + } + return 1; + } /* the mail has been written to a file. now actually send it. */ ret = smtp_client_send(client); @@ -315,6 +341,36 @@ i_free(client->return_path); i_free(client->destination); i_free(client->temp_path); + + *error_r = t_strdup(client->error); + i_free(client->error); i_free(client); - return ret < 0 ? EX_TEMPFAIL : 0; + return ret; } + +struct smtp_client * +smtp_client_open(const struct lda_settings *set, const char *destination, + const char *return_path, struct ostream **output_r) +{ + return smtp_client_init(set, destination, return_path, output_r); +} + +int smtp_client_close(struct smtp_client *client) +{ + const char *error; + int ret; + + if (!client->use_smtp) + return smtp_client_deinit_sendmail(client); + + ret = smtp_client_deinit(client, &error); + if (ret < 0) { + i_error("%s", error); + return EX_TEMPFAIL; + } + if (ret == 0) { + i_error("%s", error); + return EX_NOPERM; + } + return 0; +} diff -r eae44873dbc2 -r 0f9ce0073c8d src/lib-lda/smtp-client.h --- a/src/lib-lda/smtp-client.h Fri Apr 25 15:52:37 2014 +0300 +++ b/src/lib-lda/smtp-client.h Fri Apr 25 15:54:49 2014 +0300 @@ -4,6 +4,14 @@ #include struct smtp_client * ATTR_NULL(3) +smtp_client_init(const struct lda_settings *set, const char *destination, + const char *return_path, struct ostream **output_r); +/* Returns 1 on success, 0 on permanent failure (e.g. invalid destination), + -1 on temporary failure. */ +int smtp_client_deinit(struct smtp_client *client, const char **error_r); + +/* FIXME: obsolete API, remove in v2.3: */ +struct smtp_client * ATTR_NULL(3) smtp_client_open(const struct lda_settings *set, const char *destination, const char *return_path, struct ostream **output_r); /* Returns sysexits-compatible return value */ From dovecot at dovecot.org Fri Apr 25 13:15:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Apr 2014 13:15:53 +0000 Subject: dovecot-2.2: lib-lda: mail_send_rejection() logs permanent rejec... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dc5ba8637aa7 changeset: 17267:dc5ba8637aa7 user: Timo Sirainen date: Fri Apr 25 15:57:03 2014 +0300 description: lib-lda: mail_send_rejection() logs permanent rejections now with info level. There's no need to log an error when return-path is invalid. diffstat: src/lib-lda/mail-send.c | 13 +++++++++++-- 1 files changed, 11 insertions(+), 2 deletions(-) diffs (28 lines): diff -r 0f9ce0073c8d -r dc5ba8637aa7 src/lib-lda/mail-send.c --- a/src/lib-lda/mail-send.c Fri Apr 25 15:54:49 2014 +0300 +++ b/src/lib-lda/mail-send.c Fri Apr 25 15:57:03 2014 +0300 @@ -58,7 +58,7 @@ struct smtp_client *smtp_client; struct ostream *output; const char *return_addr, *hdr; - const char *value, *msgid, *orig_msgid, *boundary; + const char *value, *msgid, *orig_msgid, *boundary, *error; string_t *str; int ret; @@ -180,5 +180,14 @@ str_truncate(str, 0); str_printfa(str, "\r\n\r\n--%s--\r\n", boundary); o_stream_nsend(output, str_data(str), str_len(str)); - return smtp_client_close(smtp_client); + if ((ret = smtp_client_deinit(smtp_client, &error)) < 0) { + i_error("msgid=%s: Temporarily failed to send rejection: %s", + orig_msgid == NULL ? "" : str_sanitize(orig_msgid, 80), + str_sanitize(error, 512)); + } else if (ret == 0) { + i_info("msgid=%s: Permanently failed to send rejection: %s", + orig_msgid == NULL ? "" : str_sanitize(orig_msgid, 80), + str_sanitize(error, 512)); + } + return ret < 0 ? -1 : 0; } From dovecot at dovecot.org Fri Apr 25 13:15:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Apr 2014 13:15:53 +0000 Subject: dovecot-2.2: lib-lda: When non-RCPT/DATA command fails permanent... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c03076e03250 changeset: 17268:c03076e03250 user: Timo Sirainen date: Fri Apr 25 16:15:20 2014 +0300 description: lib-lda: When non-RCPT/DATA command fails permanently, translate it internally to temporary failure. We shouldn't be answering permanent failures to RCPT/DATA command when the failure really had nothing to do with the recipient/content itself. Most likely it's just some server misconfiguration, which should be treated as temporary error. diffstat: src/lib-lda/lmtp-client.c | 33 ++++++++++++++++++++++++--------- 1 files changed, 24 insertions(+), 9 deletions(-) diffs (67 lines): diff -r dc5ba8637aa7 -r c03076e03250 src/lib-lda/lmtp-client.c --- a/src/lib-lda/lmtp-client.c Fri Apr 25 15:57:03 2014 +0300 +++ b/src/lib-lda/lmtp-client.c Fri Apr 25 16:15:20 2014 +0300 @@ -458,39 +458,54 @@ client->input_multiline)) <= 0) { if (ret == 0) return 0; - lmtp_client_fail(client, line); + lmtp_client_fail(client, t_strdup_printf( + "451 4.5.0 Received invalid input: %s", line)); return -1; } switch (client->input_state) { case LMTP_INPUT_STATE_GREET: + if (reply_code != 220) { + lmtp_client_fail(client, t_strdup_printf( + "451 4.5.0 Received invalid greeting: %s", line)); + return -1; + } + lmtp_client_send_handshake(client); + client->input_state = LMTP_INPUT_STATE_LHLO; + break; case LMTP_INPUT_STATE_XCLIENT: if (reply_code != 220) { - lmtp_client_fail(client, line); + lmtp_client_fail(client, t_strdup_printf( + "451 4.5.0 XCLIENT failed: %s", line)); return -1; } lmtp_client_send_handshake(client); client->input_state = LMTP_INPUT_STATE_LHLO; break; case LMTP_INPUT_STATE_LHLO: - case LMTP_INPUT_STATE_MAIL_FROM: if (reply_code != 250) { + lmtp_client_fail(client, t_strdup_printf( + "451 4.5.0 LHLO failed: %s", line)); lmtp_client_fail(client, line); return -1; } str_append(client->input_multiline, line); lmtp_client_parse_capabilities(client, str_c(client->input_multiline)); - if (client->input_state == LMTP_INPUT_STATE_LHLO && - lmtp_client_send_xclient(client)) { + if (lmtp_client_send_xclient(client)) { client->input_state = LMTP_INPUT_STATE_XCLIENT; client->xclient_sent = TRUE; break; } - if (client->input_state == LMTP_INPUT_STATE_LHLO) { - o_stream_nsend_str(client->output, - t_strdup_printf("MAIL FROM:%s\r\n", - client->set.mail_from)); + o_stream_nsend_str(client->output, t_strdup_printf( + "MAIL FROM:%s\r\n", client->set.mail_from)); + client->input_state++; + break; + case LMTP_INPUT_STATE_MAIL_FROM: + if (reply_code != 250) { + lmtp_client_fail(client, t_strdup_printf( + "451 4.5.0 MAIL FROM failed: %s", line)); + return -1; } client->input_state++; lmtp_client_send_rcpts(client); From dovecot at dovecot.org Fri Apr 25 13:18:39 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Apr 2014 13:18:39 +0000 Subject: dovecot-2.2: lib-lda: Use the new smtp_client_init(), not _open() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b53ebeaba6aa changeset: 17269:b53ebeaba6aa user: Timo Sirainen date: Fri Apr 25 16:18:09 2014 +0300 description: lib-lda: Use the new smtp_client_init(), not _open() diffstat: src/lib-lda/mail-send.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r c03076e03250 -r b53ebeaba6aa src/lib-lda/mail-send.c --- a/src/lib-lda/mail-send.c Fri Apr 25 16:15:20 2014 +0300 +++ b/src/lib-lda/mail-send.c Fri Apr 25 16:18:09 2014 +0300 @@ -86,7 +86,7 @@ str_sanitize(reason, 512)); } - smtp_client = smtp_client_open(ctx->set, return_addr, NULL, &output); + smtp_client = smtp_client_init(ctx->set, return_addr, NULL, &output); msgid = mail_deliver_get_new_message_id(ctx); boundary = t_strdup_printf("%s/%s", my_pid, ctx->set->hostname); From dovecot at dovecot.org Fri Apr 25 13:36:30 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Apr 2014 13:36:30 +0000 Subject: dovecot-2.2: doveadm dump: Added support for mobox extensions in... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5798bb0167b9 changeset: 17270:5798bb0167b9 user: Timo Sirainen date: Fri Apr 25 16:35:51 2014 +0300 description: doveadm dump: Added support for mobox extensions in dovecot.index. diffstat: src/doveadm/doveadm-dump-index.c | 41 ++++++++++++++++++++++++++++++++++++++++ 1 files changed, 41 insertions(+), 0 deletions(-) diffs (65 lines): diff -r b53ebeaba6aa -r 5798bb0167b9 src/doveadm/doveadm-dump-index.c --- a/src/doveadm/doveadm-dump-index.c Fri Apr 25 16:18:09 2014 +0300 +++ b/src/doveadm/doveadm-dump-index.c Fri Apr 25 16:35:51 2014 +0300 @@ -53,6 +53,25 @@ unsigned char guid[GUID_128_SIZE]; unsigned char oid[GUID_128_SIZE]; }; +struct mobox_mail_index_header { + uint32_t rebuild_count; + uint32_t map_uid_validity; + uint8_t unused[4]; + guid_128_t mailbox_guid; +}; +struct mobox_mail_index_record { + uint32_t map_uid; + uint32_t save_date; +}; +struct mobox_map_mail_index_header { + uint32_t rebuild_count; +}; + +struct mobox_map_mail_index_record { + uint32_t offset; + uint32_t size; + guid_128_t oid; +}; struct mailbox_list_index_record { uint32_t name_id; uint32_t parent_uid; @@ -184,6 +203,19 @@ printf(" - mailbox_guid .. = %s\n", guid_128_to_string(hdr->mailbox_guid)); printf(" - flags ......... = 0x%x\n", hdr->flags); + } else if (strcmp(ext->name, "mobox-hdr") == 0) { + const struct mobox_mail_index_header *hdr = data; + + printf("header\n"); + printf(" - rebuild_count .. = %u\n", hdr->rebuild_count); + printf(" - map_uid_validity .. = %u\n", hdr->map_uid_validity); + printf(" - mailbox_guid ...... = %s\n", + guid_128_to_string(hdr->mailbox_guid)); + } else if (strcmp(ext->name, "mobox-map") == 0) { + const struct mobox_map_mail_index_header *hdr = data; + + printf("header\n"); + printf(" - rebuild_count .. = %u\n", hdr->rebuild_count); } else if (strcmp(ext->name, "modseq") == 0) { const struct mail_index_modseq_header *hdr = data; @@ -579,6 +611,15 @@ const struct obox_mail_index_record *orec = data; printf(" : guid = %s\n", guid_128_to_string(orec->guid)); printf(" : oid = %s\n", guid_128_to_string(orec->oid)); + } else if (strcmp(ext[i].name, "mobox") == 0) { + const struct mobox_mail_index_record *orec = data; + printf(" : map_uid = %u\n", orec->map_uid); + printf(" : save_date = %u (%s)\n", orec->save_date, unixdate2str(orec->save_date)); + } else if (strcmp(ext[i].name, "mobox-map") == 0) { + const struct mobox_map_mail_index_record *orec = data; + printf(" : offset = %u\n", orec->offset); + printf(" : size = %u\n", orec->size); + printf(" : oid = %s\n", guid_128_to_string(orec->oid)); } else if (strcmp(ext[i].name, "list") == 0) { const struct mailbox_list_index_record *lrec = data; printf(" : name_id = %u\n", lrec->name_id); From dovecot at dovecot.org Fri Apr 25 14:29:51 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Apr 2014 14:29:51 +0000 Subject: dovecot-2.2: dsync: Added more debug output Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bb5deb884303 changeset: 17271:bb5deb884303 user: Timo Sirainen date: Fri Apr 25 17:28:39 2014 +0300 description: dsync: Added more debug output diffstat: src/doveadm/dsync/dsync-brain-mailbox.c | 20 +++++++++++++ src/doveadm/dsync/dsync-brain.c | 48 ++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 4 deletions(-) diffs (131 lines): diff -r 5798bb0167b9 -r bb5deb884303 src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Fri Apr 25 16:35:51 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Fri Apr 25 17:28:39 2014 +0300 @@ -374,6 +374,16 @@ /* if mailbox's last_common_* state equals the current state, we can skip the mailbox */ if (!dsync_brain_has_mailbox_state_changed(brain, &dsync_box)) { + if (brain->debug) { + i_debug("brain %c: Skipping mailbox %s with unchanged state " + "uidvalidity=%u uidnext=%u highestmodseq=%llu highestpvtmodseq=%llu", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box.mailbox_guid), + dsync_box.uid_validity, + dsync_box.uid_next, + (unsigned long long)dsync_box.highest_modseq, + (unsigned long long)dsync_box.highest_pvt_modseq); + } mailbox_free(&box); return 0; } @@ -658,6 +668,11 @@ return TRUE; } /* another process just deleted this mailbox? */ + if (brain->debug) { + i_debug("brain %c: Skipping lost mailbox %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid)); + } dsync_brain_slave_send_mailbox_lost(brain, dsync_box); return TRUE; } @@ -670,6 +685,11 @@ if (!dsync_boxes_need_sync(brain, &local_dsync_box, dsync_box)) { /* no fields appear to have changed, skip this mailbox */ + if (brain->debug) { + i_debug("brain %c: Skipping unchanged mailbox %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid)); + } mailbox_free(&box); return TRUE; } diff -r 5798bb0167b9 -r bb5deb884303 src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Fri Apr 25 16:35:51 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain.c Fri Apr 25 17:28:39 2014 +0300 @@ -33,6 +33,8 @@ "done" }; +static void dsync_brain_mailbox_states_dump(struct dsync_brain *brain); + static const char *dsync_brain_get_proctitle(struct dsync_brain *brain) { string_t *str = t_str_new(128); @@ -174,13 +176,20 @@ brain->master_brain = TRUE; dsync_brain_set_flags(brain, flags); - if (sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE && - dsync_mailbox_states_import(brain->mailbox_states, - brain->pool, set->state, &error) < 0) { + if (sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE) + ; + else if (dsync_mailbox_states_import(brain->mailbox_states, brain->pool, + set->state, &error) < 0) { hash_table_clear(brain->mailbox_states, FALSE); i_error("Saved sync state is invalid, " "falling back to full sync: %s", error); brain->sync_type = sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL; + } else { + if (brain->debug) { + i_debug("brain %c: Imported mailbox states:", + brain->master_brain ? 'M' : 'S'); + dsync_brain_mailbox_states_dump(brain); + } } dsync_brain_mailbox_trees_init(brain); @@ -597,6 +606,26 @@ return ret; } +static void dsync_brain_mailbox_states_dump(struct dsync_brain *brain) +{ + struct hash_iterate_context *iter; + struct dsync_mailbox_state *state; + uint8_t *guid; + + iter = hash_table_iterate_init(brain->mailbox_states); + while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) { + i_debug("brain %c: Mailbox %s state: uidvalidity=%u uid=%u modseq=%llu pvt_modseq=%llu changes_during_sync=%d", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(guid), + state->last_uidvalidity, + state->last_common_uid, + (unsigned long long)state->last_common_modseq, + (unsigned long long)state->last_common_pvt_modseq, + state->changes_during_sync); + } + hash_table_iterate_deinit(&iter); +} + void dsync_brain_get_state(struct dsync_brain *brain, string_t *output) { struct hash_iterate_context *iter; @@ -622,11 +651,22 @@ node = dsync_mailbox_tree_lookup_guid(brain->local_mailbox_tree, guid); if (node == NULL || - node->existence != DSYNC_MAILBOX_NODE_EXISTS) + node->existence != DSYNC_MAILBOX_NODE_EXISTS) { + if (brain->debug) { + i_debug("brain %c: Removed state for deleted mailbox %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(guid)); + } hash_table_remove(brain->mailbox_states, guid); + } } hash_table_iterate_deinit(&iter); + if (brain->debug) { + i_debug("brain %c: Exported mailbox states:", + brain->master_brain ? 'M' : 'S'); + dsync_brain_mailbox_states_dump(brain); + } dsync_mailbox_states_export(brain->mailbox_states, output); } From dovecot at dovecot.org Fri Apr 25 14:29:51 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Apr 2014 14:29:51 +0000 Subject: dovecot-2.2: Compiler warning fixes Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/767acd575d06 changeset: 17272:767acd575d06 user: Timo Sirainen date: Fri Apr 25 17:29:18 2014 +0300 description: Compiler warning fixes diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (23 lines): diff -r bb5deb884303 -r 767acd575d06 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Fri Apr 25 17:28:39 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Fri Apr 25 17:29:18 2014 +0300 @@ -112,7 +112,7 @@ static int dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer, bool final); -static void +static void ATTR_FORMAT(2, 3) imp_debug(struct dsync_mailbox_importer *importer, const char *fmt, ...) { va_list args; @@ -2264,8 +2264,8 @@ "min_first_recent_uid=%u min_highest_modseq=%llu " "min_highest_pvt_modseq=%llu", update.min_next_uid, update.min_first_recent_uid, - update.min_highest_modseq, - update.min_highest_pvt_modseq); + (unsigned long long)update.min_highest_modseq, + (unsigned long long)update.min_highest_pvt_modseq); if (mailbox_update(importer->box, &update) < 0) { i_error("Mailbox %s: Update failed: %s", From dovecot at dovecot.org Fri Apr 25 15:04:00 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Apr 2014 15:04:00 +0000 Subject: dovecot-2.2: iostream-temp: Improved stream naming. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1c7ee3ecbfb3 changeset: 17273:1c7ee3ecbfb3 user: Timo Sirainen date: Fri Apr 25 18:03:36 2014 +0300 description: iostream-temp: Improved stream naming. diffstat: src/lib/iostream-temp.c | 21 +++++++++++++++------ 1 files changed, 15 insertions(+), 6 deletions(-) diffs (64 lines): diff -r 767acd575d06 -r 1c7ee3ecbfb3 src/lib/iostream-temp.c --- a/src/lib/iostream-temp.c Fri Apr 25 17:29:18 2014 +0300 +++ b/src/lib/iostream-temp.c Fri Apr 25 18:03:36 2014 +0300 @@ -25,6 +25,7 @@ buffer_t *buf; int fd; bool fd_tried; + uoff_t fd_size; }; static void @@ -65,6 +66,7 @@ i_close_fd(&tstream->fd); return -1; } + tstream->fd_size = tstream->buf->used; buffer_free(&tstream->buf); return 0; } @@ -84,6 +86,7 @@ bytes += iov[i].iov_len; tstream->ostream.ostream.offset += iov[i].iov_len; } + tstream->fd_size += bytes; return bytes; } @@ -198,10 +201,14 @@ i_assert(stream->ostream.offset == tstream->buf->used); buffer_write(tstream->buf, offset, data, size); stream->ostream.offset = tstream->buf->used; - } else if (pwrite_full(tstream->fd, data, size, offset) < 0) { - stream->ostream.stream_errno = errno; - i_close_fd(&tstream->fd); - return -1; + } else { + if (pwrite_full(tstream->fd, data, size, offset) < 0) { + stream->ostream.stream_errno = errno; + i_close_fd(&tstream->fd); + return -1; + } + if (tstream->fd_size < offset + size) + tstream->fd_size = offset + size; } return 0; } @@ -265,13 +272,15 @@ } else if (tstream->fd != -1) { input = i_stream_create_fd(tstream->fd, max_buffer_size, TRUE); i_stream_set_name(input, t_strdup_printf( - "(Temp file in %s)", tstream->temp_path_prefix)); + "(Temp file fd %d in %s, %"PRIuUOFF_T" bytes)", + tstream->fd, tstream->temp_path_prefix, tstream->fd_size)); tstream->fd = -1; } else { input = i_stream_create_from_data(tstream->buf->data, tstream->buf->used); i_stream_set_name(input, t_strdup_printf( - "(Temp file in %s)", tstream->temp_path_prefix)); + "(Temp buffer in %s, %"PRIuSIZE_T" bytes)", + tstream->temp_path_prefix, tstream->buf->used)); i_stream_add_destroy_callback(input, iostream_temp_buf_destroyed, tstream->buf); tstream->buf = NULL; From dovecot at dovecot.org Fri Apr 25 15:56:09 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Apr 2014 15:56:09 +0000 Subject: dovecot-2.2: lib-http: Failed payload read/write error was handl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/022994e15767 changeset: 17274:022994e15767 user: Timo Sirainen date: Fri Apr 25 18:55:31 2014 +0300 description: lib-http: Failed payload read/write error was handled wrongly as "stream input size changed". diffstat: src/lib-http/http-client-request.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 1c7ee3ecbfb3 -r 022994e15767 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Fri Apr 25 18:03:36 2014 +0300 +++ b/src/lib-http/http-client-request.c Fri Apr 25 18:55:31 2014 +0300 @@ -594,8 +594,8 @@ } if (ret < 0 || i_stream_is_eof(req->payload_input)) { - if (!req->payload_chunked && - req->payload_input->v_offset - req->payload_offset != req->payload_size) { + if (ret >= 0 && !req->payload_chunked && + req->payload_input->v_offset - req->payload_offset != req->payload_size) { *error_r = t_strdup_printf("BUG: stream '%s' input size changed: " "%"PRIuUOFF_T"-%"PRIuUOFF_T" != %"PRIuUOFF_T, i_stream_get_name(req->payload_input), From dovecot at dovecot.org Fri Apr 25 16:01:38 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Apr 2014 16:01:38 +0000 Subject: dovecot-2.2: lib-http: Better error handling fix for payload rea... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/49e9d9743f6e changeset: 17275:49e9d9743f6e user: Timo Sirainen date: Fri Apr 25 19:01:09 2014 +0300 description: lib-http: Better error handling fix for payload read/write failures diffstat: src/lib-http/http-client-request.c | 13 ++++++------- 1 files changed, 6 insertions(+), 7 deletions(-) diffs (38 lines): diff -r 022994e15767 -r 49e9d9743f6e src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Fri Apr 25 18:55:31 2014 +0300 +++ b/src/lib-http/http-client-request.c Fri Apr 25 19:01:09 2014 +0300 @@ -581,20 +581,19 @@ *error_r = t_strdup_printf("read(%s) failed: %s", i_stream_get_name(req->payload_input), i_stream_get_error(req->payload_input)); - ret = -1; + return -1; } else if (output->stream_errno != 0) { /* failed to send request */ errno = output->stream_errno; *error_r = t_strdup_printf("write(%s) failed: %s", o_stream_get_name(output), o_stream_get_error(output)); - ret = -1; - } else { - i_assert(ret >= 0); + return -1; } + i_assert(ret >= 0); - if (ret < 0 || i_stream_is_eof(req->payload_input)) { - if (ret >= 0 && !req->payload_chunked && + if (i_stream_is_eof(req->payload_input)) { + if (!req->payload_chunked && req->payload_input->v_offset - req->payload_offset != req->payload_size) { *error_r = t_strdup_printf("BUG: stream '%s' input size changed: " "%"PRIuUOFF_T"-%"PRIuUOFF_T" != %"PRIuUOFF_T, @@ -622,7 +621,7 @@ conn->io_req_payload = io_add_istream(req->payload_input, http_client_request_payload_input, req); } - return ret < 0 ? -1 : 0; + return 0; } static int http_client_request_send_real(struct http_client_request *req, From pigeonhole at rename-it.nl Fri Apr 25 21:53:53 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 25 Apr 2014 23:53:53 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: Fixed sieve_result_global_log... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/f5d11f8c614c changeset: 1847:f5d11f8c614c user: Stephan Bosch date: Fri Apr 25 23:53:40 2014 +0200 description: lib-sieve: Fixed sieve_result_global_log_error to log only as i_info in administrator log (syslog) if executed from multiscript context. Previously, this would work correcly only from the user's personal script and not from multiscript (sieve_before/sieve_after). diffstat: src/lib-sieve/sieve-error.c | 58 +++++++++++++++++++++++++++----------------- 1 files changed, 36 insertions(+), 22 deletions(-) diffs (81 lines): diff -r db8f7bbda283 -r f5d11f8c614c src/lib-sieve/sieve-error.c --- a/src/lib-sieve/sieve-error.c Tue Apr 15 23:11:46 2014 +0200 +++ b/src/lib-sieve/sieve-error.c Fri Apr 25 23:53:40 2014 +0200 @@ -77,20 +77,27 @@ unsigned int flags, const char *location, const char *fmt, va_list args) { if ( (flags & SIEVE_ERROR_FLAG_GLOBAL) != 0 && - (ehandler == NULL || ehandler->parent == NULL) && - svinst->system_ehandler != ehandler && - svinst->system_ehandler->verror != NULL ) { - va_list args_copy; + (ehandler == NULL || ehandler->parent == NULL)) { + if (svinst->system_ehandler != ehandler || + (flags & SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO) != 0) { + va_list args_copy; - VA_COPY(args_copy, args); + VA_COPY(args_copy, args); - if ( (flags & SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO) != 0 ) { - svinst->system_ehandler->vinfo - (svinst->system_ehandler, 0, location, fmt, args_copy); - } else { - svinst->system_ehandler->verror - (svinst->system_ehandler, 0, location, fmt, args_copy); + if ( (flags & SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO) != 0 ) { + if (svinst->system_ehandler->vinfo != NULL ) { + svinst->system_ehandler->vinfo + (svinst->system_ehandler, 0, location, fmt, args_copy); + } + } else { + if ( svinst->system_ehandler->verror != NULL ) { + svinst->system_ehandler->verror + (svinst->system_ehandler, 0, location, fmt, args_copy); + } + } } + if (svinst->system_ehandler == ehandler) + return; } if ( ehandler == NULL ) @@ -110,20 +117,27 @@ unsigned int flags, const char *location, const char *fmt, va_list args) { if ( (flags & SIEVE_ERROR_FLAG_GLOBAL) != 0 && - (ehandler == NULL || ehandler->parent == NULL) && - svinst->system_ehandler != ehandler && - svinst->system_ehandler->vwarning != NULL ) { - va_list args_copy; + (ehandler == NULL || ehandler->parent == NULL)) { + if (svinst->system_ehandler != ehandler || + (flags & SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO) != 0) { + va_list args_copy; - VA_COPY(args_copy, args); + VA_COPY(args_copy, args); - if ( (flags & SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO) != 0 ) { - svinst->system_ehandler->vinfo - (svinst->system_ehandler, 0, location, fmt, args_copy); - } else { - svinst->system_ehandler->vwarning - (svinst->system_ehandler, 0, location, fmt, args_copy); + if ( (flags & SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO) != 0 ) { + if (svinst->system_ehandler->vinfo != NULL ) { + svinst->system_ehandler->vinfo + (svinst->system_ehandler, 0, location, fmt, args_copy); + } + } else { + if ( svinst->system_ehandler->vwarning != NULL ) { + svinst->system_ehandler->vwarning + (svinst->system_ehandler, 0, location, fmt, args_copy); + } + } } + if (svinst->system_ehandler == ehandler) + return; } if ( ehandler == NULL ) From dovecot at dovecot.org Sat Apr 26 19:41:52 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 26 Apr 2014 19:41:52 +0000 Subject: dovecot-2.2: lib-http: Fixed problem with connections idling ind... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6895214e33cf changeset: 17276:6895214e33cf user: Stephan Bosch date: Sat Apr 26 22:40:14 2014 +0300 description: lib-http: Fixed problem with connections idling indefinitely. Forgot to check for idle connections when request finished after waiting for payload to be read by application. diffstat: src/lib-http/http-client-connection.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r 49e9d9743f6e -r 6895214e33cf src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Fri Apr 25 19:01:09 2014 +0300 +++ b/src/lib-http/http-client-connection.c Sat Apr 26 22:40:14 2014 +0300 @@ -409,6 +409,10 @@ http_client_request_finish(&req); conn->pending_request = NULL; + /* room for new requests */ + if (http_client_connection_is_ready(conn)) + http_client_peer_trigger_request_handler(conn->peer); + /* input stream may have pending input. make sure input handler gets called (but don't do it directly, since we get get here somewhere from the API user's code, which we can't really know what From dovecot at dovecot.org Sun Apr 27 16:06:04 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 27 Apr 2014 16:06:04 +0000 Subject: dovecot-2.2: lib-http: http-client: Forgot to switch dns-client ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b0add701dcee changeset: 17277:b0add701dcee user: Stephan Bosch date: Sun Apr 27 19:05:35 2014 +0300 description: lib-http: http-client: Forgot to switch dns-client io_loop in http_client_request_continue_payload() in ealier change (6c3bd941e153). diffstat: src/lib-http/http-client-request.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (29 lines): diff -r 6895214e33cf -r b0add701dcee src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sat Apr 26 22:40:14 2014 +0300 +++ b/src/lib-http/http-client-request.c Sun Apr 27 19:05:35 2014 +0300 @@ -7,6 +7,7 @@ #include "array.h" #include "istream.h" #include "ostream.h" +#include "dns-lookup.h" #include "http-url.h" #include "http-date.h" #include "http-response-parser.h" @@ -484,6 +485,8 @@ i_assert(client->ioloop == NULL); client->ioloop = io_loop_create(); http_client_switch_ioloop(client); + if (client->set.dns_client != NULL) + dns_client_switch_ioloop(client->set.dns_client); while (req->state < HTTP_REQUEST_STATE_PAYLOAD_IN) { http_client_request_debug(req, "Waiting for request to finish"); @@ -502,6 +505,8 @@ io_loop_set_current(prev_ioloop); http_client_switch_ioloop(client); + if (client->set.dns_client != NULL) + dns_client_switch_ioloop(client->set.dns_client); io_loop_set_current(client->ioloop); io_loop_destroy(&client->ioloop); From dovecot at dovecot.org Sun Apr 27 16:06:04 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 27 Apr 2014 16:06:04 +0000 Subject: dovecot-2.2: lib-http: http-client: Prevented http_client_reques... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/00efbe39263f changeset: 17278:00efbe39263f user: Stephan Bosch date: Sun Apr 27 19:05:38 2014 +0300 description: lib-http: http-client: Prevented http_client_request_continue_payload() from segfaulting when the callback sets the request pointer to NULL. diffstat: src/lib-http/http-client-request.c | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diffs (18 lines): diff -r b0add701dcee -r 00efbe39263f src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sun Apr 27 19:05:35 2014 +0300 +++ b/src/lib-http/http-client-request.c Sun Apr 27 19:05:38 2014 +0300 @@ -524,7 +524,13 @@ } req->payload_wait = FALSE; - http_client_request_unref(_req); + + /* callback may have messed with our pointer, + so unref using local variable */ + http_client_request_unref(&req); + if (req == NULL) + *_req = NULL; + if (conn != NULL) http_client_connection_unref(&conn); From dovecot at dovecot.org Mon Apr 28 15:24:45 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Apr 2014 15:24:45 +0000 Subject: dovecot-2.2: dsync: If dsync was started via doveadm-server, sho... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fac4e7320676 changeset: 17279:fac4e7320676 user: Timo Sirainen date: Mon Apr 28 18:24:04 2014 +0300 description: dsync: If dsync was started via doveadm-server, show the remote client's IP address in ps output. diffstat: src/doveadm/dsync/doveadm-dsync.c | 17 ++++++++++++++--- src/doveadm/dsync/dsync-brain.c | 8 +++++++- src/doveadm/dsync/dsync-brain.h | 3 ++- 3 files changed, 23 insertions(+), 5 deletions(-) diffs (112 lines): diff -r 00efbe39263f -r fac4e7320676 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Sun Apr 27 19:05:38 2014 +0300 +++ b/src/doveadm/dsync/doveadm-dsync.c Mon Apr 28 18:24:04 2014 +0300 @@ -369,7 +369,7 @@ return -1; } - brain2 = dsync_brain_slave_init(user2, ibc2, TRUE); + brain2 = dsync_brain_slave_init(user2, ibc2, TRUE, ""); mail_user_unref(&user2); brain1_running = brain2_running = TRUE; @@ -532,6 +532,11 @@ int status = 0, ret = 0; memset(&set, 0, sizeof(set)); + if (_ctx->cur_client_ip.family != 0) { + /* include the doveadm client's IP address in the ps output */ + set.process_title_prefix = t_strdup_printf( + "%s ", net_ip2addr(&_ctx->cur_client_ip)); + } set.sync_box = ctx->mailbox; memcpy(set.sync_box_guid, ctx->mailbox_guid, sizeof(set.sync_box_guid)); set.lock_timeout_secs = ctx->lock_timeout; @@ -1002,7 +1007,7 @@ struct dsync_brain *brain; string_t *temp_prefix, *state_str = NULL; enum dsync_brain_sync_type sync_type; - const char *name; + const char *name, *process_title_prefix = ""; if (_ctx->conn != NULL) { /* doveadm-server connection. start with a success reply. @@ -1013,6 +1018,12 @@ o_stream_nsend(ctx->output, "\n+\n", 3); i_set_failure_prefix("dsync-server(%s): ", user->username); name = i_stream_get_name(ctx->input); + + if (_ctx->cur_client_ip.family != 0) { + /* include the doveadm client's IP address in the ps output */ + process_title_prefix = t_strdup_printf( + "%s ", net_ip2addr(&_ctx->cur_client_ip)); + } } else { /* the log messages go via stderr to the remote dsync, so the names are reversed */ @@ -1026,7 +1037,7 @@ mail_user_set_get_temp_prefix(temp_prefix, user->set); ibc = cmd_dsync_icb_stream_init(ctx, name, str_c(temp_prefix)); - brain = dsync_brain_slave_init(user, ibc, FALSE); + brain = dsync_brain_slave_init(user, ibc, FALSE, process_title_prefix); io_loop_run(current_ioloop); diff -r 00efbe39263f -r fac4e7320676 src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Sun Apr 27 19:05:38 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain.c Mon Apr 28 18:24:04 2014 +0300 @@ -41,6 +41,8 @@ const char *import_title, *export_title; str_append_c(str, '['); + if (brain->process_title_prefix != NULL) + str_append(str, brain->process_title_prefix); str_append(str, brain->user->username); if (brain->box == NULL) { str_append_c(str, ' '); @@ -155,6 +157,8 @@ i_assert(N_ELEMENTS(dsync_state_names) == DSYNC_STATE_DONE+1); brain = dsync_brain_common_init(user, ibc); + brain->process_title_prefix = + p_strdup(brain->pool, set->process_title_prefix); brain->sync_type = sync_type; if (array_count(&set->sync_namespaces) > 0) { sync_ns_str = t_str_new(128); @@ -219,12 +223,14 @@ struct dsync_brain * dsync_brain_slave_init(struct mail_user *user, struct dsync_ibc *ibc, - bool local) + bool local, const char *process_title_prefix) { struct dsync_ibc_settings ibc_set; struct dsync_brain *brain; brain = dsync_brain_common_init(user, ibc); + brain->process_title_prefix = + p_strdup(brain->pool, process_title_prefix); brain->state = DSYNC_STATE_SLAVE_RECV_HANDSHAKE; if (local) { diff -r 00efbe39263f -r fac4e7320676 src/doveadm/dsync/dsync-brain.h --- a/src/doveadm/dsync/dsync-brain.h Sun Apr 27 19:05:38 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain.h Mon Apr 28 18:24:04 2014 +0300 @@ -38,6 +38,7 @@ }; struct dsync_brain_settings { + const char *process_title_prefix; /* Sync only these namespaces */ ARRAY(struct mail_namespace *) sync_namespaces; /* Sync only this mailbox name */ @@ -61,7 +62,7 @@ const struct dsync_brain_settings *set); struct dsync_brain * dsync_brain_slave_init(struct mail_user *user, struct dsync_ibc *ibc, - bool local); + bool local, const char *process_title_prefix); /* Returns 0 if everything was successful, -1 if syncing failed in some way */ int dsync_brain_deinit(struct dsync_brain **brain); From dovecot at dovecot.org Mon Apr 28 15:34:52 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Apr 2014 15:34:52 +0000 Subject: dovecot-2.2: doveadm replicator add command added. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b912147d41c8 changeset: 17280:b912147d41c8 user: Timo Sirainen date: Mon Apr 28 18:34:27 2014 +0300 description: doveadm replicator add command added. This allows quickly adding users from userdb that don't exist in replicator. For example "doveadm replicator add '*'" adds all the new users from userdb. This isn't really necessary though. Users will be automatically added as soon as their mailboxes change in some way and replicator gets notified about them. diffstat: src/doveadm/doveadm-replicator.c | 30 ++++++++++++++++++ src/replication/replicator/Makefile.am | 1 + src/replication/replicator/doveadm-connection.c | 28 +++++++++++++++++ src/replication/replicator/replicator-queue-auth.c | 36 ++++++++++++++++++++++ src/replication/replicator/replicator-queue.h | 4 ++ src/replication/replicator/replicator.c | 25 +-------------- 6 files changed, 101 insertions(+), 23 deletions(-) diffs (205 lines): diff -r fac4e7320676 -r b912147d41c8 src/doveadm/doveadm-replicator.c --- a/src/doveadm/doveadm-replicator.c Mon Apr 28 18:24:04 2014 +0300 +++ b/src/doveadm/doveadm-replicator.c Mon Apr 28 18:34:27 2014 +0300 @@ -258,6 +258,34 @@ replicator_disconnect(ctx); } +static void cmd_replicator_add(int argc, char *argv[]) +{ + struct replicator_context *ctx; + string_t *str; + const char *line; + + if (argv[1] == NULL) + replicator_cmd_help(cmd_replicator_add); + + ctx = cmd_replicator_init(&argc, &argv, "a:", cmd_replicator_add); + + str = t_str_new(128); + str_append(str, "ADD\t"); + str_append_tabescaped(str, argv[1]); + str_append_c(str, '\n'); + replicator_send(ctx, str_c(str)); + + line = i_stream_read_next_line(ctx->input); + if (line == NULL) { + i_error("Replicator disconnected unexpectedly"); + doveadm_exit_code = EX_TEMPFAIL; + } else if (line[0] != '+') { + i_error("Replicator failed: %s", line+1); + doveadm_exit_code = EX_USAGE; + } + replicator_disconnect(ctx); +} + static void cmd_replicator_remove(int argc, char *argv[]) { struct replicator_context *ctx; @@ -293,6 +321,8 @@ "[-a ]" }, { cmd_replicator_replicate, "replicator replicate", "[-a ] [-f] [-p ] " }, + { cmd_replicator_add, "replicator add", + "[-a ] " }, { cmd_replicator_remove, "replicator remove", "[-a ] " }, }; diff -r fac4e7320676 -r b912147d41c8 src/replication/replicator/Makefile.am --- a/src/replication/replicator/Makefile.am Mon Apr 28 18:24:04 2014 +0300 +++ b/src/replication/replicator/Makefile.am Mon Apr 28 18:34:27 2014 +0300 @@ -20,6 +20,7 @@ replicator.c \ replicator-brain.c \ replicator-queue.c \ + replicator-queue-auth.c \ replicator-settings.c \ notify-connection.c diff -r fac4e7320676 -r b912147d41c8 src/replication/replicator/doveadm-connection.c --- a/src/replication/replicator/doveadm-connection.c Mon Apr 28 18:24:04 2014 +0300 +++ b/src/replication/replicator/doveadm-connection.c Mon Apr 28 18:34:27 2014 +0300 @@ -10,6 +10,7 @@ #include "master-service.h" #include "replicator-brain.h" #include "replicator-queue.h" +#include "replicator-settings.h" #include "dsync-client.h" #include "doveadm-connection.h" @@ -194,6 +195,31 @@ } static int +client_input_add(struct doveadm_connection *client, const char *const *args) +{ + struct replicator_queue *queue = + replicator_brain_get_queue(client->brain); + const struct replicator_settings *set = + replicator_brain_get_settings(client->brain); + + /* */ + if (str_array_length(args) != 1) { + i_error("%s: ADD: Invalid parameters", client->conn.name); + return -1; + } + + if (strchr(args[0], '*') == NULL && strchr(args[0], '?') == NULL) { + (void)replicator_queue_add(queue, args[0], + REPLICATION_PRIORITY_NONE); + } else { + replicator_queue_add_auth_users(queue, set->auth_socket_path, + args[0], ioloop_time); + } + o_stream_send_str(client->conn.output, "+\n"); + return 0; +} + +static int client_input_remove(struct doveadm_connection *client, const char *const *args) { struct replicator_queue *queue = @@ -259,6 +285,8 @@ return client_input_status_dsyncs(client); else if (strcmp(cmd, "REPLICATE") == 0) return client_input_replicate(client, args); + else if (strcmp(cmd, "ADD") == 0) + return client_input_add(client, args); else if (strcmp(cmd, "REMOVE") == 0) return client_input_remove(client, args); else if (strcmp(cmd, "NOTIFY") == 0) diff -r fac4e7320676 -r b912147d41c8 src/replication/replicator/replicator-queue-auth.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/replicator/replicator-queue-auth.c Mon Apr 28 18:34:27 2014 +0300 @@ -0,0 +1,36 @@ +/* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "auth-master.h" +#include "replicator-queue.h" + +#define REPLICATOR_AUTH_SERVICE_NAME "replicator" + +void replicator_queue_add_auth_users(struct replicator_queue *queue, + const char *auth_socket_path, + const char *usermask, time_t last_update) +{ + struct auth_master_connection *auth_conn; + struct auth_master_user_list_ctx *ctx; + struct auth_user_info user_info; + struct replicator_user *user; + const char *username; + + auth_conn = auth_master_init(auth_socket_path, + AUTH_MASTER_FLAG_NO_IDLE_TIMEOUT); + + memset(&user_info, 0, sizeof(user_info)); + user_info.service = REPLICATOR_AUTH_SERVICE_NAME; + + /* add all users into replication queue, so that we can start doing + full syncs for everyone whose state can't be found */ + ctx = auth_master_user_list_init(auth_conn, usermask, &user_info); + while ((username = auth_master_user_list_next(ctx)) != NULL) { + user = replicator_queue_add(queue, username, + REPLICATION_PRIORITY_NONE); + user->last_update = last_update; + } + if (auth_master_user_list_deinit(&ctx) < 0) + i_error("listing users failed, can't replicate existing data"); + auth_master_deinit(&auth_conn); +} diff -r fac4e7320676 -r b912147d41c8 src/replication/replicator/replicator-queue.h --- a/src/replication/replicator/replicator-queue.h Mon Apr 28 18:24:04 2014 +0300 +++ b/src/replication/replicator/replicator-queue.h Mon Apr 28 18:34:27 2014 +0300 @@ -78,4 +78,8 @@ replicator_queue_iter_next(struct replicator_queue_iter *iter); void replicator_queue_iter_deinit(struct replicator_queue_iter **iter); +void replicator_queue_add_auth_users(struct replicator_queue *queue, + const char *auth_socket_path, + const char *usermask, time_t last_update); + #endif diff -r fac4e7320676 -r b912147d41c8 src/replication/replicator/replicator.c --- a/src/replication/replicator/replicator.c Mon Apr 28 18:24:04 2014 +0300 +++ b/src/replication/replicator/replicator.c Mon Apr 28 18:34:27 2014 +0300 @@ -12,7 +12,6 @@ #include "replicator-queue.h" #include "replicator-settings.h" -#define REPLICATOR_AUTH_SERVICE_NAME "replicator" #define REPLICATOR_DB_DUMP_INTERVAL_MSECS (1000*60*15) /* if syncing fails, try again in 5 minutes */ #define REPLICATOR_FAILURE_RESYNC_INTERVAL_SECS (60*5) @@ -35,29 +34,9 @@ static void replication_add_users(struct replicator_queue *queue) { - struct auth_master_connection *auth_conn; - struct auth_master_user_list_ctx *ctx; - struct auth_user_info user_info; - struct replicator_user *user; - const char *path, *username; + const char *path; - auth_conn = auth_master_init(set->auth_socket_path, - AUTH_MASTER_FLAG_NO_IDLE_TIMEOUT); - - memset(&user_info, 0, sizeof(user_info)); - user_info.service = REPLICATOR_AUTH_SERVICE_NAME; - - /* add all users into replication queue, so that we can start doing - full syncs for everyone whose state can't be found */ - ctx = auth_master_user_list_init(auth_conn, "", &user_info); - while ((username = auth_master_user_list_next(ctx)) != NULL) { - user = replicator_queue_add(queue, username, - REPLICATION_PRIORITY_NONE); - user->last_update = 0; - } - if (auth_master_user_list_deinit(&ctx) < 0) - i_error("listing users failed, can't replicate existing data"); - auth_master_deinit(&auth_conn); + replicator_queue_add_auth_users(queue, set->auth_socket_path, "*", 0); /* add updates from replicator db, if it exists */ path = t_strconcat(service_set->state_dir, "/"REPLICATOR_DB_FNAME, NULL); From dovecot at dovecot.org Mon Apr 28 16:12:25 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Apr 2014 16:12:25 +0000 Subject: dovecot-2.2: dsync: Finished previous change to actually compile Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e4f527711ec9 changeset: 17281:e4f527711ec9 user: Timo Sirainen date: Mon Apr 28 19:12:01 2014 +0300 description: dsync: Finished previous change to actually compile diffstat: src/doveadm/dsync/dsync-brain-private.h | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r b912147d41c8 -r e4f527711ec9 src/doveadm/dsync/dsync-brain-private.h --- a/src/doveadm/dsync/dsync-brain-private.h Mon Apr 28 18:34:27 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-private.h Mon Apr 28 19:12:01 2014 +0300 @@ -49,6 +49,7 @@ pool_t pool; struct mail_user *user; struct dsync_ibc *ibc; + const char *process_title_prefix; ARRAY(struct mail_namespace *) sync_namespaces; const char *sync_box; guid_128_t sync_box_guid; From dovecot at dovecot.org Mon Apr 28 17:11:22 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Apr 2014 17:11:22 +0000 Subject: dovecot-2.2: dsync: Include messages_count in the mailbox states. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9669c9a8984f changeset: 17282:9669c9a8984f user: Timo Sirainen date: Mon Apr 28 20:10:56 2014 +0300 description: dsync: Include messages_count in the mailbox states. This allows detecting that stateful dsync can't be done when message count is suddenly wrong. diffstat: src/doveadm/dsync/dsync-brain-mailbox.c | 20 +++++++++---- src/doveadm/dsync/dsync-brain-mails.c | 3 ++ src/doveadm/dsync/dsync-brain.c | 3 +- src/doveadm/dsync/dsync-ibc-stream.c | 9 +++++- src/doveadm/dsync/dsync-mailbox-import.c | 4 ++ src/doveadm/dsync/dsync-mailbox-import.h | 1 + src/doveadm/dsync/dsync-mailbox-state.c | 46 +++++++++++++++++++++++++++---- src/doveadm/dsync/dsync-mailbox-state.h | 1 + 8 files changed, 72 insertions(+), 15 deletions(-) diffs (287 lines): diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Mon Apr 28 19:12:01 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Mon Apr 28 20:10:56 2014 +0300 @@ -248,7 +248,7 @@ (void)dsync_mailbox_export_deinit(&brain->box_exporter, &error); } if (brain->box_importer != NULL) { - uint32_t last_common_uid; + uint32_t last_common_uid, last_messages_count; uint64_t last_common_modseq, last_common_pvt_modseq; bool changes_during_sync; @@ -258,6 +258,7 @@ &last_common_uid, &last_common_modseq, &last_common_pvt_modseq, + &last_messages_count, &changes_during_sync); } if (brain->log_scan != NULL) @@ -329,7 +330,8 @@ state->last_uidvalidity != dsync_box->uid_validity || state->last_common_uid+1 != dsync_box->uid_next || state->last_common_modseq != dsync_box->highest_modseq || - state->last_common_pvt_modseq != dsync_box->highest_pvt_modseq; + state->last_common_pvt_modseq != dsync_box->highest_pvt_modseq || + state->last_messages_count != dsync_box->messages_count; } static int @@ -376,13 +378,14 @@ if (!dsync_brain_has_mailbox_state_changed(brain, &dsync_box)) { if (brain->debug) { i_debug("brain %c: Skipping mailbox %s with unchanged state " - "uidvalidity=%u uidnext=%u highestmodseq=%llu highestpvtmodseq=%llu", + "uidvalidity=%u uidnext=%u highestmodseq=%llu highestpvtmodseq=%llu messages=%u", brain->master_brain ? 'M' : 'S', guid_128_to_string(dsync_box.mailbox_guid), dsync_box.uid_validity, dsync_box.uid_next, (unsigned long long)dsync_box.highest_modseq, - (unsigned long long)dsync_box.highest_pvt_modseq); + (unsigned long long)dsync_box.highest_pvt_modseq, + dsync_box.messages_count); } mailbox_free(&box); return 0; @@ -681,7 +684,8 @@ sizeof(dsync_box->mailbox_guid)) == 0); dsync_ibc_send_mailbox(brain->ibc, &local_dsync_box); - dsync_brain_mailbox_update_pre(brain, box, &local_dsync_box, dsync_box); + resync = !dsync_brain_mailbox_update_pre(brain, box, &local_dsync_box, + dsync_box); if (!dsync_boxes_need_sync(brain, &local_dsync_box, dsync_box)) { /* no fields appear to have changed, skip this mailbox */ @@ -696,8 +700,12 @@ /* start export/import */ dsync_brain_sync_mailbox_init(brain, box, &local_dsync_box, FALSE); - if (dsync_brain_sync_mailbox_open(brain, dsync_box) < 0) + if ((ret = dsync_brain_sync_mailbox_open(brain, dsync_box)) < 0) return TRUE; + if (ret == 0 || resync) { + brain->changes_during_sync = TRUE; + brain->require_full_resync = TRUE; + } brain->state = DSYNC_STATE_SYNC_MAILS; return TRUE; diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-brain-mails.c --- a/src/doveadm/dsync/dsync-brain-mails.c Mon Apr 28 19:12:01 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-mails.c Mon Apr 28 20:10:56 2014 +0300 @@ -200,11 +200,14 @@ brain->local_dsync_box.highest_modseq; state.last_common_pvt_modseq = brain->local_dsync_box.highest_pvt_modseq; + state.last_messages_count = + brain->local_dsync_box.messages_count; } else { if (dsync_mailbox_import_deinit(&brain->box_importer, TRUE, &state.last_common_uid, &state.last_common_modseq, &state.last_common_pvt_modseq, + &state.last_messages_count, &state.changes_during_sync) < 0) { brain->failed = TRUE; return; diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Mon Apr 28 19:12:01 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain.c Mon Apr 28 20:10:56 2014 +0300 @@ -620,13 +620,14 @@ iter = hash_table_iterate_init(brain->mailbox_states); while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) { - i_debug("brain %c: Mailbox %s state: uidvalidity=%u uid=%u modseq=%llu pvt_modseq=%llu changes_during_sync=%d", + i_debug("brain %c: Mailbox %s state: uidvalidity=%u uid=%u modseq=%llu pvt_modseq=%llu messages=%u changes_during_sync=%d", brain->master_brain ? 'M' : 'S', guid_128_to_string(guid), state->last_uidvalidity, state->last_common_uid, (unsigned long long)state->last_common_modseq, (unsigned long long)state->last_common_pvt_modseq, + state->last_messages_count, state->changes_during_sync); } hash_table_iterate_deinit(&iter); diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Mon Apr 28 19:12:01 2014 +0300 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Mon Apr 28 20:10:56 2014 +0300 @@ -80,7 +80,7 @@ { .name = "mailbox_state", .chr = 'S', .required_keys = "mailbox_guid last_uidvalidity last_common_uid " - "last_common_modseq last_common_pvt_modseq", + "last_common_modseq last_common_pvt_modseq last_messages_count", .optional_keys = "changes_during_sync" }, { .name = "mailbox_tree_node", @@ -801,6 +801,8 @@ dec2str(state->last_common_modseq)); dsync_serializer_encode_add(encoder, "last_common_pvt_modseq", dec2str(state->last_common_pvt_modseq)); + dsync_serializer_encode_add(encoder, "last_messages_count", + dec2str(state->last_messages_count)); if (state->changes_during_sync) dsync_serializer_encode_add(encoder, "changes_during_sync", ""); @@ -848,6 +850,11 @@ dsync_ibc_input_error(ibc, decoder, "Invalid last_common_pvt_modseq"); return DSYNC_IBC_RECV_RET_TRYAGAIN; } + value = dsync_deserializer_decode_get(decoder, "last_messages_count"); + if (str_to_uint32(value, &state_r->last_messages_count) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_messages_count"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } if (dsync_deserializer_decode_try(decoder, "changes_during_sync", &value)) state_r->changes_during_sync = TRUE; return DSYNC_IBC_RECV_RET_OK; diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Mon Apr 28 19:12:01 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Apr 28 20:10:56 2014 +0300 @@ -2340,9 +2340,11 @@ uint32_t *last_common_uid_r, uint64_t *last_common_modseq_r, uint64_t *last_common_pvt_modseq_r, + uint32_t *last_messages_count_r, bool *changes_during_sync_r) { struct dsync_mailbox_importer *importer = *_importer; + struct mailbox_status status; int ret; *_importer = NULL; @@ -2392,6 +2394,8 @@ *last_common_modseq_r = importer->local_initial_highestmodseq; *last_common_pvt_modseq_r = importer->local_initial_highestpvtmodseq; } + mailbox_get_open_status(importer->box, STATUS_MESSAGES, &status); + *last_messages_count_r = status.messages; ret = importer->failed ? -1 : 0; pool_unref(&importer->pool); diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-mailbox-import.h --- a/src/doveadm/dsync/dsync-mailbox-import.h Mon Apr 28 19:12:01 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.h Mon Apr 28 20:10:56 2014 +0300 @@ -41,6 +41,7 @@ uint32_t *last_common_uid_r, uint64_t *last_common_modseq_r, uint64_t *last_common_pvt_modseq_r, + uint32_t *last_messages_count_r, bool *changes_during_sync_r); const char *dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer *importer); diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-mailbox-state.c --- a/src/doveadm/dsync/dsync-mailbox-state.c Mon Apr 28 19:12:01 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-state.c Mon Apr 28 20:10:56 2014 +0300 @@ -7,7 +7,11 @@ #include "hash.h" #include "dsync-mailbox-state.h" -#define MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8) +#define DSYNC_STATE_MAJOR_VERSION 1 +#define DSYNC_STATE_MINOR_VERSION 0 + +#define V0_MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8) +#define MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8 + 4) static void put_uint32(buffer_t *output, uint32_t num) { @@ -32,6 +36,11 @@ buffer_t *buf = buffer_create_dynamic(pool_datastack_create(), 128); uint32_t crc = 0; + buffer_append_c(buf, DSYNC_STATE_MAJOR_VERSION); + buffer_append_c(buf, DSYNC_STATE_MINOR_VERSION); + buffer_append_c(buf, '\0'); + buffer_append_c(buf, '\0'); + iter = hash_table_iterate_init(states); while (hash_table_iterate(iter, states, &guid, &state)) { buffer_append(buf, state->mailbox_guid, @@ -42,6 +51,7 @@ put_uint32(buf, state->last_common_modseq >> 32); put_uint32(buf, state->last_common_pvt_modseq & 0xffffffffU); put_uint32(buf, state->last_common_pvt_modseq >> 32); + put_uint32(buf, state->last_messages_count); /* v1 */ if (buf->used % 3 == 0) { crc = crc32_data_more(crc, buf->data, buf->used); base64_encode(buf->data, buf->used, output); @@ -55,6 +65,19 @@ base64_encode(buf->data, buf->used, output); } +static int dsync_mailbox_states_retry_import_v0(const buffer_t *buf) +{ + const unsigned char *data = buf->data; + + /* v0 had no version header and no last_messages_count */ + + if ((buf->used-4) % V0_MAILBOX_SIZE != 0 || + get_uint32(data + buf->used-4) != crc32_data(data, buf->used-4)) + return -1; + /* looks like valid v0 format, silently treat it as empty state */ + return 0; +} + int dsync_mailbox_states_import(HASH_TABLE_TYPE(dsync_mailbox_state) states, pool_t pool, const char *input, const char **error_r) @@ -71,21 +94,29 @@ *error_r = "Invalid base64 data"; return -1; } - if (buf->used < 4) { + /* v1: 4 byte header, mailboxes[], CRC32 */ + data = buf->data; + + if (buf->used == 4 && get_uint32(data) == 0) { + /* v0: Empty state */ + return 0; + } + if (buf->used < 8) { *error_r = "Input too small"; return -1; } - if ((buf->used-4) % MAILBOX_SIZE != 0) { + + if ((buf->used-8) % MAILBOX_SIZE != 0) { *error_r = "Invalid input size"; - return -1; + return dsync_mailbox_states_retry_import_v0(buf); } - data = buf->data; - count = (buf->used-4) / MAILBOX_SIZE; if (get_uint32(data + buf->used-4) != crc32_data(data, buf->used-4)) { *error_r = "CRC32 mismatch"; - return -1; + return dsync_mailbox_states_retry_import_v0(buf); } + data += 4; + count = (buf->used-8) / MAILBOX_SIZE; for (i = 0; i < count; i++, data += MAILBOX_SIZE) { state = p_new(pool, struct dsync_mailbox_state, 1); @@ -98,6 +129,7 @@ state->last_common_pvt_modseq = get_uint32(data + GUID_128_SIZE + 16) | (uint64_t)get_uint32(data + GUID_128_SIZE + 20) << 32; + state->last_messages_count = get_uint32(data + GUID_128_SIZE + 24); guid_p = state->mailbox_guid; hash_table_insert(states, guid_p, state); } diff -r e4f527711ec9 -r 9669c9a8984f src/doveadm/dsync/dsync-mailbox-state.h --- a/src/doveadm/dsync/dsync-mailbox-state.h Mon Apr 28 19:12:01 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-state.h Mon Apr 28 20:10:56 2014 +0300 @@ -9,6 +9,7 @@ uint32_t last_common_uid; uint64_t last_common_modseq; uint64_t last_common_pvt_modseq; + uint32_t last_messages_count; bool changes_during_sync; }; ARRAY_DEFINE_TYPE(dsync_mailbox_state, struct dsync_mailbox_state); From dovecot at dovecot.org Mon Apr 28 17:14:34 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Apr 2014 17:14:34 +0000 Subject: dovecot-2.2: dsync: If incremental dsync finds that its state is... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0d34a90b2760 changeset: 17283:0d34a90b2760 user: Timo Sirainen date: Mon Apr 28 20:14:03 2014 +0300 description: dsync: If incremental dsync finds that its state is stale, retry (later) with full sync. This works only when the master itself notices such changes locally. Remote changes aren't noticed, because the master doesn't send the state to remote dsync and the master is also the one that decides which mailboxes are synced. The fix to this probably means sending the state string to remote dsync and have it check if the state is still valid (= protocol change). diffstat: src/doveadm/dsync/dsync-brain-mailbox.c | 52 +++++++++++++++++++++----- src/doveadm/dsync/dsync-brain-mails.c | 14 +++++- src/doveadm/dsync/dsync-brain-private.h | 3 +- src/doveadm/dsync/dsync-transaction-log-scan.c | 22 ++++++++--- 4 files changed, 71 insertions(+), 20 deletions(-) diffs (261 lines): diff -r 9669c9a8984f -r 0d34a90b2760 src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Mon Apr 28 20:10:56 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Mon Apr 28 20:14:03 2014 +0300 @@ -196,9 +196,11 @@ int dsync_brain_sync_mailbox_open(struct dsync_brain *brain, const struct dsync_mailbox *remote_dsync_box) { + struct mailbox_status status; enum dsync_mailbox_exporter_flags exporter_flags = 0; uint32_t last_common_uid, highest_wanted_uid; uint64_t last_common_modseq, last_common_pvt_modseq; + int ret; i_assert(brain->log_scan == NULL); i_assert(brain->box_exporter == NULL); @@ -208,18 +210,33 @@ last_common_pvt_modseq = brain->mailbox_state.last_common_pvt_modseq; highest_wanted_uid = last_common_uid == 0 ? (uint32_t)-1 : last_common_uid; - if (dsync_transaction_log_scan_init(brain->box->view, - brain->box->view_pvt, - highest_wanted_uid, - last_common_modseq, - last_common_pvt_modseq, - &brain->log_scan) < 0) { + ret = dsync_transaction_log_scan_init(brain->box->view, + brain->box->view_pvt, + highest_wanted_uid, + last_common_modseq, + last_common_pvt_modseq, + &brain->log_scan); + if (ret < 0) { i_error("Failed to read transaction log for mailbox %s", mailbox_get_vname(brain->box)); brain->failed = TRUE; return -1; } + if (last_common_uid != 0) { + mailbox_get_open_status(brain->box, STATUS_UIDNEXT | + STATUS_HIGHESTMODSEQ | + STATUS_HIGHESTPVTMODSEQ, &status); + if (status.uidnext < last_common_uid || + status.highest_modseq < last_common_modseq || + status.highest_pvt_modseq < last_common_pvt_modseq) { + /* last_common_* is higher than our current ones. + incremental sync state is stale, we need to do + a full resync */ + ret = 0; + } + } + if (!brain->mail_requests) exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS; if (remote_dsync_box->have_save_guids && @@ -232,13 +249,25 @@ last_common_uid, exporter_flags); dsync_brain_sync_mailbox_init_remote(brain, remote_dsync_box); - return 0; + if (ret == 0) { + i_warning("Failed to do incremental sync for mailbox %s, " + "retry with a full sync", + mailbox_get_vname(brain->box)); + brain->changes_during_sync = TRUE; + brain->require_full_resync = TRUE; + return 0; + } + return 1; } void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain) { i_assert(brain->box != NULL); + if (brain->require_full_resync) { + brain->mailbox_state.last_uidvalidity = 0; + brain->require_full_resync = FALSE; + } array_append(&brain->remote_mailbox_states, &brain->mailbox_state, 1); if (brain->box_exporter != NULL) { const char *error; @@ -544,13 +573,14 @@ } } -void dsync_brain_mailbox_update_pre(struct dsync_brain *brain, +bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain, struct mailbox *box, const struct dsync_mailbox *local_box, const struct dsync_mailbox *remote_box) { struct mailbox_update update; const struct dsync_mailbox_state *state; + bool ret = TRUE; memset(&update, 0, sizeof(update)); @@ -568,7 +598,7 @@ session, because the other side already started sending mailbox changes, but not for all mails. */ dsync_mailbox_state_remove(brain, local_box->mailbox_guid); - // FIXME: handle this properly + ret = FALSE; } } @@ -577,7 +607,7 @@ if (update.uid_validity == 0 && update.cache_updates == NULL) { /* no changes */ - return; + return ret; } if (mailbox_update(box, &update) < 0) { @@ -586,6 +616,7 @@ mailbox_get_last_error(box, NULL)); brain->failed = TRUE; } + return ret; } static void @@ -614,6 +645,7 @@ struct mailbox *box; const char *error; int ret; + bool resync; i_assert(!brain->master_brain); i_assert(brain->box == NULL); diff -r 9669c9a8984f -r 0d34a90b2760 src/doveadm/dsync/dsync-brain-mails.c --- a/src/doveadm/dsync/dsync-brain-mails.c Mon Apr 28 20:10:56 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-mails.c Mon Apr 28 20:14:03 2014 +0300 @@ -22,6 +22,7 @@ { const struct dsync_mailbox *dsync_box; enum dsync_ibc_recv_ret ret; + bool resync; i_assert(brain->master_brain); @@ -42,19 +43,26 @@ if (dsync_box->mailbox_lost) { /* remote lost the mailbox. it's probably already deleted, but verify it on next sync just to be sure */ + brain->changes_during_sync = TRUE; + brain->require_full_resync = TRUE; dsync_brain_sync_mailbox_deinit(brain); return TRUE; } - dsync_brain_mailbox_update_pre(brain, brain->box, - &brain->local_dsync_box, dsync_box); + resync = !dsync_brain_mailbox_update_pre(brain, brain->box, + &brain->local_dsync_box, + dsync_box); if (!dsync_boxes_need_sync(brain, &brain->local_dsync_box, dsync_box)) { /* no fields appear to have changed, skip this mailbox */ dsync_brain_sync_mailbox_deinit(brain); return TRUE; } - if (dsync_brain_sync_mailbox_open(brain, dsync_box) < 0) + if ((ret = dsync_brain_sync_mailbox_open(brain, dsync_box)) < 0) return TRUE; + if (ret == 0 || resync) { + brain->changes_during_sync = TRUE; + brain->require_full_resync = TRUE; + } dsync_brain_sync_init_box_states(brain); return TRUE; } diff -r 9669c9a8984f -r 0d34a90b2760 src/doveadm/dsync/dsync-brain-private.h --- a/src/doveadm/dsync/dsync-brain-private.h Mon Apr 28 20:10:56 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-private.h Mon Apr 28 20:14:03 2014 +0300 @@ -99,6 +99,7 @@ unsigned int no_mail_sync:1; unsigned int no_backup_overwrite:1; unsigned int changes_during_sync:1; + unsigned int require_full_resync:1; unsigned int verbose_proctitle:1; unsigned int failed:1; }; @@ -116,7 +117,7 @@ void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain); int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid, struct mailbox **box_r, const char **error_r); -void dsync_brain_mailbox_update_pre(struct dsync_brain *brain, +bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain, struct mailbox *box, const struct dsync_mailbox *local_box, const struct dsync_mailbox *remote_box); diff -r 9669c9a8984f -r 0d34a90b2760 src/doveadm/dsync/dsync-transaction-log-scan.c --- a/src/doveadm/dsync/dsync-transaction-log-scan.c Mon Apr 28 20:10:56 2014 +0300 +++ b/src/doveadm/dsync/dsync-transaction-log-scan.c Mon Apr 28 20:14:03 2014 +0300 @@ -399,7 +399,13 @@ view->index->filepath, log_seq, end_seq); ret = -1; } - return ret < 0 ? -1 : 0; + if (ret < 0) + return -1; + if (modseq != 0) { + /* we didn't see all the changes that we wanted to */ + return 0; + } + return 1; } static int @@ -412,9 +418,10 @@ uint32_t file_seq, max_seq; uoff_t file_offset, max_offset; uint64_t cur_modseq; + int ret; log_view = mail_transaction_log_view_open(view->index->log); - if (dsync_log_set(ctx, view, pvt_scan, log_view, modseq) < 0) { + if ((ret = dsync_log_set(ctx, view, pvt_scan, log_view, modseq)) < 0) { mail_transaction_log_view_close(&log_view); return -1; } @@ -475,7 +482,7 @@ ctx->last_log_offset = file_offset; } mail_transaction_log_view_close(&log_view); - return 0; + return ret; } static int @@ -503,6 +510,7 @@ { struct dsync_transaction_log_scan *ctx; pool_t pool; + int ret, ret2; pool = pool_alloconly_create(MEMPOOL_GROWING"dsync transaction log scan", 10240); @@ -515,15 +523,17 @@ ctx->view = view; ctx->highest_wanted_uid = highest_wanted_uid; - if (dsync_log_scan(ctx, view, modseq, FALSE) < 0) + if ((ret = dsync_log_scan(ctx, view, modseq, FALSE)) < 0) return -1; if (pvt_view != NULL) { - if (dsync_log_scan(ctx, pvt_view, pvt_modseq, TRUE) < 0) + if ((ret2 = dsync_log_scan(ctx, pvt_view, pvt_modseq, TRUE)) < 0) return -1; + if (ret2 == 0) + ret = 0; } *scan_r = ctx; - return 0; + return ret; } HASH_TABLE_TYPE(dsync_uid_mail_change) From pigeonhole at rename-it.nl Mon Apr 28 20:09:01 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 28 Apr 2014 22:09:01 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: Switched to new Dovecot SMTP ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/c6c320f4918b changeset: 1848:c6c320f4918b user: Stephan Bosch date: Mon Apr 28 22:08:47 2014 +0200 description: lib-sieve: Switched to new Dovecot SMTP API. diffstat: src/lib-sieve/cmd-redirect.c | 20 +++++++++++++---- src/lib-sieve/plugins/enotify/mailto/ntfy-mailto.c | 24 ++++++++++++++------- src/lib-sieve/plugins/enotify/sieve-ext-enotify.h | 3 ++ src/lib-sieve/plugins/notify/cmd-notify.c | 24 ++++++++++++++------- src/lib-sieve/plugins/vacation/cmd-vacation.c | 17 ++++++++++----- src/lib-sieve/sieve-actions.c | 21 ++++++++++++------ src/lib-sieve/sieve-smtp.c | 19 ++++++++--------- src/lib-sieve/sieve-smtp.h | 9 ++++--- src/lib-sieve/sieve-types.h | 9 +++++-- src/plugins/lda-sieve/lda-sieve-plugin.c | 9 ++++--- src/sieve-tools/sieve-test.c | 7 +++-- src/testsuite/testsuite-smtp.c | 7 +++-- src/testsuite/testsuite-smtp.h | 4 +- 13 files changed, 110 insertions(+), 63 deletions(-) diffs (truncated from 379 to 300 lines): diff -r f5d11f8c614c -r c6c320f4918b src/lib-sieve/cmd-redirect.c --- a/src/lib-sieve/cmd-redirect.c Fri Apr 25 23:53:40 2014 +0200 +++ b/src/lib-sieve/cmd-redirect.c Mon Apr 28 22:08:47 2014 +0200 @@ -320,7 +320,9 @@ const char *recipient = sieve_message_get_final_recipient(msgctx); struct istream *input; struct ostream *output; + const char *error; void *smtp_handle; + int ret; /* Just to be sure */ if ( !sieve_smtp_available(senv) ) { @@ -356,11 +358,19 @@ i_stream_unref(&input); /* Close SMTP transport */ - if ( !sieve_smtp_close(senv, smtp_handle) ) { - sieve_result_global_error(aenv, - "failed to redirect message to <%s> " - "(refer to server log for more information)", - str_sanitize(ctx->to_address, 80)); + if ( (ret=sieve_smtp_close(senv, smtp_handle, &error)) <= 0 ) { + if ( ret < 0 ) { + sieve_result_global_log_error(aenv, + "failed to redirect message to <%s>: %s " + "(temporary failure)", + str_sanitize(ctx->to_address, 256), str_sanitize(error, 512)); + } else { + sieve_result_global_error(aenv, + "failed to redirect message to <%s>: %s " + "(permanent failure)", + str_sanitize(ctx->to_address, 256), str_sanitize(error, 512)); + + } return SIEVE_EXEC_FAILURE; } diff -r f5d11f8c614c -r c6c320f4918b src/lib-sieve/plugins/enotify/mailto/ntfy-mailto.c --- a/src/lib-sieve/plugins/enotify/mailto/ntfy-mailto.c Fri Apr 25 23:53:40 2014 +0200 +++ b/src/lib-sieve/plugins/enotify/mailto/ntfy-mailto.c Mon Apr 28 22:08:47 2014 +0200 @@ -392,7 +392,8 @@ struct ostream *output; string_t *msg; unsigned int count, i, hcount, h; - const char *outmsgid; + const char *outmsgid, *error; + int ret; /* Get recipients */ recipients = array_get(&mtctx->uri->recipients, &count); @@ -540,15 +541,22 @@ o_stream_send(output, str_data(msg), str_len(msg)); - if ( sieve_smtp_close(senv, smtp_handle) ) { + if ( (ret=sieve_smtp_close(senv, smtp_handle, &error)) <= 0 ) { + if (ret < 0) { + sieve_enotify_global_log_error(nenv, + "failed to send mail notification to <%s>: %s (temporary failure)", + str_sanitize(recipients[i].normalized, 256), + str_sanitize(error, 512)); + } else { + sieve_enotify_global_error(nenv, + "failed to send mail notification to <%s>: %s (permanent failure)", + str_sanitize(recipients[i].normalized, 256), + str_sanitize(error, 512)); + } + } else { sieve_enotify_global_info(nenv, "sent mail notification to <%s>", - str_sanitize(recipients[i].normalized, 80)); - } else { - sieve_enotify_global_error(nenv, - "failed to send mail notification to <%s> " - "(refer to system log for more information)", - str_sanitize(recipients[i].normalized, 80)); + str_sanitize(recipients[i].normalized, 256)); } } diff -r f5d11f8c614c -r c6c320f4918b src/lib-sieve/plugins/enotify/sieve-ext-enotify.h --- a/src/lib-sieve/plugins/enotify/sieve-ext-enotify.h Fri Apr 25 23:53:40 2014 +0200 +++ b/src/lib-sieve/plugins/enotify/sieve-ext-enotify.h Mon Apr 28 22:08:47 2014 +0200 @@ -167,5 +167,8 @@ #define sieve_enotify_global_info(ENV, ...) \ sieve_global_info((ENV)->svinst, (ENV)->ehandler, NULL, __VA_ARGS__ ) +#define sieve_enotify_global_log_error(ENV, ...) \ + sieve_global_info_error((ENV)->svinst, (ENV)->ehandler, NULL, __VA_ARGS__ ) + #endif /* __SIEVE_EXT_ENOTIFY_H */ diff -r f5d11f8c614c -r c6c320f4918b src/lib-sieve/plugins/notify/cmd-notify.c --- a/src/lib-sieve/plugins/notify/cmd-notify.c Fri Apr 25 23:53:40 2014 +0200 +++ b/src/lib-sieve/plugins/notify/cmd-notify.c Mon Apr 28 22:08:47 2014 +0200 @@ -693,8 +693,9 @@ unsigned int count, i; struct ostream *output; string_t *msg; - const char *outmsgid; + const char *outmsgid, *error; size_t hdr_size; + int ret; /* Get recipients */ recipients = array_get(&act->recipients, &count); @@ -774,15 +775,22 @@ o_stream_send(output, str_data(msg), str_len(msg)); - if ( sieve_smtp_close(senv, smtp_handle) ) { + if ( (ret=sieve_smtp_close(senv, smtp_handle, &error)) <= 0 ) { + if (ret < 0) { + sieve_result_global_log_error(aenv, + "failed to send mail notification to <%s>: %s (temporary failure)", + str_sanitize(recipients[i].normalized, 256), + str_sanitize(error, 512)); + } else { + sieve_result_global_error(aenv, + "failed to send mail notification to <%s>: %s (permanent failure)", + str_sanitize(recipients[i].normalized, 256), + str_sanitize(error, 512)); + } + } else { sieve_result_global_log(aenv, "sent mail notification to <%s>", - str_sanitize(recipients[i].normalized, 80)); - } else { - sieve_result_global_error(aenv, - "failed to send mail notification to <%s> " - "(refer to system log for more information)", - str_sanitize(recipients[i].normalized, 80)); + str_sanitize(recipients[i].normalized, 256)); } } diff -r f5d11f8c614c -r c6c320f4918b src/lib-sieve/plugins/vacation/cmd-vacation.c --- a/src/lib-sieve/plugins/vacation/cmd-vacation.c Fri Apr 25 23:53:40 2014 +0200 +++ b/src/lib-sieve/plugins/vacation/cmd-vacation.c Mon Apr 28 22:08:47 2014 +0200 @@ -905,7 +905,7 @@ struct ostream *output; string_t *msg; const char *const *headers; - const char *outmsgid, *subject; + const char *outmsgid, *subject, *error; int ret; /* Check smpt functions just to be sure */ @@ -992,11 +992,16 @@ o_stream_send(output, str_data(msg), str_len(msg)); /* Close smtp session */ - if ( !sieve_smtp_close(senv, smtp_handle) ) { - sieve_result_global_error(aenv, - "failed to send vacation response to <%s> " - "(refer to server log for more information)", - str_sanitize(reply_to, 128)); + if ( (ret=sieve_smtp_close(senv, smtp_handle, &error)) <= 0 ) { + if ( ret < 0 ) { + sieve_result_global_log_error(aenv, + "failed to send vacation response to <%s>: %s (temporary error)", + str_sanitize(reply_to, 256), str_sanitize(error, 512)); + } else { + sieve_result_global_error(aenv, + "failed to send vacation response to <%s>: %s (permanent error)", + str_sanitize(reply_to, 256), str_sanitize(error, 512)); + } return FALSE; } diff -r f5d11f8c614c -r c6c320f4918b src/lib-sieve/sieve-actions.c --- a/src/lib-sieve/sieve-actions.c Fri Apr 25 23:53:40 2014 +0200 +++ b/src/lib-sieve/sieve-actions.c Mon Apr 28 22:08:47 2014 +0200 @@ -784,8 +784,7 @@ struct istream *input; struct ostream *output; void *smtp_handle; - const char *new_msgid, *boundary; - const char *header; + const char *new_msgid, *boundary, *header, *error; string_t *hdr; int ret; @@ -875,11 +874,19 @@ str_printfa(hdr, "\r\n\r\n--%s--\r\n", boundary); o_stream_send(output, str_data(hdr), str_len(hdr)); - if ( !sieve_smtp_close(senv, smtp_handle) ) { - sieve_result_global_error(aenv, - "failed to send rejection message to <%s> " - "(refer to server log for more information)", - str_sanitize(sender, 80)); + if ( (ret=sieve_smtp_close(senv, smtp_handle, &error)) <= 0 ) { + if ( ret < 0 ) { + sieve_result_global_log_error(aenv, + "failed to send rejection message to <%s>: %s " + "(temporary failure)", + str_sanitize(sender, 256), str_sanitize(error, 512)); + } else { + sieve_result_global_error(aenv, + "failed to send rejection message to <%s>: %s " + "(permanent failure)", + str_sanitize(sender, 256), str_sanitize(error, 512)); + + } return FALSE; } diff -r f5d11f8c614c -r c6c320f4918b src/lib-sieve/sieve-smtp.c --- a/src/lib-sieve/sieve-smtp.c Fri Apr 25 23:53:40 2014 +0200 +++ b/src/lib-sieve/sieve-smtp.c Mon Apr 28 22:08:47 2014 +0200 @@ -5,29 +5,28 @@ #include "sieve-common.h" #include "sieve-smtp.h" - bool sieve_smtp_available (const struct sieve_script_env *senv) { - return ( senv->smtp_open != NULL && senv->smtp_close != NULL ); + return ( senv->smtp_open != NULL && senv->smtp_close != NULL ); } void *sieve_smtp_open (const struct sieve_script_env *senv, const char *destination, const char *return_path, struct ostream **output_r) { - if ( senv->smtp_open == NULL || senv->smtp_close == NULL ) - return NULL; + if ( !sieve_smtp_available(senv) ) + return NULL; - return senv->smtp_open(senv, destination, return_path, output_r); + return senv->smtp_open(senv, destination, return_path, output_r); } -bool sieve_smtp_close -(const struct sieve_script_env *senv, void *handle) +int sieve_smtp_close +(const struct sieve_script_env *senv, void *handle, + const char **error_r) { - if ( senv->smtp_open == NULL || senv->smtp_close == NULL ) - return NULL; + i_assert( sieve_smtp_available(senv) ); - return senv->smtp_close(senv, handle); + return senv->smtp_close(senv, handle, error_r); } diff -r f5d11f8c614c -r c6c320f4918b src/lib-sieve/sieve-smtp.h --- a/src/lib-sieve/sieve-smtp.h Fri Apr 25 23:53:40 2014 +0200 +++ b/src/lib-sieve/sieve-smtp.h Mon Apr 28 22:08:47 2014 +0200 @@ -9,11 +9,12 @@ bool sieve_smtp_available (const struct sieve_script_env *senv); +// FIXME: support multiple recipients void *sieve_smtp_open (const struct sieve_script_env *senv, const char *destination, - const char *return_path, struct ostream **output_r); - -bool sieve_smtp_close - (const struct sieve_script_env *senv, void *handle); + const char *return_path, struct ostream **output_r); +int sieve_smtp_close + (const struct sieve_script_env *senv, void *handle, + const char **error_r); #endif /* __SIEVE_SMTP_H */ diff -r f5d11f8c614c -r c6c320f4918b src/lib-sieve/sieve-types.h --- a/src/lib-sieve/sieve-types.h Fri Apr 25 23:53:40 2014 +0200 +++ b/src/lib-sieve/sieve-types.h Mon Apr 28 22:08:47 2014 +0200 @@ -195,7 +195,10 @@ void *(*smtp_open) (const struct sieve_script_env *senv, const char *destination, const char *return_path, struct ostream **output_r); - bool (*smtp_close)(const struct sieve_script_env *senv, void *handle); + /* Returns 1 on success, 0 on permanent failure, -1 on temporary failure. */ + int (*smtp_close) + (const struct sieve_script_env *senv, void *handle, + const char **error_r); /* Interface for marking and checking duplicates */ int (*duplicate_check) @@ -205,8 +208,8 @@ time_t time); /* Interface for rejecting mail */ - int (*reject_mail)(const struct sieve_script_env *senv, const char *recipient, - const char *reason); + int (*reject_mail)(const struct sieve_script_env *senv, + const char *recipient, const char *reason); /* Execution status record */ struct sieve_exec_status *exec_status; diff -r f5d11f8c614c -r c6c320f4918b src/plugins/lda-sieve/lda-sieve-plugin.c --- a/src/plugins/lda-sieve/lda-sieve-plugin.c Fri Apr 25 23:53:40 2014 +0200 +++ b/src/plugins/lda-sieve/lda-sieve-plugin.c Mon Apr 28 22:08:47 2014 +0200 From dovecot at dovecot.org Mon Apr 28 21:49:34 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Apr 2014 21:49:34 +0000 Subject: dovecot-2.2: dsync: Fixed new dsync versions to work again with ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cd6079c61c2b changeset: 17284:cd6079c61c2b user: Timo Sirainen date: Tue Apr 29 00:48:54 2014 +0300 description: dsync: Fixed new dsync versions to work again with older versions. diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 0d34a90b2760 -r cd6079c61c2b src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Mon Apr 28 20:14:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Tue Apr 29 00:48:54 2014 +0300 @@ -80,8 +80,8 @@ { .name = "mailbox_state", .chr = 'S', .required_keys = "mailbox_guid last_uidvalidity last_common_uid " - "last_common_modseq last_common_pvt_modseq last_messages_count", - .optional_keys = "changes_during_sync" + "last_common_modseq last_common_pvt_modseq", + .optional_keys = "last_messages_count changes_during_sync" }, { .name = "mailbox_tree_node", .chr = 'N', From dovecot at dovecot.org Mon Apr 28 21:50:37 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Apr 2014 21:50:37 +0000 Subject: dovecot-2.2: lib-lda: More smtp-client.h API changes to support ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/605df40b0aef changeset: 17285:605df40b0aef user: Timo Sirainen date: Tue Apr 29 00:50:08 2014 +0300 description: lib-lda: More smtp-client.h API changes to support multiple destination addresses. diffstat: src/lib-lda/mail-send.c | 4 +- src/lib-lda/smtp-client.c | 176 +++++++++++++++++++++------------------------ src/lib-lda/smtp-client.h | 8 +- 3 files changed, 93 insertions(+), 95 deletions(-) diffs (truncated from 320 to 300 lines): diff -r cd6079c61c2b -r 605df40b0aef src/lib-lda/mail-send.c --- a/src/lib-lda/mail-send.c Tue Apr 29 00:48:54 2014 +0300 +++ b/src/lib-lda/mail-send.c Tue Apr 29 00:50:08 2014 +0300 @@ -86,7 +86,9 @@ str_sanitize(reason, 512)); } - smtp_client = smtp_client_init(ctx->set, return_addr, NULL, &output); + smtp_client = smtp_client_init(ctx->set, NULL); + smtp_client_add_rcpt(smtp_client, return_addr); + output = smtp_client_send(smtp_client); msgid = mail_deliver_get_new_message_id(ctx); boundary = t_strdup_printf("%s/%s", my_pid, ctx->set->hostname); diff -r cd6079c61c2b -r 605df40b0aef src/lib-lda/smtp-client.c --- a/src/lib-lda/smtp-client.c Tue Apr 29 00:48:54 2014 +0300 +++ b/src/lib-lda/smtp-client.c Tue Apr 29 00:50:08 2014 +0300 @@ -21,8 +21,8 @@ #define DEFAULT_SUBMISSION_PORT 25 struct smtp_client { + pool_t pool; struct ostream *output; - buffer_t *buf; int temp_fd; pid_t pid; @@ -31,49 +31,33 @@ bool finished; const struct lda_settings *set; - char *temp_path; - char *destination; - char *return_path; - char *error; + const char *temp_path; + ARRAY_TYPE(const_string) destinations; + const char *return_path; + const char *error; bool tempfail; }; -static struct smtp_client *smtp_client_devnull(struct ostream **output_r) -{ - struct smtp_client *client; - - client = i_new(struct smtp_client, 1); - client->buf = buffer_create_dynamic(default_pool, 1); - client->output = o_stream_create_buffer(client->buf); - o_stream_close(client->output); - client->pid = (pid_t)-1; - - *output_r = client->output; - return client; -} - static void ATTR_NORETURN -smtp_client_run_sendmail(const struct lda_settings *set, - const char *destination, - const char *return_path, int fd) +smtp_client_run_sendmail(struct smtp_client *client, int fd) { const char *const *sendmail_args, *const *argv, *str; ARRAY_TYPE(const_string) args; unsigned int i; - sendmail_args = t_strsplit(set->sendmail_path, " "); + sendmail_args = t_strsplit(client->set->sendmail_path, " "); t_array_init(&args, 16); for (i = 0; sendmail_args[i] != NULL; i++) array_append(&args, &sendmail_args[i], 1); str = "-i"; array_append(&args, &str, 1); /* ignore dots */ str = "-f"; array_append(&args, &str, 1); - str = return_path != NULL && *return_path != '\0' ? - return_path : "<>"; + str = client->return_path != NULL && *client->return_path != '\0' ? + client->return_path : "<>"; array_append(&args, &str, 1); str = "--"; array_append(&args, &str, 1); - array_append(&args, &destination, 1); + array_append_array(&args, &client->destinations); array_append_zero(&args); argv = array_idx(&args, 0); @@ -85,41 +69,6 @@ execv_const(argv[0], argv); } -static struct smtp_client * -smtp_client_init_sendmail(const struct lda_settings *set, - const char *destination, const char *return_path, - struct ostream **output_r) -{ - struct smtp_client *client; - int fd[2]; - pid_t pid; - - if (pipe(fd) < 0) { - i_error("pipe() failed: %m"); - return smtp_client_devnull(output_r); - } - - if ((pid = fork()) == (pid_t)-1) { - i_error("fork() failed: %m"); - i_close_fd(&fd[0]); i_close_fd(&fd[1]); - return smtp_client_devnull(output_r); - } - if (pid == 0) { - /* child */ - i_close_fd(&fd[1]); - smtp_client_run_sendmail(set, destination, return_path, fd[0]); - } - i_close_fd(&fd[0]); - - client = i_new(struct smtp_client, 1); - client->output = o_stream_create_fd(fd[1], IO_BLOCK_SIZE, TRUE); - o_stream_set_no_error_handling(client->output, TRUE); - client->pid = pid; - - *output_r = client->output; - return client; -} - static int create_temp_file(const char **path_r) { string_t *path; @@ -149,33 +98,75 @@ } struct smtp_client * -smtp_client_init(const struct lda_settings *set, const char *destination, - const char *return_path, struct ostream **output_r) +smtp_client_init(const struct lda_settings *set, const char *return_path) { struct smtp_client *client; + pool_t pool; + + pool = pool_alloconly_create("smtp client", 256); + client = p_new(pool, struct smtp_client, 1); + client->pool = pool; + client->set = set; + client->return_path = p_strdup(pool, return_path); + client->use_smtp = *set->submission_host != '\0'; + p_array_init(&client->destinations, pool, 2); + client->pid = (pid_t)-1; + return client; +} + +void smtp_client_add_rcpt(struct smtp_client *client, const char *address) +{ + i_assert(client->output == NULL); + + address = p_strdup(client->pool, address); + array_append(&client->destinations, &address, 1); +} + +static struct ostream *smtp_client_send_sendmail(struct smtp_client *client) +{ + int fd[2]; + pid_t pid; + + if (pipe(fd) < 0) { + i_error("pipe() failed: %m"); + return o_stream_create_error(errno); + } + + if ((pid = fork()) == (pid_t)-1) { + i_error("fork() failed: %m"); + i_close_fd(&fd[0]); i_close_fd(&fd[1]); + return o_stream_create_error(errno); + } + if (pid == 0) { + /* child */ + i_close_fd(&fd[1]); + smtp_client_run_sendmail(client, fd[0]); + } + i_close_fd(&fd[0]); + + client->output = o_stream_create_fd(fd[1], IO_BLOCK_SIZE, TRUE); + o_stream_set_no_error_handling(client->output, TRUE); + client->pid = pid; + return client->output; +} + +struct ostream *smtp_client_send(struct smtp_client *client) +{ const char *path; int fd; - if (*set->submission_host == '\0') { - return smtp_client_init_sendmail(set, destination, - return_path, output_r); - } + i_assert(array_count(&client->destinations) > 0); + + if (!client->use_smtp) + return smtp_client_send_sendmail(client); if ((fd = create_temp_file(&path)) == -1) - return smtp_client_devnull(output_r); - - client = i_new(struct smtp_client, 1); - client->set = set; + return o_stream_create_error(errno); client->temp_path = i_strdup(path); - client->destination = i_strdup(destination); - client->return_path = i_strdup(return_path); client->temp_fd = fd; client->output = o_stream_create_fd(fd, IO_BLOCK_SIZE, TRUE); o_stream_set_no_error_handling(client->output, TRUE); - client->use_smtp = TRUE; - - *output_r = client->output; - return client; + return client->output; } static int smtp_client_deinit_sendmail(struct smtp_client *client) @@ -185,7 +176,7 @@ o_stream_destroy(&client->output); if (client->pid == (pid_t)-1) { - /* smtp_client_init() failed already */ + /* smtp_client_send() failed already */ } else if (waitpid(client->pid, &status, 0) < 0) i_error("waitpid() failed: %m"); else if (WIFEXITED(status)) { @@ -204,9 +195,7 @@ i_error("Sendmail process terminated abnormally, " "return status %d", status); } - if (client->buf != NULL) - buffer_free(&client->buf); - i_free(client); + pool_unref(&client->pool); return ret; } @@ -257,13 +246,13 @@ } } -static int smtp_client_send(struct smtp_client *smtp_client) +static int smtp_client_send_flush(struct smtp_client *smtp_client) { struct lmtp_client_settings client_set; struct lmtp_client *client; struct ioloop *ioloop; struct istream *input; - const char *host, *p; + const char *host, *p, *const *destp; unsigned int port = DEFAULT_SUBMISSION_PORT; host = smtp_client->set->submission_host; @@ -303,8 +292,10 @@ return -1; } - lmtp_client_add_rcpt(client, smtp_client->destination, - rcpt_to_callback, data_callback, smtp_client); + array_foreach(&smtp_client->destinations, destp) { + lmtp_client_add_rcpt(client, *destp, rcpt_to_callback, + data_callback, smtp_client); + } input = i_stream_create_fd(smtp_client->temp_fd, (size_t)-1, FALSE); lmtp_client_send(client, input); @@ -335,16 +326,12 @@ } /* the mail has been written to a file. now actually send it. */ - ret = smtp_client_send(client); + ret = smtp_client_send_flush(client); o_stream_destroy(&client->output); - i_free(client->return_path); - i_free(client->destination); - i_free(client->temp_path); *error_r = t_strdup(client->error); - i_free(client->error); - i_free(client); + pool_unref(&client->pool); return ret; } @@ -352,7 +339,12 @@ smtp_client_open(const struct lda_settings *set, const char *destination, const char *return_path, struct ostream **output_r) { - return smtp_client_init(set, destination, return_path, output_r); + struct smtp_client *client; + + client = smtp_client_init(set, return_path); + smtp_client_add_rcpt(client, destination); + *output_r = smtp_client_send(client); + return client; } From dovecot at dovecot.org Mon Apr 28 21:53:27 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Apr 2014 21:53:27 +0000 Subject: dovecot-2.2: dsync: Yet another fix to working with old versions. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/507aee5fef6d changeset: 17286:507aee5fef6d user: Timo Sirainen date: Tue Apr 29 00:53:01 2014 +0300 description: dsync: Yet another fix to working with old versions. diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 605df40b0aef -r 507aee5fef6d src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Tue Apr 29 00:50:08 2014 +0300 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Tue Apr 29 00:53:01 2014 +0300 @@ -850,8 +850,8 @@ dsync_ibc_input_error(ibc, decoder, "Invalid last_common_pvt_modseq"); return DSYNC_IBC_RECV_RET_TRYAGAIN; } - value = dsync_deserializer_decode_get(decoder, "last_messages_count"); - if (str_to_uint32(value, &state_r->last_messages_count) < 0) { + if (dsync_deserializer_decode_try(decoder, "last_messages_count", &value) && + str_to_uint32(value, &state_r->last_messages_count) < 0) { dsync_ibc_input_error(ibc, decoder, "Invalid last_messages_count"); return DSYNC_IBC_RECV_RET_TRYAGAIN; } From pigeonhole at rename-it.nl Tue Apr 29 00:15:01 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 29 Apr 2014 02:15:01 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: Adjusted to more SMTP API cha... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/439b1b2fe05d changeset: 1849:439b1b2fe05d user: Stephan Bosch date: Tue Apr 29 02:13:35 2014 +0200 description: lib-sieve: Adjusted to more SMTP API changes in Dovecot. SMTP API now supports adding multiple recipients to a single SMTP transaction. This makes enotify/notify more efficient, since only one message is actually sent. diffstat: src/lib-sieve/cmd-redirect.c | 6 +- src/lib-sieve/plugins/enotify/mailto/ntfy-mailto.c | 50 ++++++++----- src/lib-sieve/plugins/notify/cmd-notify.c | 78 ++++++++++++--------- src/lib-sieve/plugins/vacation/cmd-vacation.c | 7 +- src/lib-sieve/sieve-actions.c | 11 +- src/lib-sieve/sieve-smtp.c | 69 ++++++++++++++++--- src/lib-sieve/sieve-smtp.h | 20 +++- src/lib-sieve/sieve-types.h | 14 ++- src/plugins/lda-sieve/lda-sieve-plugin.c | 33 ++++++-- src/sieve-tools/sieve-filter.c | 2 - src/sieve-tools/sieve-test.c | 40 +++++++--- src/testsuite/testsuite-script.c | 12 ++- src/testsuite/testsuite-smtp.c | 60 +++++++++++----- src/testsuite/testsuite-smtp.h | 12 +- src/testsuite/testsuite.c | 6 +- tests/deprecated/notify/mailto.svtest | 4 +- 16 files changed, 274 insertions(+), 150 deletions(-) diffs (truncated from 754 to 300 lines): diff -r c6c320f4918b -r 439b1b2fe05d src/lib-sieve/cmd-redirect.c --- a/src/lib-sieve/cmd-redirect.c Mon Apr 28 22:08:47 2014 +0200 +++ b/src/lib-sieve/cmd-redirect.c Tue Apr 29 02:13:35 2014 +0200 @@ -321,7 +321,7 @@ struct istream *input; struct ostream *output; const char *error; - void *smtp_handle; + struct sieve_smtp_context *sctx; int ret; /* Just to be sure */ @@ -335,7 +335,7 @@ return SIEVE_EXEC_TEMP_FAILURE; /* Open SMTP transport */ - smtp_handle = sieve_smtp_open(senv, ctx->to_address, sender, &output); + sctx = sieve_smtp_start_single(senv, ctx->to_address, sender, &output); /* Remove unwanted headers */ input = i_stream_create_header_filter @@ -358,7 +358,7 @@ i_stream_unref(&input); /* Close SMTP transport */ - if ( (ret=sieve_smtp_close(senv, smtp_handle, &error)) <= 0 ) { + if ( (ret=sieve_smtp_finish(sctx, &error)) <= 0 ) { if ( ret < 0 ) { sieve_result_global_log_error(aenv, "failed to redirect message to <%s>: %s " diff -r c6c320f4918b -r 439b1b2fe05d src/lib-sieve/plugins/enotify/mailto/ntfy-mailto.c --- a/src/lib-sieve/plugins/enotify/mailto/ntfy-mailto.c Mon Apr 28 22:08:47 2014 +0200 +++ b/src/lib-sieve/plugins/enotify/mailto/ntfy-mailto.c Tue Apr 29 02:13:35 2014 +0200 @@ -385,10 +385,10 @@ const char *from = NULL, *from_smtp = NULL; const char *subject = mtctx->uri->subject; const char *body = mtctx->uri->body; - string_t *to, *cc; + string_t *to, *cc, *all; const struct uri_mailto_recipient *recipients; const struct uri_mailto_header_field *headers; - void *smtp_handle; + struct sieve_smtp_context *sctx; struct ostream *output; string_t *msg; unsigned int count, i, hcount, h; @@ -445,6 +445,7 @@ /* Compose To and Cc headers */ to = NULL; cc = NULL; + all = t_str_new(256); for ( i = 0; i < count; i++ ) { if ( recipients[i].carbon_copy ) { if ( cc == NULL ) { @@ -463,6 +464,15 @@ str_append(to, recipients[i].full); } } + if ( i < 3) { + if ( i > 0 ) + str_append(all, ", "); + str_append_c(all, '<'); + str_append(all, str_sanitize(recipients[i].normalized, 256)); + str_append_c(all, '>'); + } else if (i == 3) { + str_printfa(all, ", ... (%u total)", count); + } } msg = t_str_new(512); @@ -534,30 +544,28 @@ str_append(msg, "\r\nNotification of new message.\r\n"); } + sctx = sieve_smtp_start(senv, from_smtp); + /* Send message to all recipients */ - for ( i = 0; i < count; i++ ) { - smtp_handle = sieve_smtp_open - (senv, recipients[i].normalized, from_smtp, &output); + for ( i = 0; i < count; i++ ) + sieve_smtp_add_rcpt(sctx, recipients[i].normalized); - o_stream_send(output, str_data(msg), str_len(msg)); + output = sieve_smtp_send(sctx); + o_stream_send(output, str_data(msg), str_len(msg)); - if ( (ret=sieve_smtp_close(senv, smtp_handle, &error)) <= 0 ) { - if (ret < 0) { - sieve_enotify_global_log_error(nenv, - "failed to send mail notification to <%s>: %s (temporary failure)", - str_sanitize(recipients[i].normalized, 256), - str_sanitize(error, 512)); - } else { - sieve_enotify_global_error(nenv, - "failed to send mail notification to <%s>: %s (permanent failure)", - str_sanitize(recipients[i].normalized, 256), - str_sanitize(error, 512)); - } + if ( (ret=sieve_smtp_finish(sctx, &error)) <= 0 ) { + if (ret < 0) { + sieve_enotify_global_log_error(nenv, + "failed to send mail notification to %s: %s (temporary failure)", + str_c(all), str_sanitize(error, 512)); } else { - sieve_enotify_global_info(nenv, - "sent mail notification to <%s>", - str_sanitize(recipients[i].normalized, 256)); + sieve_enotify_global_error(nenv, + "failed to send mail notification to %s: %s (permanent failure)", + str_c(all), str_sanitize(error, 512)); } + } else { + sieve_enotify_global_info(nenv, + "sent mail notification to %s", str_c(all)); } return TRUE; diff -r c6c320f4918b -r 439b1b2fe05d src/lib-sieve/plugins/notify/cmd-notify.c --- a/src/lib-sieve/plugins/notify/cmd-notify.c Mon Apr 28 22:08:47 2014 +0200 +++ b/src/lib-sieve/plugins/notify/cmd-notify.c Tue Apr 29 02:13:35 2014 +0200 @@ -689,12 +689,11 @@ { const struct sieve_script_env *senv = aenv->scriptenv; const struct ext_notify_recipient *recipients; - void *smtp_handle; + struct sieve_smtp_context *sctx; unsigned int count, i; struct ostream *output; - string_t *msg; + string_t *msg, *to, *all; const char *outmsgid, *error; - size_t hdr_size; int ret; /* Get recipients */ @@ -706,7 +705,7 @@ } /* Just to be sure */ - if ( senv->smtp_open == NULL || senv->smtp_close == NULL ) { + if ( !sieve_smtp_available(senv) ) { sieve_result_global_warning(aenv, "notify action has no means to send mail"); return TRUE; @@ -753,45 +752,54 @@ rfc2822_header_write(msg, "Content-Transfer-Encoding", "7bit"); } - hdr_size = str_len(msg); + outmsgid = sieve_message_get_new_id(aenv->svinst); + rfc2822_header_write(msg, "Message-ID", outmsgid); - /* Send message to all recipients */ + if ( sieve_message_get_sender(aenv->msgctx) != NULL ) + sctx = sieve_smtp_start(senv, senv->postmaster_address); + else + sctx = sieve_smtp_start(senv, NULL); + + /* Add all recipients (and compose To header field) */ + to = t_str_new(128); + all = t_str_new(256); for ( i = 0; i < count; i++ ) { - if ( sieve_message_get_sender(aenv->msgctx) != NULL ) - smtp_handle = sieve_smtp_open - (senv, recipients[i].normalized, senv->postmaster_address, &output); - else - smtp_handle = sieve_smtp_open - (senv, recipients[i].normalized, NULL, &output); + sieve_smtp_add_rcpt(sctx, recipients[i].normalized); + if ( i > 0 ) + str_append(to, ", "); + str_append(to, recipients[i].full); + if ( i < 3) { + if ( i > 0 ) + str_append(all, ", "); + str_append_c(all, '<'); + str_append(all, str_sanitize(recipients[i].normalized, 256)); + str_append_c(all, '>'); + } else if (i == 3) { + str_printfa(all, ", ... (%u total)", count); + } + } - str_truncate(msg, hdr_size); + rfc2822_header_write(msg, "To", str_c(to)); - outmsgid = sieve_message_get_new_id(aenv->svinst); - rfc2822_header_write(msg, "Message-ID", outmsgid); - rfc2822_header_write(msg, "To", recipients[i].full); + /* Generate message body */ + str_printfa(msg, "\r\n%s\r\n", act->message); - /* Generate message body */ - str_printfa(msg, "\r\n%s\r\n", act->message); + output = sieve_smtp_send(sctx); + o_stream_send(output, str_data(msg), str_len(msg)); - o_stream_send(output, str_data(msg), str_len(msg)); - - if ( (ret=sieve_smtp_close(senv, smtp_handle, &error)) <= 0 ) { - if (ret < 0) { - sieve_result_global_log_error(aenv, - "failed to send mail notification to <%s>: %s (temporary failure)", - str_sanitize(recipients[i].normalized, 256), - str_sanitize(error, 512)); - } else { - sieve_result_global_error(aenv, - "failed to send mail notification to <%s>: %s (permanent failure)", - str_sanitize(recipients[i].normalized, 256), - str_sanitize(error, 512)); - } + if ( (ret=sieve_smtp_finish(sctx, &error)) <= 0 ) { + if (ret < 0) { + sieve_result_global_log_error(aenv, + "failed to send mail notification to %s: %s (temporary failure)", + str_c(all), str_sanitize(error, 512)); } else { - sieve_result_global_log(aenv, - "sent mail notification to <%s>", - str_sanitize(recipients[i].normalized, 256)); + sieve_result_global_error(aenv, + "failed to send mail notification to %s: %s (permanent failure)", + str_c(all), str_sanitize(error, 512)); } + } else { + sieve_result_global_log(aenv, + "sent mail notification to %s", str_c(all)); } return TRUE; diff -r c6c320f4918b -r 439b1b2fe05d src/lib-sieve/plugins/vacation/cmd-vacation.c --- a/src/lib-sieve/plugins/vacation/cmd-vacation.c Mon Apr 28 22:08:47 2014 +0200 +++ b/src/lib-sieve/plugins/vacation/cmd-vacation.c Tue Apr 29 02:13:35 2014 +0200 @@ -901,7 +901,7 @@ { const struct sieve_message_data *msgdata = aenv->msgdata; const struct sieve_script_env *senv = aenv->scriptenv; - void *smtp_handle; + struct sieve_smtp_context *sctx; struct ostream *output; string_t *msg; const char *const *headers; @@ -933,7 +933,8 @@ /* Open smtp session */ - smtp_handle = sieve_smtp_open(senv, reply_to, smtp_from, &output); + sctx = sieve_smtp_start_single(senv, reply_to, smtp_from, &output); + outmsgid = sieve_message_get_new_id(aenv->svinst); /* Produce a proper reply */ @@ -992,7 +993,7 @@ o_stream_send(output, str_data(msg), str_len(msg)); /* Close smtp session */ - if ( (ret=sieve_smtp_close(senv, smtp_handle, &error)) <= 0 ) { + if ( (ret=sieve_smtp_finish(sctx, &error)) <= 0 ) { if ( ret < 0 ) { sieve_result_global_log_error(aenv, "failed to send vacation response to <%s>: %s (temporary error)", diff -r c6c320f4918b -r 439b1b2fe05d src/lib-sieve/sieve-actions.c --- a/src/lib-sieve/sieve-actions.c Mon Apr 28 22:08:47 2014 +0200 +++ b/src/lib-sieve/sieve-actions.c Tue Apr 29 02:13:35 2014 +0200 @@ -783,20 +783,20 @@ const struct sieve_message_data *msgdata = aenv->msgdata; struct istream *input; struct ostream *output; - void *smtp_handle; + struct sieve_smtp_context *sctx; const char *new_msgid, *boundary, *header, *error; string_t *hdr; int ret; + sctx = sieve_smtp_start_single(senv, sender, NULL, &output); + /* Just to be sure */ - if ( !sieve_smtp_available(senv) ) { + if ( sctx == NULL ) { sieve_result_global_warning (aenv, "reject action has no means to send mail"); return TRUE; } - smtp_handle = sieve_smtp_open(senv, sender, NULL, &output); - new_msgid = sieve_message_get_new_id(svinst); boundary = t_strdup_printf("%s/%s", my_pid, svinst->hostname); @@ -874,7 +874,7 @@ str_printfa(hdr, "\r\n\r\n--%s--\r\n", boundary); o_stream_send(output, str_data(hdr), str_len(hdr)); - if ( (ret=sieve_smtp_close(senv, smtp_handle, &error)) <= 0 ) { + if ( (ret=sieve_smtp_finish(sctx, &error)) <= 0 ) { if ( ret < 0 ) { sieve_result_global_log_error(aenv, "failed to send rejection message to <%s>: %s " @@ -885,7 +885,6 @@ From dovecot at dovecot.org Tue Apr 29 09:35:37 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Apr 2014 09:35:37 +0000 Subject: dovecot-2.2: istream-lf: Cleanup & performance improvement. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/66620a924bfa changeset: 17287:66620a924bfa user: Timo Sirainen date: Tue Apr 29 12:35:05 2014 +0300 description: istream-lf: Cleanup & performance improvement. memchr() is faster than lopping through the data ourself. diffstat: src/lib/istream-crlf.c | 68 ++++++++++++++++++++++++++----------------------- 1 files changed, 36 insertions(+), 32 deletions(-) diffs (91 lines): diff -r 507aee5fef6d -r 66620a924bfa src/lib/istream-crlf.c --- a/src/lib/istream-crlf.c Tue Apr 29 00:53:01 2014 +0300 +++ b/src/lib/istream-crlf.c Tue Apr 29 12:35:05 2014 +0300 @@ -114,10 +114,10 @@ static ssize_t i_stream_crlf_read_lf(struct istream_private *stream) { struct crlf_istream *cstream = (struct crlf_istream *)stream; - const unsigned char *data; - size_t i, dest, size; + const unsigned char *data, *p; + size_t i, dest, size, max; ssize_t ret; - int diff; + bool pending_cr; ret = i_stream_crlf_read_common(cstream); if (ret <= 0) @@ -126,40 +126,44 @@ data = i_stream_get_data(stream->parent, &size); /* @UNSAFE */ + /* \r\n -> \n + \r -> \r + \r\r\n -> \r\n */ dest = stream->pos; - if (data[0] == '\n') { - stream->w_buffer[dest++] = '\n'; - cstream->pending_cr = FALSE; - } else { - if (cstream->pending_cr) { - /* CR without LF */ + pending_cr = cstream->pending_cr; + for (i = 0; i < size && dest < stream->buffer_size; ) { + if (data[i] == '\r') { + if (pending_cr) { + /* \r\r */ + stream->w_buffer[dest++] = '\r'; + } else { + pending_cr = TRUE; + } + i++; + } else if (data[i] == '\n') { + /* [\r]\n */ + pending_cr = FALSE; + stream->w_buffer[dest++] = '\n'; + i++; + } else if (pending_cr) { + /* \r */ + pending_cr = FALSE; stream->w_buffer[dest++] = '\r'; - if (dest == stream->buffer_size) { - stream->pos++; - cstream->pending_cr = FALSE; - return 1; - } + } else { + /* copy everything until the next \r */ + max = I_MIN(size - i, stream->buffer_size - dest); + p = memchr(data + i, '\r', max); + if (p != NULL) + max = p - (data+i); + memcpy(stream->w_buffer + dest, data + i, max); + dest += max; + i += max; } - if (data[0] != '\r') - stream->w_buffer[dest++] = data[0]; } + i_assert(i <= size); + i_assert(dest <= stream->buffer_size); - diff = 1; - for (i = 1; i < size && dest < stream->buffer_size; i++) { - if (data[i] == '\r') { - if (data[i-1] != '\r') - continue; - } else if (data[i-1] == '\r' && data[i] != '\n') { - stream->w_buffer[dest++] = '\r'; - if (dest == stream->buffer_size) { - diff = 0; - break; - } - } - - stream->w_buffer[dest++] = data[i]; - } - cstream->pending_cr = data[i-diff] == '\r'; + cstream->pending_cr = pending_cr; i_stream_skip(stream->parent, i); ret = dest - stream->pos; From dovecot at dovecot.org Tue Apr 29 10:10:55 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Apr 2014 10:10:55 +0000 Subject: dovecot-2.2: lib-index: Small code cleanup Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4e3a518f6fcd changeset: 17288:4e3a518f6fcd user: Timo Sirainen date: Tue Apr 29 13:10:28 2014 +0300 description: lib-index: Small code cleanup diffstat: src/lib-index/mail-index-sync-update.c | 19 +++++++------------ 1 files changed, 7 insertions(+), 12 deletions(-) diffs (53 lines): diff -r 66620a924bfa -r 4e3a518f6fcd src/lib-index/mail-index-sync-update.c --- a/src/lib-index/mail-index-sync-update.c Tue Apr 29 12:35:05 2014 +0300 +++ b/src/lib-index/mail-index-sync-update.c Tue Apr 29 13:10:28 2014 +0300 @@ -165,14 +165,11 @@ static void mail_index_sync_header_update_counts(struct mail_index_sync_map_ctx *ctx, uint32_t uid, uint8_t old_flags, - uint8_t new_flags, bool all) + uint8_t new_flags) { const char *error; - if (all) { - mail_index_sync_header_update_counts_all(ctx, uid, old_flags, - new_flags); - } else if (uid >= ctx->view->map->hdr.next_uid) { + if (uid >= ctx->view->map->hdr.next_uid) { mail_index_sync_set_corrupted(ctx, "uid %u >= next_uid %u", uid, ctx->view->map->hdr.next_uid); } else { @@ -250,8 +247,7 @@ map = mail_index_sync_get_atomic_map(ctx); for (seq = seq1; seq <= seq2; seq++) { rec = MAIL_INDEX_MAP_IDX(map, seq-1); - mail_index_sync_header_update_counts(ctx, rec->uid, rec->flags, - 0, FALSE); + mail_index_sync_header_update_counts(ctx, rec->uid, rec->flags, 0); } /* @UNSAFE */ @@ -393,8 +389,7 @@ map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; mail_index_header_update_lowwaters(ctx, rec->uid, new_flags); - mail_index_sync_header_update_counts(ctx, rec->uid, - 0, new_flags, FALSE); + mail_index_sync_header_update_counts(ctx, rec->uid, 0, new_flags); return 1; } @@ -436,9 +431,9 @@ mail_index_header_update_lowwaters(ctx, rec->uid, rec->flags); - mail_index_sync_header_update_counts(ctx, rec->uid, - old_flags, - rec->flags, TRUE); + mail_index_sync_header_update_counts_all(ctx, rec->uid, + old_flags, + rec->flags); } } return 1; From dovecot at dovecot.org Tue Apr 29 10:13:35 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Apr 2014 10:13:35 +0000 Subject: dovecot-2.1: lib-index: Replaced assert-crash with "index corrup... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/b20b4071a282 changeset: 15014:b20b4071a282 user: Timo Sirainen date: Mon Aug 05 14:06:23 2013 +0300 description: lib-index: Replaced assert-crash with "index corrupted" error. diffstat: src/lib-index/mail-index-sync-update.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (15 lines): diff -r 2799cc960c0d -r b20b4071a282 src/lib-index/mail-index-sync-update.c --- a/src/lib-index/mail-index-sync-update.c Tue Mar 18 16:49:10 2014 +0200 +++ b/src/lib-index/mail-index-sync-update.c Mon Aug 05 14:06:23 2013 +0300 @@ -176,8 +176,10 @@ if (all) { mail_index_sync_header_update_counts_all(ctx, uid, old_flags, new_flags); + } else if (uid >= ctx->view->map->hdr.next_uid) { + mail_index_sync_set_corrupted(ctx, "uid %u >= next_uid %u", + uid, ctx->view->map->hdr.next_uid); } else { - i_assert(uid < ctx->view->map->hdr.next_uid); if (mail_index_header_update_counts(&ctx->view->map->hdr, old_flags, new_flags, &error) < 0) From dovecot at dovecot.org Tue Apr 29 10:36:27 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Apr 2014 10:36:27 +0000 Subject: dovecot-2.2: lib-storage: Fixed crash if mailbox_save_cancel() w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/db216ddbb5c2 changeset: 17289:db216ddbb5c2 user: Timo Sirainen date: Tue Apr 29 13:35:06 2014 +0300 description: lib-storage: Fixed crash if mailbox_save_cancel() was called in save_finish()/copy() method. diffstat: src/lib-storage/mail-storage-private.h | 2 ++ src/lib-storage/mail-storage.c | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletions(-) diffs (45 lines): diff -r 4e3a518f6fcd -r db216ddbb5c2 src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Tue Apr 29 13:10:28 2014 +0300 +++ b/src/lib-storage/mail-storage-private.h Tue Apr 29 13:35:06 2014 +0300 @@ -559,6 +559,8 @@ /* mailbox_save_alloc() called, but finish/cancel not. the same context is usually returned by the backends for reuse. */ unsigned int unfinished:1; + /* mailbox_save_finish() or mailbox_copy() is being called. */ + unsigned int finishing:1; /* mail was copied using saving */ unsigned int copying_via_save:1; /* mail is being saved, not copied */ diff -r 4e3a518f6fcd -r db216ddbb5c2 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Tue Apr 29 13:10:28 2014 +0300 +++ b/src/lib-storage/mail-storage.c Tue Apr 29 13:35:06 2014 +0300 @@ -2081,7 +2081,10 @@ } *_ctx = NULL; + ctx->finishing = TRUE; ret = t->box->v.save_finish(ctx); + ctx->finishing = FALSE; + if (ret == 0 && !copying_via_save) { if (pvt_flags != 0) mailbox_save_add_pvt_flags(t, pvt_flags); @@ -2102,7 +2105,7 @@ *_ctx = NULL; ctx->transaction->box->v.save_cancel(ctx); - if (keywords != NULL) + if (keywords != NULL && !ctx->finishing) mailbox_keywords_unref(&keywords); if (ctx->dest_mail != NULL) { /* the dest_mail is no longer valid. if we're still saving @@ -2144,7 +2147,9 @@ mailbox_save_cancel(&ctx); return -1; } + ctx->finishing = TRUE; ret = t->box->v.copy(ctx, backend_mail); + ctx->finishing = FALSE; if (ret == 0) { if (pvt_flags != 0) mailbox_save_add_pvt_flags(t, pvt_flags); From dovecot at dovecot.org Tue Apr 29 10:58:15 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Apr 2014 10:58:15 +0000 Subject: dovecot-2.2: lib-mail: Added more tests to test-istream-dot. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/59ada187a702 changeset: 17290:59ada187a702 user: Timo Sirainen date: Tue Apr 29 13:57:44 2014 +0300 description: lib-mail: Added more tests to test-istream-dot. diffstat: src/lib-mail/test-istream-dot.c | 93 +++++++++++++++++++++++++++++++++++++++- 1 files changed, 90 insertions(+), 3 deletions(-) diffs (134 lines): diff -r db216ddbb5c2 -r 59ada187a702 src/lib-mail/test-istream-dot.c --- a/src/lib-mail/test-istream-dot.c Tue Apr 29 13:35:06 2014 +0300 +++ b/src/lib-mail/test-istream-dot.c Tue Apr 29 13:57:44 2014 +0300 @@ -32,7 +32,8 @@ (test->input[input_len-1] == '\n' || strstr(test->input, "\n.\n") != NULL || strstr(test->input, "\n.\r\n") != NULL)) { - if (test->output[output_len-1] == '\n') { + if (output_len > 0 && + test->output[output_len-1] == '\n') { output_len--; if (output_len > 0 && test->output[output_len-1] == '\r') @@ -90,6 +91,7 @@ data = i_stream_get_data(input, &size); str_append_n(str, data, size); } + test_assert(input->stream_errno == 0); test_assert(str_len(str) == output_len); test_assert(memcmp(str_data(str), test->output, output_len) == 0); @@ -101,6 +103,61 @@ i_stream_unref(&input); } +static void test_istream_dot_error(const char *input_str, bool test_bufsize) +{ + struct istream *test_input, *input; + unsigned int i, outsize, input_len; + uoff_t offset; + int ret; + + test_input = test_istream_create(input_str); + input = i_stream_create_dot(test_input, FALSE); + + input_len = strlen(input_str); + + if (!test_bufsize) { + outsize = 1; i = 0; + i_stream_set_max_buffer_size(input, outsize); + test_istream_set_size(test_input, 1); + while ((ret = i_stream_read(input)) != -1) { + switch (ret) { + case -2: + i_stream_set_max_buffer_size(input, ++outsize); + offset = test_input->v_offset; + /* seek one byte backwards so stream gets + reset */ + i_stream_seek(test_input, offset - 1); + /* go back to original position */ + test_istream_set_size(test_input, offset); + i_stream_skip(test_input, 1); + /* and finally allow reading one more byte */ + test_istream_set_size(test_input, offset + 1); + break; + case 0: + test_istream_set_size(test_input, ++i); + break; + default: + test_assert(ret > 0); + } + } + test_istream_set_size(test_input, input_len); + (void)i_stream_read(test_input); + } else { + test_istream_set_size(test_input, input_len); + for (i = 1; i <= input_len; i++) { + i_stream_set_max_buffer_size(input, i); + (void)i_stream_read(input); + (void)i_stream_read(input); + } + i_stream_set_max_buffer_size(input, i+1); + (void)i_stream_read(input); + } + test_assert(input->stream_errno == EPIPE); + + i_stream_unref(&test_input); + i_stream_unref(&input); +} + static void test_istream_dot(void) { static struct dot_test tests[] = { @@ -108,11 +165,37 @@ { "..foo\r\n..\r\n.foo\r\n.\r\nfoo", ".foo\r\n.\r\nfoo\r\n", "foo" }, { "\r.\r\n.\r\n", "\r.\r\n", "" }, { "\n\r.\r\r\n.\r\n", "\n\r.\r\r\n", "" }, + { "\r\n.\rfoo\n.\n", "\r\n\rfoo\n", "" }, { "\r\n.\r\n", "\r\n", "" }, { "\n.\r\n", "\n", "" }, { "\n.\n", "\n", "" }, - { "\n.", "\n", "" }, - { ".", "", "" } + { ".\r\n", "", "" }, + { ".\n", "", "" } + }; + static const char *error_tests[] = { + "", + ".", + "..", + ".\r", + ".\rx", + "..\r\n", + "\r.", + "\r.\r", + "\r.\rx", + "\r.\r\n", + "\r.\n", + "\r..\n", + "\r\n", + "\r\n.", + "\r\n.\r", + "\r\n.\rx", + "\r\n.\rx\n", + "\r\n..\r\n", + "\n", + "\n.", + "\n.\r", + "\n.\rx", + "\n..\r\n" }; unsigned int i; @@ -123,6 +206,10 @@ test_istream_dot_one(&tests[i], FALSE, TRUE); test_istream_dot_one(&tests[i], FALSE, FALSE); } + for (i = 0; i < N_ELEMENTS(error_tests); i++) { + test_istream_dot_error(error_tests[i], FALSE); + test_istream_dot_error(error_tests[i], TRUE); + } test_end(); } From dovecot at dovecot.org Tue Apr 29 13:12:15 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Apr 2014 13:12:15 +0000 Subject: dovecot-2.2: lib-mail: istream-attachment-extractor handled mixe... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e9f4acfc7ce9 changeset: 17291:e9f4acfc7ce9 user: Timo Sirainen date: Tue Apr 29 16:03:17 2014 +0300 description: lib-mail: istream-attachment-extractor handled mixed LFs vs CRLFs correctly. Unless all the lines have the same newlines, the attachment can't be recreated back exactly the same. diffstat: src/lib-mail/istream-attachment-extractor.c | 12 +++++++++++- 1 files changed, 11 insertions(+), 1 deletions(-) diffs (29 lines): diff -r 59ada187a702 -r e9f4acfc7ce9 src/lib-mail/istream-attachment-extractor.c --- a/src/lib-mail/istream-attachment-extractor.c Tue Apr 29 13:57:44 2014 +0300 +++ b/src/lib-mail/istream-attachment-extractor.c Tue Apr 29 16:03:17 2014 +0300 @@ -148,6 +148,10 @@ static int astream_base64_decode_lf(struct attachment_istream_part *part) { + if (part->base64_have_crlf && part->base64_state != BASE64_STATE_CR) { + /* mixed LF vs CRLFs */ + return -1; + } part->base64_state = BASE64_STATE_0; if (part->cur_base64_blocks < part->base64_line_blocks) { /* last line */ @@ -212,7 +216,13 @@ case BASE64_STATE_CR: if (chr != '\n') return -1; - part->base64_have_crlf = TRUE; + if (!part->base64_have_crlf) { + if (part->base64_line_blocks != 0) { + /* mixed LF vs CRLFs */ + return -1; + } + part->base64_have_crlf = TRUE; + } return astream_base64_decode_lf(part); case BASE64_STATE_EOB: if (chr != '=') From dovecot at dovecot.org Tue Apr 29 13:16:09 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Apr 2014 13:16:09 +0000 Subject: dovecot-2.2: lib-mail: istream-attachment-extractor didn't corre... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c0455a1f650c changeset: 17292:c0455a1f650c user: Timo Sirainen date: Tue Apr 29 16:15:09 2014 +0300 description: lib-mail: istream-attachment-extractor didn't correctly handle increasing base64 line length. Based on patch by Pavel Stano diffstat: src/lib-mail/istream-attachment-extractor.c | 12 ++++++++++++ 1 files changed, 12 insertions(+), 0 deletions(-) diffs (29 lines): diff -r e9f4acfc7ce9 -r c0455a1f650c src/lib-mail/istream-attachment-extractor.c --- a/src/lib-mail/istream-attachment-extractor.c Tue Apr 29 16:03:17 2014 +0300 +++ b/src/lib-mail/istream-attachment-extractor.c Tue Apr 29 16:15:09 2014 +0300 @@ -208,6 +208,12 @@ } else if (chr == '=') { part->base64_state = BASE64_STATE_EOM; part->cur_base64_blocks++; + + if (part->cur_base64_blocks > part->base64_line_blocks && + part->base64_line_blocks > 0) { + /* too many blocks */ + return -1; + } return 0; } else { return -1; @@ -231,6 +237,12 @@ part->base64_bytes = part->temp_output->offset + pos + 1; part->base64_state = BASE64_STATE_EOM; part->cur_base64_blocks++; + + if (part->cur_base64_blocks > part->base64_line_blocks && + part->base64_line_blocks > 0) { + /* too many blocks */ + return -1; + } return 0; case BASE64_STATE_EOM: i_unreached(); From dovecot at dovecot.org Tue Apr 29 13:16:09 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Apr 2014 13:16:09 +0000 Subject: dovecot-2.2: lib-mail: test-istream-attachment updated Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dc4fd0ab5c14 changeset: 17293:dc4fd0ab5c14 user: Timo Sirainen date: Tue Apr 29 16:14:14 2014 +0300 description: lib-mail: test-istream-attachment updated diffstat: src/lib-mail/test-istream-attachment.c | 114 +++++++++++++++++++++++++++++++++ 1 files changed, 114 insertions(+), 0 deletions(-) diffs (138 lines): diff -r c0455a1f650c -r dc4fd0ab5c14 src/lib-mail/test-istream-attachment.c --- a/src/lib-mail/test-istream-attachment.c Tue Apr 29 16:15:09 2014 +0300 +++ b/src/lib-mail/test-istream-attachment.c Tue Apr 29 16:14:14 2014 +0300 @@ -59,6 +59,60 @@ "\r\n" "\r\n--bound--\r\n"; +static const char *mail_broken_input_body_prefix = +"MIME-Version: 1.0\r\n" +"Content-Type: multipart/alternative;\r\n boundary=\"bound\"\r\n" +"\r\n" +"--bound\r\n" +"Content-Transfer-Encoding: base64\r\n" +"Content-Type: text/plain\r\n" +"\r\n"; + +static const char *mail_broken_input_bodies[] = { + /* broken base64 input */ + "Zm9vCg=\n", + "Zm9vCg\n", + "Zm9vC\n", + /* extra whitespace */ + "Zm9v\n Zm9v\n", + "Zm9v \nZm9v\n", + /* mixed LF vs CRLFs */ + "Zm9vYmFy\r\nZm9vYmFy\n", + "Zm9vYmFy\nZm9vYmFy\r\n", + /* line length increases */ + "Zm9v\nZm9vYmFy\n", + "Zm9v\nZm9vCg==", + "Zm9v\nZm9vYgo=" +}; + +static const char *mail_nonbroken_input_bodies[] = { + /* suffixes with explicit '=' end */ + "Zm9vCg==", + "Zm9vCg==\n", + "Zm9vCg==\r\n", + "Zm9vCg==\nfoo\n", + "Zm9vCg==\r\nfoo\n", + "Zm9vCg== \t\t\n\n", + /* suffixes with shorter line length */ + "Zm9vYmFy\nZm9v\n", + "Zm9vYmFy\r\nZm9v\r\n", + "Zm9vYmFy\nZm9v\nfoo\n", + "Zm9vYmFy\r\nZm9v\r\nfoo\n", + "Zm9vYmFy\nZm9v\n \t\t\n\n", + /* suffixes with empty line */ + "Zm9v\n\n", + "Zm9v\r\n\r\n", + "Zm9v\n\nfoo\n" + "Zm9v\r\n\nfoo\n" + "Zm9v\r\n\r\nfoo\n" +#if 0 + /* the whitespace here could be handled as suffixes, but for now + they're not: */ + "Zm9v ", + "Zm9v \n" +#endif +}; + struct attachment { size_t buffer_offset; uoff_t start_offset; @@ -262,6 +316,65 @@ BINARY_TEXT_SHORT, strlen(BINARY_TEXT_SHORT)) == 0); i_stream_unref(&input); i_stream_unref(&datainput); + + if (attachment_data != NULL) + buffer_free(&attachment_data); + if (array_is_created(&attachments)) + array_free(&attachments); + test_end(); +} + +static bool test_istream_attachment_extractor_one(const char *body) +{ + const unsigned int prefix_len = strlen(mail_broken_input_body_prefix); + struct istream_attachment_settings set; + struct istream *datainput, *input; + char *mail_text; + const unsigned char *data; + size_t size; + int ret; + bool unchanged; + + mail_text = i_strconcat(mail_broken_input_body_prefix, body, NULL); + datainput = test_istream_create_data(mail_text, strlen(mail_text)); + + get_istream_attachment_settings(&set); + input = i_stream_create_attachment_extractor(datainput, &set, NULL); + + while ((ret = i_stream_read(input)) > 0) ; + + data = i_stream_get_data(input, &size); + i_assert(size >= prefix_len && + memcmp(data, mail_broken_input_body_prefix, prefix_len) == 0); + data += prefix_len; + size -= prefix_len; + + i_assert(attachment_data != NULL); + unchanged = attachment_data->used <= strlen(body) && + memcmp(attachment_data->data, body, attachment_data->used) == 0 && + strlen(body) - attachment_data->used == size && + memcmp(data, body + attachment_data->used, size) == 0; + + if (attachment_data != NULL) + buffer_free(&attachment_data); + if (array_is_created(&attachments)) + array_free(&attachments); + + i_stream_unref(&input); + i_stream_unref(&datainput); + i_free(mail_text); + return unchanged; +} + +static void test_istream_attachment_extractor(void) +{ + unsigned int i; + + test_begin("istream attachment extractor"); + for (i = 0; i < N_ELEMENTS(mail_broken_input_bodies); i++) + test_assert(test_istream_attachment_extractor_one(mail_broken_input_bodies[i])); + for (i = 0; i < N_ELEMENTS(mail_nonbroken_input_bodies); i++) + test_assert(!test_istream_attachment_extractor_one(mail_nonbroken_input_bodies[i])); test_end(); } @@ -288,6 +401,7 @@ { static void (*test_functions[])(void) = { test_istream_attachment, + test_istream_attachment_extractor, NULL }; if (argc > 1) From dovecot at dovecot.org Tue Apr 29 13:46:25 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Apr 2014 13:46:25 +0000 Subject: dovecot-2.2: maildir: Make sure we don't replace correct mail si... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/021f78ba15c1 changeset: 17294:021f78ba15c1 user: Timo Sirainen date: Tue Apr 29 16:44:58 2014 +0300 description: maildir: Make sure we don't replace correct mail sizes with broken ones when preserving GUID. diffstat: src/lib-storage/index/maildir/maildir-save.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (18 lines): diff -r dc4fd0ab5c14 -r 021f78ba15c1 src/lib-storage/index/maildir/maildir-save.c --- a/src/lib-storage/index/maildir/maildir-save.c Tue Apr 29 16:14:14 2014 +0300 +++ b/src/lib-storage/index/maildir/maildir-save.c Tue Apr 29 16:44:58 2014 +0300 @@ -851,10 +851,12 @@ mf->dest_basename) != NULL) { /* file already exists. give it another name. but preserve the size/vsize in the filename if possible */ - if (maildir_filename_get_size(mf->dest_basename, + if (mf->size == (uoff_t)-1 && + maildir_filename_get_size(mf->dest_basename, MAILDIR_EXTRA_FILE_SIZE, &size)) mf->size = size; - if (maildir_filename_get_size(mf->dest_basename, + if (mf->vsize == (uoff_t)-1 && + maildir_filename_get_size(mf->dest_basename, MAILDIR_EXTRA_VIRTUAL_SIZE, &size)) mf->vsize = size; From dovecot at dovecot.org Wed Apr 30 03:10:11 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Apr 2014 03:10:11 +0000 Subject: dovecot-2.2: maildir: Make sure the filename contains S=size and... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/494b5e18be58 changeset: 17295:494b5e18be58 user: Timo Sirainen date: Wed Apr 30 06:03:56 2014 +0300 description: maildir: Make sure the filename contains S=size and W=size even if GUID doesn't. diffstat: src/lib-storage/index/maildir/maildir-save.c | 26 ++++++++++++++++---------- 1 files changed, 16 insertions(+), 10 deletions(-) diffs (47 lines): diff -r 021f78ba15c1 -r 494b5e18be58 src/lib-storage/index/maildir/maildir-save.c --- a/src/lib-storage/index/maildir/maildir-save.c Tue Apr 29 16:44:58 2014 +0300 +++ b/src/lib-storage/index/maildir/maildir-save.c Wed Apr 30 06:03:56 2014 +0300 @@ -838,27 +838,33 @@ struct maildir_filename *mf, struct maildir_filename *prev_mf) { - uoff_t size; + uoff_t size, vsize; if (!ctx->locked_uidlist_refresh && ctx->locked) { (void)maildir_uidlist_refresh(ctx->mbox->uidlist); ctx->locked_uidlist_refresh = TRUE; } - if (!ctx->locked_uidlist_refresh || + if (!maildir_filename_get_size(mf->dest_basename, + MAILDIR_EXTRA_FILE_SIZE, &size)) + size = (uoff_t)-1; + if (!maildir_filename_get_size(mf->dest_basename, + MAILDIR_EXTRA_VIRTUAL_SIZE, &vsize)) + vsize = (uoff_t)-1; + + if (size != mf->size || vsize != mf->vsize || + !ctx->locked_uidlist_refresh || (prev_mf != NULL && maildir_filename_has_conflict(mf, prev_mf)) || maildir_uidlist_get_full_filename(ctx->mbox->uidlist, mf->dest_basename) != NULL) { - /* file already exists. give it another name. + /* a) dest_basename didn't contain the (correct) size/vsize. + they're required for good performance. + + b) file already exists. give it another name. but preserve the size/vsize in the filename if possible */ - if (mf->size == (uoff_t)-1 && - maildir_filename_get_size(mf->dest_basename, - MAILDIR_EXTRA_FILE_SIZE, &size)) + if (mf->size == (uoff_t)-1) mf->size = size; - if (mf->vsize == (uoff_t)-1 && - maildir_filename_get_size(mf->dest_basename, - MAILDIR_EXTRA_VIRTUAL_SIZE, - &size)) + if (mf->vsize == (uoff_t)-1) mf->vsize = size; mf->guid = mf->dest_basename; From dovecot at dovecot.org Wed Apr 30 03:10:12 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Apr 2014 03:10:12 +0000 Subject: dovecot-2.2: dsync: saved-date doesn't need to be looked up unti... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/99a4788770cb changeset: 17296:99a4788770cb user: Timo Sirainen date: Wed Apr 30 06:08:46 2014 +0300 description: dsync: saved-date doesn't need to be looked up until mail body is being read. This should improve the performance when the saved-date isn't already cached or otherwise quickly accessible. This change also makes dsync slightly incompatible with earlier versions. When using dsync with an earlier version the saved-dates aren't synced. It would be too much trouble to try to preserve full backwards compatibility, especially because saved-date doesn't matter so much and isn't even visible to IMAP clients. diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 22 +++++++++++----------- src/doveadm/dsync/dsync-mail.c | 5 ++++- src/doveadm/dsync/dsync-mail.h | 3 +-- src/doveadm/dsync/dsync-mailbox-export.c | 11 +++-------- src/doveadm/dsync/dsync-mailbox-import.c | 3 ++- 5 files changed, 21 insertions(+), 23 deletions(-) diffs (175 lines): diff -r 494b5e18be58 -r 99a4788770cb src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Wed Apr 30 06:03:56 2014 +0300 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Wed Apr 30 06:08:46 2014 +0300 @@ -109,7 +109,7 @@ { .name = "mail_change", .chr = 'C', .required_keys = "type uid", - .optional_keys = "guid hdr_hash modseq pvt_modseq save_timestamp " + .optional_keys = "guid hdr_hash modseq pvt_modseq " "add_flags remove_flags final_flags " "keywords_reset keyword_changes" }, @@ -119,7 +119,7 @@ }, { .name = "mail", .chr = 'M', - .optional_keys = "guid uid pop3_uidl pop3_order received_date stream" + .optional_keys = "guid uid pop3_uidl pop3_order received_date saved_date stream" }, { .name = "mailbox_cache_field", .chr = 'c', @@ -1508,10 +1508,6 @@ dsync_serializer_encode_add(encoder, "pvt_modseq", dec2str(change->pvt_modseq)); } - if (change->save_timestamp != 0) { - dsync_serializer_encode_add(encoder, "save_timestamp", - dec2str(change->save_timestamp)); - } if (change->add_flags != 0) { dsync_serializer_encode_add(encoder, "add_flags", t_strdup_printf("%x", change->add_flags)); @@ -1601,11 +1597,6 @@ dsync_ibc_input_error(ibc, decoder, "Invalid pvt_modseq"); return DSYNC_IBC_RECV_RET_TRYAGAIN; } - if (dsync_deserializer_decode_try(decoder, "save_timestamp", &value) && - str_to_time(value, &change->save_timestamp) < 0) { - dsync_ibc_input_error(ibc, decoder, "Invalid save_timestamp"); - return DSYNC_IBC_RECV_RET_TRYAGAIN; - } if (dsync_deserializer_decode_try(decoder, "add_flags", &value)) change->add_flags = strtoul(value, NULL, 16); @@ -1709,6 +1700,10 @@ dsync_serializer_encode_add(encoder, "received_date", dec2str(mail->received_date)); } + if (mail->saved_date != 0) { + dsync_serializer_encode_add(encoder, "saved_date", + dec2str(mail->saved_date)); + } if (mail->input != NULL) dsync_serializer_encode_add(encoder, "stream", ""); @@ -1770,6 +1765,11 @@ dsync_ibc_input_error(ibc, decoder, "Invalid received_date"); return DSYNC_IBC_RECV_RET_TRYAGAIN; } + if (dsync_deserializer_decode_try(decoder, "saved_date", &value) && + str_to_time(value, &mail->saved_date) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid saved_date"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } if (dsync_deserializer_decode_try(decoder, "stream", &value)) { mail->input = dsync_ibc_stream_input_stream(ibc); if (dsync_ibc_stream_read_mail_stream(ibc) <= 0) { diff -r 494b5e18be58 -r 99a4788770cb src/doveadm/dsync/dsync-mail.c --- a/src/doveadm/dsync/dsync-mail.c Wed Apr 30 06:03:56 2014 +0300 +++ b/src/doveadm/dsync/dsync-mail.c Wed Apr 30 06:08:46 2014 +0300 @@ -97,6 +97,10 @@ *error_field_r = "received-date"; return -1; } + if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) { + *error_field_r = "saved-date"; + return -1; + } return 0; } @@ -133,7 +137,6 @@ dest_r->hdr_hash = p_strdup(pool, src->hdr_hash); dest_r->modseq = src->modseq; dest_r->pvt_modseq = src->pvt_modseq; - dest_r->save_timestamp = src->save_timestamp; dest_r->add_flags = src->add_flags; dest_r->remove_flags = src->remove_flags; diff -r 494b5e18be58 -r 99a4788770cb src/doveadm/dsync/dsync-mail.h --- a/src/doveadm/dsync/dsync-mail.h Wed Apr 30 06:03:56 2014 +0300 +++ b/src/doveadm/dsync/dsync-mail.h Wed Apr 30 06:08:46 2014 +0300 @@ -14,6 +14,7 @@ const char *pop3_uidl; unsigned int pop3_order; time_t received_date; + time_t saved_date; /* Input stream containing the message text, or NULL if all instances of the message were already expunged from this mailbox. */ @@ -60,8 +61,6 @@ /* Message's current private modseq (for private flags in shared mailboxes, otherwise 0) */ uint64_t pvt_modseq; - /* Message's save timestamp (saves) */ - time_t save_timestamp; /* List of flag/keyword changes: (saves, flag changes) */ diff -r 494b5e18be58 -r 99a4788770cb src/doveadm/dsync/dsync-mailbox-export.c --- a/src/doveadm/dsync/dsync-mailbox-export.c Wed Apr 30 06:03:56 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Wed Apr 30 06:08:46 2014 +0300 @@ -270,24 +270,18 @@ { struct dsync_mail_change *change; const char *guid, *hdr_hash; - time_t save_timestamp; int ret; /* update wanted fields in case we didn't already set them for the search */ - mail_add_temp_wanted_fields(mail, MAIL_FETCH_GUID | - MAIL_FETCH_SAVE_DATE, + mail_add_temp_wanted_fields(mail, MAIL_FETCH_GUID, exporter->wanted_headers); /* If message is already expunged here, just skip it */ if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) <= 0) return ret; - if (mail_get_save_date(mail, &save_timestamp) < 0) - return dsync_mail_error(exporter, mail, "save-date"); change = export_save_change_get(exporter, mail->uid); - change->save_timestamp = save_timestamp; - change->guid = *guid == '\0' ? "" : p_strdup(exporter->pool, guid); change->hdr_hash = p_strdup(exporter->pool, hdr_hash); @@ -364,7 +358,7 @@ if (exporter->last_common_uid == 0) { /* we're syncing all mails, so we can request the wanted fields for all the mails */ - wanted_fields = MAIL_FETCH_GUID | MAIL_FETCH_SAVE_DATE; + wanted_fields = MAIL_FETCH_GUID; wanted_headers = exporter->wanted_headers; } @@ -727,6 +721,7 @@ MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_POP3_ORDER | MAIL_FETCH_RECEIVED_DATE | + MAIL_FETCH_SAVE_DATE | MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY, NULL); mail_search_args_unref(&search_args); diff -r 494b5e18be58 -r 99a4788770cb src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Wed Apr 30 06:03:56 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Wed Apr 30 06:08:46 2014 +0300 @@ -1872,7 +1872,6 @@ if (keywords != NULL) mailbox_keywords_unref(&keywords); - mailbox_save_set_save_date(save_ctx, change->save_timestamp); if (change->modseq > 1) { (void)mailbox_enable(importer->box, MAILBOX_FEATURE_CONDSTORE); mailbox_save_set_min_modseq(save_ctx, change->modseq); @@ -1915,6 +1914,8 @@ mailbox_save_set_uid(save_ctx, newmail->final_uid); if (*mail->guid != '\0') mailbox_save_set_guid(save_ctx, mail->guid); + if (mail->saved_date != 0) + mailbox_save_set_save_date(save_ctx, mail->saved_date); dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change); if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0') mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl); From dovecot at dovecot.org Wed Apr 30 14:38:28 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Apr 2014 14:38:28 +0000 Subject: dovecot-2.2: lib-storage: Partially reverted the mail.get_real_m... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5c4205a95028 changeset: 17297:5c4205a95028 user: Timo Sirainen date: Wed Apr 30 17:37:43 2014 +0300 description: lib-storage: Partially reverted the mail.get_real_mail() API change for backwards compatibility. I hadn't realized Pigeonhole was also using it. (I thought it was using only the mail_get_real_mail() public API.) This fixes Pigeonhole v0.4.2 to work again. diffstat: src/lib-storage/fail-mail.c | 5 ++--- src/lib-storage/index/index-mail.c | 5 ++--- src/lib-storage/index/index-mail.h | 2 +- src/lib-storage/mail-storage-private.h | 5 ++++- src/lib-storage/mail.c | 3 ++- src/plugins/virtual/virtual-mail.c | 11 ++++++----- 6 files changed, 17 insertions(+), 14 deletions(-) diffs (98 lines): diff -r 99a4788770cb -r 5c4205a95028 src/lib-storage/fail-mail.c --- a/src/lib-storage/fail-mail.c Wed Apr 30 06:08:46 2014 +0300 +++ b/src/lib-storage/fail-mail.c Wed Apr 30 17:37:43 2014 +0300 @@ -196,10 +196,9 @@ return -1; } -static int fail_mail_get_real_mail(struct mail *mail, struct mail **real_mail_r) +static struct mail * fail_mail_get_real_mail(struct mail *mail) { - *real_mail_r = mail; - return 0; + return mail; } static void diff -r 99a4788770cb -r 5c4205a95028 src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Wed Apr 30 06:08:46 2014 +0300 +++ b/src/lib-storage/index/index-mail.c Wed Apr 30 17:37:43 2014 +0300 @@ -1211,10 +1211,9 @@ } } -int index_mail_get_real_mail(struct mail *mail, struct mail **real_mail_r) +struct mail *index_mail_get_real_mail(struct mail *mail) { - *real_mail_r = mail; - return 0; + return mail; } struct mail * diff -r 99a4788770cb -r 5c4205a95028 src/lib-storage/index/index-mail.h --- a/src/lib-storage/index/index-mail.h Wed Apr 30 06:08:46 2014 +0300 +++ b/src/lib-storage/index/index-mail.h Wed Apr 30 17:37:43 2014 +0300 @@ -216,7 +216,7 @@ struct istream **stream_r); int index_mail_get_special(struct mail *_mail, enum mail_fetch_field field, const char **value_r); -int index_mail_get_real_mail(struct mail *mail, struct mail **real_mail_r); +struct mail *index_mail_get_real_mail(struct mail *mail); void index_mail_update_flags(struct mail *mail, enum modify_type modify_type, enum mail_flags flags); diff -r 99a4788770cb -r 5c4205a95028 src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Wed Apr 30 06:08:46 2014 +0300 +++ b/src/lib-storage/mail-storage-private.h Wed Apr 30 17:37:43 2014 +0300 @@ -391,7 +391,10 @@ int (*get_special)(struct mail *mail, enum mail_fetch_field field, const char **value_r); - int (*get_real_mail)(struct mail *mail, struct mail **real_mail_r); + /* FIXME: v2.3 API should change this to return -1 on failure. + for now NULL means failure so we don't break backwards + compatibility. */ + struct mail *(*get_real_mail)(struct mail *mail); void (*update_flags)(struct mail *mail, enum modify_type modify_type, enum mail_flags flags); diff -r 99a4788770cb -r 5c4205a95028 src/lib-storage/mail.c --- a/src/lib-storage/mail.c Wed Apr 30 06:08:46 2014 +0300 +++ b/src/lib-storage/mail.c Wed Apr 30 17:37:43 2014 +0300 @@ -276,7 +276,8 @@ { struct mail_private *p = (struct mail_private *)mail; - return p->v.get_real_mail(mail, real_mail_r); + *real_mail_r = p->v.get_real_mail(mail); + return *real_mail_r == NULL ? -1 : 0; } struct mail *mail_get_real_mail(struct mail *mail) diff -r 99a4788770cb -r 5c4205a95028 src/plugins/virtual/virtual-mail.c --- a/src/plugins/virtual/virtual-mail.c Wed Apr 30 06:08:46 2014 +0300 +++ b/src/plugins/virtual/virtual-mail.c Wed Apr 30 17:37:43 2014 +0300 @@ -430,16 +430,17 @@ return 0; } -static int -virtual_mail_get_real_mail(struct mail *mail, struct mail **real_mail_r) +static struct mail *virtual_mail_get_real_mail(struct mail *mail) { struct virtual_mail *vmail = (struct virtual_mail *)mail; - struct mail *backend_mail; + struct mail *backend_mail, *real_mail; if (backend_mail_get(vmail, &backend_mail) < 0) - return -1; + return NULL; - return mail_get_backend_mail(backend_mail, real_mail_r); + if (mail_get_backend_mail(backend_mail, &real_mail) < 0) + return NULL; + return real_mail; } static void virtual_mail_update_pop3_uidl(struct mail *mail, const char *uidl) From dovecot at dovecot.org Wed Apr 30 15:47:24 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Apr 2014 15:47:24 +0000 Subject: dovecot-2.2: lib-storage: mail_never_cache_fields=* means now to... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c1719de47d5e changeset: 17298:c1719de47d5e user: Timo Sirainen date: Wed Apr 30 18:46:55 2014 +0300 description: lib-storage: mail_never_cache_fields=* means now to disable all caching. diffstat: src/lib-storage/index/index-mail.c | 3 ++- src/lib-storage/index/index-storage.c | 7 +++++++ src/lib-storage/mail-storage-private.h | 2 ++ 3 files changed, 11 insertions(+), 1 deletions(-) diffs (42 lines): diff -r 5c4205a95028 -r c1719de47d5e src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Wed Apr 30 17:37:43 2014 +0300 +++ b/src/lib-storage/index/index-mail.c Wed Apr 30 18:46:55 2014 +0300 @@ -512,7 +512,8 @@ } if (!mail->data.no_caching && - mail->data.dont_cache_field_idx != field_idx) { + mail->data.dont_cache_field_idx != field_idx && + !_mail->box->mail_cache_disabled) { mail_cache_add(_mail->transaction->cache_trans, _mail->seq, field_idx, data, data_size); } diff -r 5c4205a95028 -r c1719de47d5e src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Wed Apr 30 17:37:43 2014 +0300 +++ b/src/lib-storage/index/index-storage.c Wed Apr 30 18:46:55 2014 +0300 @@ -71,6 +71,13 @@ sizeof(global_cache_fields)); mail_cache_register_fields(cache, ibox->cache_fields, MAIL_INDEX_CACHE_FIELD_COUNT); + + if (strcmp(set->mail_never_cache_fields, "*") == 0) { + /* all caching disabled for now */ + box->mail_cache_disabled = TRUE; + return; + } + set_cache_decisions(cache, "mail_cache_fields", set->mail_cache_fields, MAIL_CACHE_DECISION_TEMP); diff -r 5c4205a95028 -r c1719de47d5e src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Wed Apr 30 17:37:43 2014 +0300 +++ b/src/lib-storage/mail-storage-private.h Wed Apr 30 18:46:55 2014 +0300 @@ -343,6 +343,8 @@ unsigned int disallow_new_keywords:1; /* Mailbox has been synced at least once */ unsigned int synced:1; + /* Updating cache file is disabled */ + unsigned int mail_cache_disabled:1; }; struct mail_vfuncs { From pigeonhole at rename-it.nl Wed Apr 30 22:07:36 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 01 May 2014 00:07:36 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: editheader: Adjusted to chang... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/378362cb36a6 changeset: 1850:378362cb36a6 user: Stephan Bosch date: Thu May 01 00:07:27 2014 +0200 description: lib-sieve: editheader: Adjusted to change in Dovecot lib-storage mail API. diffstat: src/lib-sieve/util/edit-mail.c | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-) diffs (20 lines): diff -r 439b1b2fe05d -r 378362cb36a6 src/lib-sieve/util/edit-mail.c --- a/src/lib-sieve/util/edit-mail.c Tue Apr 29 02:13:35 2014 +0200 +++ b/src/lib-sieve/util/edit-mail.c Thu May 01 00:07:27 2014 +0200 @@ -1379,13 +1379,12 @@ return edmail->wrapped->v.get_special(&edmail->wrapped->mail, field, value_r); } -static int -edit_mail_get_real_mail(struct mail *mail, struct mail **real_mail_r) +static struct mail * +edit_mail_get_real_mail(struct mail *mail) { struct edit_mail *edmail = (struct edit_mail *)mail; - *real_mail_r = edit_mail_get_mail(edmail); - return 0; + return edit_mail_get_mail(edmail); } static void edit_mail_update_flags