From dovecot at dovecot.org Sun May 3 13:28:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 May 2015 13:28:34 +0000 Subject: dovecot-2.2: lib-mail: Added qp-decoder, which is a rewritten qu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a9f8a617dc02 changeset: 18504:a9f8a617dc02 user: Timo Sirainen date: Sun May 03 14:50:01 2015 +0300 description: lib-mail: Added qp-decoder, which is a rewritten quoted_printable_decode() The main benefit is that qp-decoder allows feeding data to it in smaller pieces. It can also give better error reporting. diffstat: src/lib-mail/Makefile.am | 7 + src/lib-mail/qp-decoder.c | 285 +++++++++++++++++++++++++++++++++++++++++ src/lib-mail/qp-decoder.h | 19 ++ src/lib-mail/test-qp-decoder.c | 95 +++++++++++++ 4 files changed, 406 insertions(+), 0 deletions(-) diffs (truncated from 449 to 300 lines): diff -r 333533e2d231 -r a9f8a617dc02 src/lib-mail/Makefile.am --- a/src/lib-mail/Makefile.am Wed Apr 29 17:45:30 2015 +0200 +++ b/src/lib-mail/Makefile.am Sun May 03 14:50:01 2015 +0300 @@ -31,6 +31,7 @@ message-size.c \ message-snippet.c \ ostream-dot.c \ + qp-decoder.c \ quoted-printable.c \ rfc2231-parser.c \ rfc822-parser.c @@ -65,6 +66,7 @@ message-size.h \ message-snippet.h \ ostream-dot.h \ + qp-decoder.h \ quoted-printable.h \ rfc2231-parser.h \ rfc822-parser.h @@ -91,6 +93,7 @@ test-message-part \ test-message-snippet \ test-ostream-dot \ + test-qp-decoder \ test-quoted-printable \ test-rfc2231-parser @@ -181,6 +184,10 @@ test_ostream_dot_LDADD = ostream-dot.lo $(test_libs) test_ostream_dot_DEPENDENCIES = $(test_deps) +test_qp_decoder_SOURCES = test-qp-decoder.c +test_qp_decoder_LDADD = qp-decoder.lo $(test_libs) +test_qp_decoder_DEPENDENCIES = $(test_deps) + test_quoted_printable_SOURCES = test-quoted-printable.c test_quoted_printable_LDADD = quoted-printable.lo $(test_libs) test_quoted_printable_DEPENDENCIES = $(test_deps) diff -r 333533e2d231 -r a9f8a617dc02 src/lib-mail/qp-decoder.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-mail/qp-decoder.c Sun May 03 14:50:01 2015 +0300 @@ -0,0 +1,285 @@ +/* Copyright (c) 2002-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "hex-binary.h" +#include "qp-decoder.h" + +/* quoted-printable lines can be max 76 characters. if we've seen more than + that much whitespace, it means there really shouldn't be anything else left + in the line except trailing whitespace. */ +#define QP_MAX_WHITESPACE_LEN 76 + +#define QP_IS_TRAILING_WHITESPACE(c) \ + ((c) == ' ' || (c) == '\t') + +enum qp_state { + STATE_TEXT = 0, + STATE_WHITESPACE, + STATE_EQUALS, + STATE_EQUALS_WHITESPACE, + STATE_HEX2, + STATE_CR, + STATE_SOFTCR +}; + +struct qp_decoder { + buffer_t *dest; + buffer_t *whitespace; + enum qp_state state; + char hexchar; +}; + +struct qp_decoder *qp_decoder_init(buffer_t *dest) +{ + struct qp_decoder *qp; + + qp = i_new(struct qp_decoder, 1); + qp->dest = dest; + qp->whitespace = buffer_create_dynamic(default_pool, 80); + return qp; +} + +void qp_decoder_deinit(struct qp_decoder **_qp) +{ + struct qp_decoder *qp = *_qp; + + buffer_free(&qp->whitespace); + i_free(qp); +} + +static size_t +qp_decoder_more_text(struct qp_decoder *qp, const unsigned char *src, + size_t src_size) +{ + size_t i, start = 0, ret = src_size; + + for (i = 0; i < src_size; i++) { + if (src[i] > '=') { + /* fast path */ + continue; + } + switch (src[i]) { + case '=': + qp->state = STATE_EQUALS; + break; + case '\r': + qp->state = STATE_CR; + break; + case '\n': + /* LF without preceding CR */ + buffer_append(qp->dest, src+start, i-start); + buffer_append(qp->dest, "\r\n", 2); + start = i+1; + continue; + case ' ': + case '\t': + i_assert(qp->whitespace->used == 0); + qp->state = STATE_WHITESPACE; + buffer_append_c(qp->whitespace, src[i]); + break; + default: + continue; + } + ret = i+1; + break; + } + buffer_append(qp->dest, src+start, i-start); + return ret; +} + +static void qp_decoder_invalid(struct qp_decoder *qp, const char **error_r) +{ + switch (qp->state) { + case STATE_EQUALS: + buffer_append_c(qp->dest, '='); + *error_r = "'=' not followed by two hex digits"; + break; + case STATE_HEX2: + buffer_append_c(qp->dest, '='); + buffer_append_c(qp->dest, qp->hexchar); + *error_r = "'=' not followed by a hex digit"; + break; + case STATE_EQUALS_WHITESPACE: + buffer_append_c(qp->dest, '='); + buffer_append_buf(qp->dest, qp->whitespace, 0, (size_t)-1); + buffer_set_used_size(qp->whitespace, 0); + *error_r = "'=' not followed by newline"; + break; + case STATE_CR: + buffer_append_buf(qp->dest, qp->whitespace, 0, (size_t)-1); + buffer_set_used_size(qp->whitespace, 0); + buffer_append_c(qp->dest, '\r'); + *error_r = "CR not followed by LF"; + break; + case STATE_SOFTCR: + buffer_append_c(qp->dest, '='); + buffer_append_buf(qp->dest, qp->whitespace, 0, (size_t)-1); + buffer_set_used_size(qp->whitespace, 0); + buffer_append_c(qp->dest, '\r'); + *error_r = "CR not followed by LF"; + break; + case STATE_TEXT: + case STATE_WHITESPACE: + i_unreached(); + } + qp->state = STATE_TEXT; + i_assert(*error_r != NULL); +} + +int qp_decoder_more(struct qp_decoder *qp, const unsigned char *src, + size_t src_size, size_t *invalid_src_pos_r, + const char **error_r) +{ + const char *error; + size_t i; + + *invalid_src_pos_r = (size_t)-1; + *error_r = NULL; + + for (i = 0; i < src_size; ) { + switch (qp->state) { + case STATE_TEXT: + i += qp_decoder_more_text(qp, src+i, src_size-i); + /* don't increment i any more than we already did, + so continue instead of break */ + continue; + case STATE_WHITESPACE: + if (QP_IS_TRAILING_WHITESPACE(src[i])) { + /* more whitespace */ + if (qp->whitespace->used <= QP_MAX_WHITESPACE_LEN) + buffer_append_c(qp->whitespace, src[i]); + } else if (src[i] == '\r') { + qp->state = STATE_CR; + } else if (src[i] == '\n') { + /* drop the trailing whitespace */ + buffer_append(qp->dest, "\r\n", 2); + buffer_set_used_size(qp->whitespace, 0); + } else { + /* this wasn't trailing whitespace. + put it back. */ + buffer_append_buf(qp->dest, qp->whitespace, + 0, (size_t)-1); + if (qp->whitespace->used > QP_MAX_WHITESPACE_LEN) { + /* we already truncated some of the + whitespace away, because the line + is too long */ + if (*invalid_src_pos_r == (size_t)-1) { + *invalid_src_pos_r = i; + *error_r = "Too much whitespace"; + } + } + buffer_set_used_size(qp->whitespace, 0); + qp->state = STATE_TEXT; + continue; /* don't increment i */ + } + break; + case STATE_EQUALS: + if ((src[i] >= '0' && src[i] <= '9') || + (src[i] >= 'A' && src[i] <= 'F') || + /* lowercase hex isn't strictly valid, but allow */ + (src[i] >= 'a' && src[i] <= 'f')) { + qp->hexchar = src[i]; + qp->state = STATE_HEX2; + } else if (QP_IS_TRAILING_WHITESPACE(src[i])) { + i_assert(qp->whitespace->used == 0); + buffer_append_c(qp->whitespace, src[i]); + qp->state = STATE_EQUALS_WHITESPACE; + } else if (src[i] == '\r') + qp->state = STATE_SOFTCR; + else if (src[i] == '\n') { + qp->state = STATE_TEXT; + } else { + /* invalid input */ + qp_decoder_invalid(qp, &error); + if (*invalid_src_pos_r == (size_t)-1) { + *invalid_src_pos_r = i; + *error_r = error; + } + continue; /* don't increment i */ + } + break; + case STATE_HEX2: + if ((src[i] >= '0' && src[i] <= '9') || + (src[i] >= 'A' && src[i] <= 'F') || + (src[i] >= 'a' && src[i] <= 'f')) { + char data[3]; + + data[0] = qp->hexchar; + data[1] = src[i]; + data[2] = '\0'; + if (hex_to_binary(data, qp->dest) < 0) + i_unreached(); + qp->state = STATE_TEXT; + } else { + /* invalid input */ + qp_decoder_invalid(qp, &error); + if (*invalid_src_pos_r == (size_t)-1) { + *invalid_src_pos_r = i; + *error_r = error; + } + continue; /* don't increment i */ + } + break; + case STATE_EQUALS_WHITESPACE: + if (QP_IS_TRAILING_WHITESPACE(src[i])) { + if (qp->whitespace->used <= QP_MAX_WHITESPACE_LEN) + buffer_append_c(qp->whitespace, src[i]); + else { + /* if this isn't going to get truncated + anyway, it's going to be an error */ + } + } else if (src[i] == '\r') + qp->state = STATE_SOFTCR; + else if (src[i] == '\n') { + buffer_set_used_size(qp->whitespace, 0); + qp->state = STATE_TEXT; + } else { + /* = not followed by [CR]LF + is invalid. */ + qp_decoder_invalid(qp, &error); + if (*invalid_src_pos_r == (size_t)-1) { + *invalid_src_pos_r = i; + *error_r = error; + } + continue; /* don't increment i */ + } + break; + case STATE_CR: + case STATE_SOFTCR: + if (src[i] == '\n') { + buffer_set_used_size(qp->whitespace, 0); + if (qp->state != STATE_SOFTCR) + buffer_append(qp->dest, "\r\n", 2); + qp->state = STATE_TEXT; + } else { + qp_decoder_invalid(qp, &error); + if (*invalid_src_pos_r == (size_t)-1) { + *invalid_src_pos_r = i; From dovecot at dovecot.org Sun May 3 13:28:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 May 2015 13:28:36 +0000 Subject: dovecot-2.2: lib-mail: Fixed various istream-qp-decoder bugs by ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a8e9fdcb17c5 changeset: 18505:a8e9fdcb17c5 user: Timo Sirainen date: Sun May 03 16:22:55 2015 +0300 description: lib-mail: Fixed various istream-qp-decoder bugs by switching to qp-decoder API diffstat: src/lib-mail/Makefile.am | 2 +- src/lib-mail/istream-qp-decoder.c | 149 +++++++++++++------------------- src/lib-mail/test-istream-qp-decoder.c | 42 ++++++-- 3 files changed, 89 insertions(+), 104 deletions(-) diffs (truncated from 309 to 300 lines): diff -r a9f8a617dc02 -r a8e9fdcb17c5 src/lib-mail/Makefile.am --- a/src/lib-mail/Makefile.am Sun May 03 14:50:01 2015 +0300 +++ b/src/lib-mail/Makefile.am Sun May 03 16:22:55 2015 +0300 @@ -110,7 +110,7 @@ test_istream_dot_DEPENDENCIES = $(test_deps) test_istream_qp_decoder_SOURCES = test-istream-qp-decoder.c -test_istream_qp_decoder_LDADD = istream-qp-decoder.lo quoted-printable.lo $(test_libs) +test_istream_qp_decoder_LDADD = istream-qp-decoder.lo qp-decoder.lo $(test_libs) test_istream_qp_decoder_DEPENDENCIES = $(test_deps) message_parser_objects = \ diff -r a9f8a617dc02 -r a8e9fdcb17c5 src/lib-mail/istream-qp-decoder.c --- a/src/lib-mail/istream-qp-decoder.c Sun May 03 14:50:01 2015 +0300 +++ b/src/lib-mail/istream-qp-decoder.c Sun May 03 16:22:55 2015 +0300 @@ -3,78 +3,28 @@ #include "lib.h" #include "buffer.h" #include "hex-binary.h" +#include "qp-decoder.h" #include "istream-private.h" -#include "quoted-printable.h" #include "istream-qp.h" struct qp_decoder_istream { struct istream_private istream; + buffer_t *buf; + struct qp_decoder *qp; }; -static int -i_stream_read_parent(struct istream_private *stream, size_t *prev_size) +static void i_stream_qp_decoder_close(struct iostream_private *stream, + bool close_parent) { - size_t size; - ssize_t ret; + struct qp_decoder_istream *bstream = + (struct qp_decoder_istream *)stream; - size = i_stream_get_data_size(stream->parent); - if (size >= 4 && size != *prev_size) { - *prev_size = size; - return 1; - } - - ret = i_stream_read(stream->parent); - if (ret <= 0) { - stream->istream.stream_errno = stream->parent->stream_errno; - stream->istream.eof = stream->parent->eof; - return ret; - } - *prev_size = i_stream_get_data_size(stream->parent); - return 1; -} - -static int -i_stream_qp_try_decode_input(struct qp_decoder_istream *bstream, bool eof) -{ - struct istream_private *stream = &bstream->istream; - const unsigned char *data; - size_t size, avail, buffer_avail, pos; - buffer_t buf; - int ret; - - data = i_stream_get_data(stream->parent, &size); - if (size == 0) - return 0; - - /* normally the decoded quoted-printable content can't be larger than - the encoded content, but because we always use CRLFs, it may use - twice as much space by only converting LFs to CRLFs. */ - i_stream_try_alloc(stream, size, &avail); - buffer_avail = stream->buffer_size - stream->pos; - - if (size > buffer_avail/2) { - /* can't fit everything to destination buffer. - write as much as we can. */ - size = buffer_avail/2; - if (size == 0) - return -2; - } - - buffer_create_from_data(&buf, stream->w_buffer + stream->pos, - buffer_avail); - ret = !eof ? quoted_printable_decode(data, size, &pos, &buf) : - quoted_printable_decode_final(data, size, &pos, &buf); - if (ret < 0) { - io_stream_set_error(&stream->iostream, - "Invalid quoted-printable data: 0x%s", - binary_to_hex(data+pos, I_MAX(size-pos, 8))); - stream->istream.stream_errno = EINVAL; - return -1; - } - - stream->pos += buf.used; - i_stream_skip(stream->parent, pos); - return pos > 0 ? 1 : 0; + if (bstream->qp != NULL) + qp_decoder_deinit(&bstream->qp); + if (bstream->buf != NULL) + buffer_free(&bstream->buf); + if (close_parent) + i_stream_close(bstream->istream.parent); } static ssize_t i_stream_qp_decoder_read(struct istream_private *stream) @@ -82,49 +32,63 @@ struct qp_decoder_istream *bstream = (struct qp_decoder_istream *)stream; const unsigned char *data; - size_t pre_count, post_count, size; + size_t size, avail, error_pos; + const char *error; int ret; - size_t prev_size = 0; - do { - ret = i_stream_read_parent(stream, &prev_size); + for (;;) { + /* if something is already decoded, return as much of it as + we can */ + if (bstream->buf->used > 0) { + i_stream_try_alloc(stream, bstream->buf->used, &avail); + if (avail == 0) + return -2; + size = I_MIN(avail, bstream->buf->used); + memcpy(stream->w_buffer + stream->pos, + bstream->buf->data, size); + buffer_delete(bstream->buf, 0, size); + stream->pos += size; + return size; + } + /* need to read more input */ + ret = i_stream_read_data(stream->parent, &data, &size, 0); if (ret <= 0) { + stream->istream.stream_errno = stream->parent->stream_errno; + stream->istream.eof = stream->parent->eof; if (ret != -1 || stream->istream.stream_errno != 0) - return 0; - - ret = i_stream_qp_try_decode_input(bstream, TRUE); - if (ret == 0) { - /* ended with =[whitespace] but without LF */ - stream->istream.eof = TRUE; + return ret; + /* end of quoted-printable stream. verify that the + ending is ok. */ + if (qp_decoder_finish(bstream->qp, &error) == 0) { + i_assert(bstream->buf->used == 0); return -1; } - /* partial qp input */ - i_assert(ret < 0); - data = i_stream_get_data(stream->parent, &size); io_stream_set_error(&stream->iostream, - "quoted-printable input ends with a partial block: 0x%s", - binary_to_hex(data, size)); + "Invalid quoted-printable input trailer: %s", error); stream->istream.stream_errno = EINVAL; return -1; } - - /* encode as much data as fits into destination buffer */ - pre_count = stream->pos - stream->skip; - while ((ret = i_stream_qp_try_decode_input(bstream, FALSE)) > 0) ; - post_count = stream->pos - stream->skip; - } while (ret == 0 && pre_count == post_count); - - if (ret < 0) - return ret; - - i_assert(post_count > pre_count); - return post_count - pre_count; + if (qp_decoder_more(bstream->qp, data, size, + &error_pos, &error) < 0) { + i_assert(error_pos < size); + io_stream_set_error(&stream->iostream, + "Invalid quoted-printable input 0x%s: %s", + binary_to_hex(data+error_pos, I_MIN(size-error_pos, 8)), error); + stream->istream.stream_errno = EINVAL; + return -1; + } + i_stream_skip(stream->parent, size); + } } static void i_stream_qp_decoder_seek(struct istream_private *stream, uoff_t v_offset, bool mark) { + struct qp_decoder_istream *bstream = + (struct qp_decoder_istream *)stream; + const char *error; + if (v_offset < stream->istream.v_offset) { /* seeking backwards - go back to beginning and seek forward from there. */ @@ -132,6 +96,8 @@ stream->skip = stream->pos = 0; stream->istream.v_offset = 0; i_stream_seek(stream->parent, 0); + (void)qp_decoder_finish(bstream->qp, &error); + buffer_set_used_size(bstream->buf, 0); } i_stream_default_seek_nonseekable(stream, v_offset, mark); } @@ -142,7 +108,10 @@ bstream = i_new(struct qp_decoder_istream, 1); bstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + bstream->buf = buffer_create_dynamic(default_pool, 128); + bstream->qp = qp_decoder_init(bstream->buf); + bstream->istream.iostream.close = i_stream_qp_decoder_close; bstream->istream.read = i_stream_qp_decoder_read; bstream->istream.seek = i_stream_qp_decoder_seek; diff -r a9f8a617dc02 -r a8e9fdcb17c5 src/lib-mail/test-istream-qp-decoder.c --- a/src/lib-mail/test-istream-qp-decoder.c Sun May 03 14:50:01 2015 +0300 +++ b/src/lib-mail/test-istream-qp-decoder.c Sun May 03 16:22:55 2015 +0300 @@ -5,58 +5,74 @@ #include "istream-private.h" #include "istream-qp.h" -static const -struct { +static const struct { const char *input; const char *output; + int ret; } tests[] = { - { "p=C3=A4=C3=A4t=C3=B6s", "p??t?s" }, - { "p=c3=a4=c3=a4t=c3=b6s= ", "p??t?s" }, - { "p=c3=a4=c3=a4t=c3=b6s= \n", "p??t?s" }, - { "p=c3=a4= \t \n=c3=\r\n=a4t= \r\n=c3=b6s", "p??t?s" }, + { "p=C3=A4=C3=A4t=C3=B6s", "p??t?s", 0 }, + { "p=c3=a4=c3=a4t=c3=b6s= \n", "p??t?s", 0 }, + { "p=c3=a4= \t \n=c3=\r\n=a4t= \r\n=c3=b6s", "p??t?s", 0 }, + + { "p=c3=a4\rasdf", "p?", -1 }, + { "p=c", "p", -1 }, + { "p=A", "p", -1 }, + { "p=Ax", "p", -1 }, + { "p=c3=a4=c3=a4t=c3=b6s= ", "p??t?s", -1 } }; static void -decode_test(const char *qp_input, const char *output, bool broken_input) +decode_test(const char *qp_input, const char *output, bool broken_input, + unsigned int buffer_size) { unsigned int qp_input_len = strlen(qp_input); struct istream *input_data, *input; const unsigned char *data; size_t i, size; + string_t *str = t_str_new(32); int ret = 0; input_data = test_istream_create_data(qp_input, qp_input_len); + test_istream_set_max_buffer_size(input_data, buffer_size); test_istream_set_allow_eof(input_data, FALSE); input = i_stream_create_qp_decoder(input_data); for (i = 1; i <= qp_input_len; i++) { test_istream_set_size(input_data, i); - while ((ret = i_stream_read(input)) > 0) ; + while ((ret = i_stream_read_data(input, &data, &size, 0)) > 0) { + str_append_n(str, data, size); + i_stream_skip(input, size); + } if (ret == -1 && broken_input) break; test_assert(ret == 0); } if (ret == 0) { test_istream_set_allow_eof(input_data, TRUE); - while ((ret = i_stream_read(input)) > 0) ; + while ((ret = i_stream_read_data(input, &data, &size, 0)) > 0) { + str_append_n(str, data, size); + i_stream_skip(input, size); + } } test_assert(ret == -1); test_assert((input->stream_errno == 0 && !broken_input) || (input->stream_errno == EINVAL && broken_input)); - data = i_stream_get_data(input, &size); - test_assert(size == strlen(output) && memcmp(data, output, size) == 0); + test_assert(strcmp(str_c(str), output) == 0); i_stream_unref(&input); i_stream_unref(&input_data); } static void test_istream_qp_decoder(void) { - unsigned int i; + unsigned int i, j; for (i = 0; i < N_ELEMENTS(tests); i++) { From dovecot at dovecot.org Sun May 3 13:28:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 May 2015 13:28:36 +0000 Subject: dovecot-2.2: lib-mail: Switched message-decoder to use qp-decoder. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b10aebbb42df changeset: 18506:b10aebbb42df user: Timo Sirainen date: Sun May 03 16:23:41 2015 +0300 description: lib-mail: Switched message-decoder to use qp-decoder. This probably doesn't fix any actual bugs, but allows getting rid of the quoted_printable_decode*() diffstat: src/lib-mail/Makefile.am | 2 +- src/lib-mail/message-decoder.c | 33 ++++++++++++++++++++------------- src/lib-mail/test-message-decoder.c | 1 - 3 files changed, 21 insertions(+), 15 deletions(-) diffs (111 lines): diff -r a8e9fdcb17c5 -r b10aebbb42df src/lib-mail/Makefile.am --- a/src/lib-mail/Makefile.am Sun May 03 16:22:55 2015 +0300 +++ b/src/lib-mail/Makefile.am Sun May 03 16:23:41 2015 +0300 @@ -145,7 +145,7 @@ test_message_date_DEPENDENCIES = $(test_deps) test_message_decoder_SOURCES = test-message-decoder.c -test_message_decoder_LDADD = message-decoder.lo quoted-printable.lo rfc822-parser.lo rfc2231-parser.lo ../lib-charset/libcharset.la $(test_libs) +test_message_decoder_LDADD = message-decoder.lo qp-decoder.lo quoted-printable.lo rfc822-parser.lo rfc2231-parser.lo ../lib-charset/libcharset.la $(test_libs) test_message_decoder_DEPENDENCIES = ../lib-charset/libcharset.la $(test_deps) test_message_header_decode_SOURCES = test-message-header-decode.c diff -r a8e9fdcb17c5 -r b10aebbb42df src/lib-mail/message-decoder.c --- a/src/lib-mail/message-decoder.c Sun May 03 16:22:55 2015 +0300 +++ b/src/lib-mail/message-decoder.c Sun May 03 16:23:41 2015 +0300 @@ -6,7 +6,7 @@ #include "str.h" #include "unichar.h" #include "charset-utf8.h" -#include "quoted-printable.h" +#include "qp-decoder.h" #include "rfc822-parser.h" #include "rfc2231-parser.h" #include "message-parser.h" @@ -29,6 +29,7 @@ char translation_buf[CHARSET_MAX_PENDING_BUF_SIZE]; unsigned int translation_size; + struct qp_decoder *qp; buffer_t *encoding_buf; char *content_type, *content_charset; @@ -64,6 +65,8 @@ if (ctx->charset_trans != NULL) charset_to_utf8_end(&ctx->charset_trans); + if (ctx->qp != NULL) + qp_decoder_deinit(&ctx->qp); buffer_free(&ctx->encoding_buf); buffer_free(&ctx->buf); @@ -265,12 +268,11 @@ { const unsigned char *data = NULL; size_t pos = 0, size = 0; + const char *error; int ret; - if (ctx->encoding_buf->used != 0) { - /* @UNSAFE */ + if (ctx->encoding_buf->used != 0) buffer_append(ctx->encoding_buf, input->data, input->size); - } switch (ctx->message_cte) { case MESSAGE_CTE_UNKNOWN: @@ -279,22 +281,23 @@ case MESSAGE_CTE_78BIT: case MESSAGE_CTE_BINARY: + i_assert(ctx->encoding_buf->used == 0); data = input->data; size = pos = input->size; break; - case MESSAGE_CTE_QP: + case MESSAGE_CTE_QP: { + i_assert(ctx->encoding_buf->used == 0); buffer_set_used_size(ctx->buf, 0); - if (ctx->encoding_buf->used != 0) { - (void)quoted_printable_decode(ctx->encoding_buf->data, - ctx->encoding_buf->used, - &pos, ctx->buf); - } else { - (void)quoted_printable_decode(input->data, input->size, - &pos, ctx->buf); - } + if (ctx->qp == NULL) + ctx->qp = qp_decoder_init(ctx->buf); + (void)qp_decoder_more(ctx->qp, input->data, input->size, + &pos, &error); data = ctx->buf->data; size = ctx->buf->used; + /* eat away all input. qp-decoder buffers it internally. */ + pos = input->size; break; + } case MESSAGE_CTE_BASE64: buffer_set_used_size(ctx->buf, 0); if (ctx->encoding_buf->used != 0) { @@ -386,6 +389,10 @@ void message_decoder_decode_reset(struct message_decoder_context *ctx) { + const char *error; + + if (ctx->qp != NULL) + (void)qp_decoder_finish(ctx->qp, &error); i_free_and_null(ctx->content_type); i_free_and_null(ctx->content_charset); ctx->message_cte = MESSAGE_CTE_78BIT; diff -r a8e9fdcb17c5 -r b10aebbb42df src/lib-mail/test-message-decoder.c --- a/src/lib-mail/test-message-decoder.c Sun May 03 16:22:55 2015 +0300 +++ b/src/lib-mail/test-message-decoder.c Sun May 03 16:23:41 2015 +0300 @@ -3,7 +3,6 @@ #include "lib.h" #include "buffer.h" #include "charset-utf8.h" -#include "quoted-printable.h" #include "message-parser.h" #include "message-header-decode.h" #include "message-decoder.h" From dovecot at dovecot.org Sun May 3 13:28:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 May 2015 13:28:46 +0000 Subject: dovecot-2.2: lib-mail: Removed quoted_printable_decode*() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/780a8f4544ec changeset: 18507:780a8f4544ec user: Timo Sirainen date: Sun May 03 16:26:46 2015 +0300 description: lib-mail: Removed quoted_printable_decode*() quoted_printable_q_decode() was still left in quoted-printable.h, but maybe it should be moved to qp-decoder.c as well. diffstat: src/lib-mail/quoted-printable.c | 110 ----------------------------------- src/lib-mail/quoted-printable.h | 13 ---- src/lib-mail/test-quoted-printable.c | 88 ---------------------------- 3 files changed, 0 insertions(+), 211 deletions(-) diffs (248 lines): diff -r b10aebbb42df -r 780a8f4544ec src/lib-mail/quoted-printable.c --- a/src/lib-mail/quoted-printable.c Sun May 03 16:23:41 2015 +0300 +++ b/src/lib-mail/quoted-printable.c Sun May 03 16:26:46 2015 +0300 @@ -5,116 +5,6 @@ #include "hex-binary.h" #include "quoted-printable.h" -#define QP_IS_TRAILING_SPACE(c) \ - ((c) == ' ' || (c) == '\t') - -static int -qp_is_end_of_line(const unsigned char *src, size_t *src_pos, size_t size) -{ - size_t i = *src_pos; - - i_assert(src[i] == '='); - for (i++; i < size; i++) { - if (QP_IS_TRAILING_SPACE(src[i]) || src[i] == '\r') - continue; - - if (src[i] != '\n') - return 0; - - *src_pos = i; - return 1; - } - return -1; -} - -static int -quoted_printable_decode_full(const unsigned char *src, size_t src_size, - size_t *src_pos_r, buffer_t *dest, bool eof) -{ - char hexbuf[3]; - size_t src_pos, pos, next; - bool errors = FALSE; - int ret; - - hexbuf[2] = '\0'; - - next = 0; - for (src_pos = 0; src_pos < src_size; src_pos++) { - if (src[src_pos] != '=' && src[src_pos] != '\n') - continue; - - if (src[src_pos] == '\n') { - /* drop trailing whitespace */ - pos = src_pos; - if (pos > 0 && src[pos-1] == '\r') - pos--; - while (pos > 0 && QP_IS_TRAILING_SPACE(src[pos-1])) - pos--; - buffer_append(dest, src + next, pos - next); - next = src_pos+1; - buffer_append_c(dest, '\r'); - buffer_append_c(dest, '\n'); - continue; - } - - /* '=' */ - buffer_append(dest, src + next, src_pos - next); - next = src_pos; - - if ((ret = qp_is_end_of_line(src, &src_pos, src_size)) > 0) { - /* =[whitespace][\r]\n */ - next = src_pos+1; - continue; - } - if (ret < 0) { - /* '=' was followed only by whitespace */ - break; - } - if (src_pos+2 >= src_size) { - /* '=' was followed by non-whitespace */ - if (eof) - errors = TRUE; - break; - } - - /* = */ - hexbuf[0] = src[src_pos+1]; - hexbuf[1] = src[src_pos+2]; - - if (hex_to_binary(hexbuf, dest) == 0) { - src_pos += 2; - next = src_pos + 1; - } else { - /* non-hex data, show as-is */ - errors = TRUE; - next = src_pos; - } - } - if (src_pos == src_size) { - /* add everything but trailing spaces */ - if (src_pos > 0 && src[src_pos-1] == '\r') - src_pos--; - while (src_pos > 0 && QP_IS_TRAILING_SPACE(src[src_pos-1])) - src_pos--; - buffer_append(dest, src + next, src_pos - next); - next = src_pos; - } - *src_pos_r = next; - return errors ? -1 : 0; -} - -int quoted_printable_decode(const unsigned char *src, size_t src_size, - size_t *src_pos_r, buffer_t *dest) -{ - return quoted_printable_decode_full(src, src_size, src_pos_r, dest, FALSE); -} - -int quoted_printable_decode_final(const unsigned char *src, size_t src_size, - size_t *src_pos_r, buffer_t *dest) -{ - return quoted_printable_decode_full(src, src_size, src_pos_r, dest, TRUE); -} - int quoted_printable_q_decode(const unsigned char *src, size_t src_size, buffer_t *dest) { diff -r b10aebbb42df -r 780a8f4544ec src/lib-mail/quoted-printable.h --- a/src/lib-mail/quoted-printable.h Sun May 03 16:23:41 2015 +0300 +++ b/src/lib-mail/quoted-printable.h Sun May 03 16:26:46 2015 +0300 @@ -1,19 +1,6 @@ #ifndef QUOTED_PRINTABLE_H #define QUOTED_PRINTABLE_H -/* Translates quoted printable data into binary. dest must be at least the - size of src, and may be same as src. Returns 0 if input was valid, -1 if - there were some decoding errors (which were skipped over). LFs without - preceding CR are returned as CRLF (but =0A isn't). - - This function may be called multiple times for parsing the same stream. - src_pos is updated to first non-translated character in src. */ -int quoted_printable_decode(const unsigned char *src, size_t src_size, - size_t *src_pos_r, buffer_t *dest); -/* Like quoted_printable_decode(), but handle src as the final block. - This allows src to end without LF. */ -int quoted_printable_decode_final(const unsigned char *src, size_t src_size, - size_t *src_pos_r, buffer_t *dest); /* Decode MIME "Q" encoding. */ int quoted_printable_q_decode(const unsigned char *src, size_t src_size, buffer_t *dest); diff -r b10aebbb42df -r 780a8f4544ec src/lib-mail/test-quoted-printable.c --- a/src/lib-mail/test-quoted-printable.c Sun May 03 16:23:41 2015 +0300 +++ b/src/lib-mail/test-quoted-printable.c Sun May 03 16:26:46 2015 +0300 @@ -6,92 +6,6 @@ #include "quoted-printable.h" #include "test-common.h" -struct test_quoted_printable_decode_data { - const char *input; - const char *output; - int end_skip; - int ret; -}; - -static void test_quoted_printable_decode(void) -{ - static struct test_quoted_printable_decode_data data[] = { - { "foo \r\nbar=", "foo\r\nbar", 1, 0 }, - { "foo\t=\nbar", "foo\tbar", 0, 0 }, - { "foo = \n=01", "foo \001", 0, 0 }, - { "foo =\t\r\nbar", "foo bar", 0, 0 }, - { "foo =\r\n=01", "foo \001", 0, 0 }, - { "foo \nbar=", "foo\r\nbar", 1, 0 }, - { "=0A=0D ", "\n\r", 2, 0 }, - { "foo_bar", "foo_bar", 0, 0 }, - { "foo=", "foo", 1, 0 }, - { "foo= ", "foo", 3, 0 }, - { "foo=A", "foo", 2, 0 }, - { "foo=Ax", "foo=Ax", 0, -1 }, - { "foo=Ax=xy", "foo=Ax=xy", 0, -1 } - }; - buffer_t *buf; - unsigned int i, start, end, len; - size_t src_pos; - int ret; - - test_begin("quoted printable decode"); - buf = buffer_create_dynamic(pool_datastack_create(), 128); - for (i = 0; i < N_ELEMENTS(data); i++) { - len = strlen(data[i].input); - ret = quoted_printable_decode((const void *)data[i].input, len, - &src_pos, buf); - test_assert(ret == data[i].ret); - test_assert(src_pos + data[i].end_skip == len); - test_assert(strcmp(data[i].output, str_c(buf)) == 0); - - buffer_set_used_size(buf, 0); - for (start = 0, end = 1; end <= len; ) { - quoted_printable_decode(CONST_PTR_OFFSET(data[i].input, start), - end - start, &src_pos, buf); - src_pos += start; - start = src_pos; - if (src_pos <= end) - end++; - else - end = src_pos + 1; - } - test_assert(src_pos + data[i].end_skip == len); - test_assert(strcmp(data[i].output, str_c(buf)) == 0); - buffer_set_used_size(buf, 0); - } - test_end(); -} - -static void test_quoted_printable_decode_final(void) -{ - static struct test_quoted_printable_decode_data data[] = { - { "=0A=0D ", "\n\r", 2, 0 }, - { "foo=", "foo", 1, 0 }, - { "foo ", "foo", 2, 0 }, - { "foo= ", "foo", 3, 0 }, - { "foo=A", "foo", 2, -1 } - }; - buffer_t *buf; - unsigned int i, len; - size_t src_pos; - int ret; - - test_begin("quoted printable decode final"); - buf = buffer_create_dynamic(pool_datastack_create(), 128); - for (i = 0; i < N_ELEMENTS(data); i++) { - len = strlen(data[i].input); - ret = quoted_printable_decode_final((const void *)data[i].input, - len, &src_pos, buf); - test_assert(ret == data[i].ret); - test_assert(src_pos + data[i].end_skip == len); - test_assert(strcmp(data[i].output, str_c(buf)) == 0); - - buffer_set_used_size(buf, 0); - } - test_end(); -} - static void test_quoted_printable_q_decode(void) { const char *data[] = { @@ -119,8 +33,6 @@ int main(void) { static void (*test_functions[])(void) = { - test_quoted_printable_decode, - test_quoted_printable_decode_final, test_quoted_printable_q_decode, NULL }; From dovecot at dovecot.org Sun May 3 14:11:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 May 2015 14:11:50 +0000 Subject: dovecot-2.2: acl: Log more debug info with mail_debug=yes Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a0b2217b1eab changeset: 18508:a0b2217b1eab user: Timo Sirainen date: Sun May 03 17:07:02 2015 +0300 description: acl: Log more debug info with mail_debug=yes diffstat: src/plugins/acl/acl-backend-vfile.c | 3 ++- src/plugins/acl/acl-global-file.c | 9 ++++++++- src/plugins/acl/acl-global-file.h | 3 ++- 3 files changed, 12 insertions(+), 3 deletions(-) diffs (61 lines): diff -r 780a8f4544ec -r a0b2217b1eab src/plugins/acl/acl-backend-vfile.c --- a/src/plugins/acl/acl-backend-vfile.c Sun May 03 16:26:46 2015 +0300 +++ b/src/plugins/acl/acl-backend-vfile.c Sun May 03 17:07:02 2015 +0300 @@ -63,7 +63,8 @@ } } else if (!S_ISDIR(st.st_mode)) { _backend->global_file = - acl_global_file_init(backend->global_path, backend->cache_secs); + acl_global_file_init(backend->global_path, backend->cache_secs, + _backend->debug); } } if (_backend->debug) { diff -r 780a8f4544ec -r a0b2217b1eab src/plugins/acl/acl-global-file.c --- a/src/plugins/acl/acl-global-file.c Sun May 03 16:26:46 2015 +0300 +++ b/src/plugins/acl/acl-global-file.c Sun May 03 17:07:02 2015 +0300 @@ -30,16 +30,19 @@ ARRAY(struct acl_global_rights) rights; unsigned int refresh_interval_secs; + bool debug; }; struct acl_global_file * -acl_global_file_init(const char *path, unsigned int refresh_interval_secs) +acl_global_file_init(const char *path, unsigned int refresh_interval_secs, + bool debug) { struct acl_global_file *file; file = i_new(struct acl_global_file, 1); file->path = i_strdup(path); file->refresh_interval_secs = refresh_interval_secs; + file->debug = debug; i_array_init(&file->rights, 32); file->rights_pool = pool_alloconly_create("acl global file rights", 1024); return file; @@ -198,6 +201,10 @@ array_foreach_modifiable(&file->rights, global_rights) { if (!wildcard_match(vname, global_rights->vpattern)) continue; + if (file->debug) { + i_debug("Mailbox '%s' matches global ACL pattern '%s'", + vname, global_rights->vpattern); + } array_foreach(&global_rights->rights, rights) { new_rights = array_append_space(rights_r); acl_rights_dup(rights, pool, new_rights); diff -r 780a8f4544ec -r a0b2217b1eab src/plugins/acl/acl-global-file.h --- a/src/plugins/acl/acl-global-file.h Sun May 03 16:26:46 2015 +0300 +++ b/src/plugins/acl/acl-global-file.h Sun May 03 17:07:02 2015 +0300 @@ -4,7 +4,8 @@ #include "acl-api.h" struct acl_global_file * -acl_global_file_init(const char *path, unsigned int refresh_interval_secs); +acl_global_file_init(const char *path, unsigned int refresh_interval_secs, + bool debug); void acl_global_file_deinit(struct acl_global_file **file); /* Read the global ACLs into memory. */ From pigeonhole at rename-it.nl Sun May 3 22:23:53 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 04 May 2015 00:23:53 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: util: program client: Fixed h... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/05f8ce7b5c2e changeset: 2040:05f8ce7b5c2e user: Stephan Bosch date: Mon May 04 00:22:09 2015 +0200 description: lib-sieve: util: program client: Fixed handling of connecting asynchronously. diffstat: src/lib-sieve/util/program-client-remote.c | 2 +- src/lib-sieve/util/program-client.c | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diffs (40 lines): diff -r dd430dfa2ddb -r 05f8ce7b5c2e src/lib-sieve/util/program-client-remote.c --- a/src/lib-sieve/util/program-client-remote.c Thu Apr 23 02:04:38 2015 +0200 +++ b/src/lib-sieve/util/program-client-remote.c Mon May 04 00:22:09 2015 +0200 @@ -239,7 +239,7 @@ !pclient->output_seekable ? -1 : fd ); pclient->fd_out = fd; pclient->io = io_add(fd, IO_WRITE, program_client_remote_connected, pclient); - return 1; + return 0; } static int program_client_remote_close_output(struct program_client *pclient) diff -r dd430dfa2ddb -r 05f8ce7b5c2e src/lib-sieve/util/program-client.c --- a/src/lib-sieve/util/program-client.c Thu Apr 23 02:04:38 2015 +0200 +++ b/src/lib-sieve/util/program-client.c Mon May 04 00:22:09 2015 +0200 @@ -490,13 +490,11 @@ pclient->exit_code = 0; pclient->error = PROGRAM_CLIENT_ERROR_NONE; - pclient->ioloop = io_loop_create(); if ( program_client_connect(pclient) >= 0 ) { /* run output */ - ret = 1; - if ( pclient->program_output != NULL && + if ( ret > 0 && pclient->program_output != NULL && (ret=o_stream_flush(pclient->program_output)) == 0 ) { o_stream_set_flush_callback (pclient->program_output, program_client_program_output, pclient); @@ -505,7 +503,8 @@ /* run i/o event loop */ if ( ret < 0 ) { pclient->error = PROGRAM_CLIENT_ERROR_IO; - } else if ( ret == 0 || program_client_input_pending(pclient) ) { + } else if ( !pclient->disconnected && + (ret == 0 || program_client_input_pending(pclient)) ) { io_loop_run(pclient->ioloop); } From pigeonhole at rename-it.nl Sun May 3 22:30:31 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 04 May 2015 00:30:31 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: util: program client: Fixed o... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/1eb0362461f0 changeset: 2041:1eb0362461f0 user: Stephan Bosch date: Mon May 04 00:26:58 2015 +0200 description: lib-sieve: util: program client: Fixed omission in previous change. diffstat: src/lib-sieve/util/program-client.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 05f8ce7b5c2e -r 1eb0362461f0 src/lib-sieve/util/program-client.c --- a/src/lib-sieve/util/program-client.c Mon May 04 00:22:09 2015 +0200 +++ b/src/lib-sieve/util/program-client.c Mon May 04 00:26:58 2015 +0200 @@ -492,7 +492,7 @@ pclient->ioloop = io_loop_create(); - if ( program_client_connect(pclient) >= 0 ) { + if ( (ret=program_client_connect(pclient)) >= 0 ) { /* run output */ if ( ret > 0 && pclient->program_output != NULL && (ret=o_stream_flush(pclient->program_output)) == 0 ) { From dovecot at dovecot.org Mon May 4 08:56:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 04 May 2015 08:56:59 +0000 Subject: dovecot-2.2: man: Fixed -Dv parameter position in doveadm-sync.1 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e9cf0ac3a629 changeset: 18509:e9cf0ac3a629 user: Timo Sirainen date: Mon May 04 11:55:11 2015 +0300 description: man: Fixed -Dv parameter position in doveadm-sync.1 diffstat: doc/man/doveadm-sync.1.in | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (27 lines): diff -r a0b2217b1eab -r e9cf0ac3a629 doc/man/doveadm-sync.1.in --- a/doc/man/doveadm-sync.1.in Sun May 03 17:07:02 2015 +0300 +++ b/doc/man/doveadm-sync.1.in Mon May 04 11:55:11 2015 +0300 @@ -1,12 +1,12 @@ .\" Copyright (c) 2014-2015 Dovecot authors, see the included COPYING file -.TH DOVEADM\-SYNC 1 "2014-10-19" "Dovecot v2.2" "Dovecot" +.TH DOVEADM\-SYNC 1 "2015-05-04" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-sync \- Dovecot\(aqs two\-way mailbox synchronization utility .br doveadm\-backup \- Dovecot\(aqs one\-way mailbox synchronization utility .\"------------------------------------------------------------------------ .SH SYNOPSIS -.BR "doveadm sync" " [" \-Dv ] +.BR doveadm " [" \-Dv "] " sync [\fB\-u\fP \fIuser\fP|\fB\-A\fP] [\fB\-S\fP \fIsocket_path\fP] .RB [ \-1fPRU ] @@ -20,7 +20,7 @@ \fB\-d\fP|\fIdestination\fP .\"------------------------------------- .PP -.BR "doveadm backup" " [" \-Dv ] +.BR doveadm " [" \-Dv "] " backup [\fB\-u\fP \fIuser\fP|\fB\-A\fP] [\fB\-S\fP \fIsocket_path\fP] .RB [ \-fPRU ] From dovecot at dovecot.org Mon May 4 09:09:04 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 04 May 2015 09:09:04 +0000 Subject: dovecot-2.2: configure: When compiling with sqlite3, don't forci... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e4ad83ed88c9 changeset: 18510:e4ad83ed88c9 user: Timo Sirainen date: Mon May 04 12:07:16 2015 +0300 description: configure: When compiling with sqlite3, don't forcibly link with zlib. Looks like -lz was originally added when it was copy&pasted from some other code. Hopefully this isn't actually needed by any (current) system. diffstat: configure.ac | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r e9cf0ac3a629 -r e4ad83ed88c9 configure.ac --- a/configure.ac Mon May 04 11:55:11 2015 +0300 +++ b/configure.ac Mon May 04 12:07:16 2015 +0300 @@ -2304,7 +2304,7 @@ if test $want_sqlite != no; then AC_CHECK_LIB(sqlite3, sqlite3_open, [ AC_CHECK_HEADER(sqlite3.h, [ - SQLITE_LIBS="$SQLITE_LIBS -lsqlite3 -lz" + SQLITE_LIBS="$SQLITE_LIBS -lsqlite3" AC_DEFINE(HAVE_SQLITE,, Build with SQLite3 support) found_sql_drivers="$found_sql_drivers sqlite" From dovecot at dovecot.org Mon May 4 15:16:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 04 May 2015 15:16:50 +0000 Subject: dovecot-2.2: imapc: imapc-features=rfc822.size didn't work for f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8f20aa806bcc changeset: 18511:8f20aa806bcc user: Timo Sirainen date: Mon May 04 18:15:00 2015 +0300 description: imapc: imapc-features=rfc822.size didn't work for fetching mail's virtual size. Physical and virtual sizes are the same for imapc. diffstat: src/lib-storage/index/imapc/imapc-mail.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r e4ad83ed88c9 -r 8f20aa806bcc src/lib-storage/index/imapc/imapc-mail.c --- a/src/lib-storage/index/imapc/imapc-mail.c Mon May 04 12:07:16 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-mail.c Mon May 04 18:15:00 2015 +0300 @@ -532,7 +532,7 @@ index_mail_get_date, imapc_mail_get_received_date, imapc_mail_get_save_date, - index_mail_get_virtual_size, + imapc_mail_get_physical_size, imapc_mail_get_physical_size, imapc_mail_get_first_header, imapc_mail_get_headers, From dovecot at dovecot.org Mon May 4 15:33:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 04 May 2015 15:33:17 +0000 Subject: dovecot-2.2: imapc: Various optimization fixes to fetching messa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d350a23207c2 changeset: 18512:d350a23207c2 user: Timo Sirainen date: Mon May 04 18:31:27 2015 +0300 description: imapc: Various optimization fixes to fetching messages' virtual size. diffstat: src/lib-storage/index/imapc/imapc-mail-fetch.c | 11 ++++++----- src/lib-storage/index/imapc/imapc-mail.c | 25 ++++++++++++++++++------- 2 files changed, 24 insertions(+), 12 deletions(-) diffs (96 lines): diff -r 8f20aa806bcc -r d350a23207c2 src/lib-storage/index/imapc/imapc-mail-fetch.c --- a/src/lib-storage/index/imapc/imapc-mail-fetch.c Mon May 04 18:15:00 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c Mon May 04 18:31:27 2015 +0300 @@ -212,7 +212,7 @@ str_printfa(str, "UID FETCH %u (", _mail->uid); if ((fields & MAIL_FETCH_RECEIVED_DATE) != 0) str_append(str, "INTERNALDATE "); - if ((fields & MAIL_FETCH_PHYSICAL_SIZE) != 0) + if ((fields & (MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE)) != 0) str_append(str, "RFC822.SIZE "); if ((fields & MAIL_FETCH_GUID) != 0) { str_append(str, mbox->guid_fetch_field_name); @@ -296,10 +296,11 @@ if ((data->wanted_fields & MAIL_FETCH_SAVE_DATE) != 0 && data->save_date == (time_t)-1 && data->received_date == (time_t)-1) fields |= MAIL_FETCH_RECEIVED_DATE; - if ((data->wanted_fields & MAIL_FETCH_PHYSICAL_SIZE) != 0 && + if ((data->wanted_fields & (MAIL_FETCH_PHYSICAL_SIZE | + MAIL_FETCH_VIRTUAL_SIZE)) != 0 && data->physical_size == (uoff_t)-1 && IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) - fields |= MAIL_FETCH_PHYSICAL_SIZE; + fields |= MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE; if ((data->wanted_fields & MAIL_FETCH_GUID) != 0 && data->guid == NULL && mbox->guid_fetch_field_name != NULL) fields |= MAIL_FETCH_GUID; @@ -343,10 +344,10 @@ return FALSE; fields &= ~MAIL_FETCH_RECEIVED_DATE; } - if ((fields & MAIL_FETCH_PHYSICAL_SIZE) != 0) { + if ((fields & (MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE)) != 0) { if (imail->imail.data.physical_size == (uoff_t)-1) return FALSE; - fields &= ~MAIL_FETCH_PHYSICAL_SIZE; + fields &= ~(MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE); } if ((fields & MAIL_FETCH_GUID) != 0) { if (imail->imail.data.guid == NULL) diff -r 8f20aa806bcc -r d350a23207c2 src/lib-storage/index/imapc/imapc-mail.c --- a/src/lib-storage/index/imapc/imapc-mail.c Mon May 04 18:15:00 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-mail.c Mon May 04 18:31:27 2015 +0300 @@ -125,12 +125,11 @@ uoff_t old_offset; int ret; - if (data->physical_size == (uoff_t)-1) { + if (data->physical_size == (uoff_t)-1) (void)index_mail_get_physical_size(_mail, size_r); - if (data->physical_size != (uoff_t)-1) { - *size_r = data->physical_size; - return 0; - } + if (data->physical_size != (uoff_t)-1) { + *size_r = data->physical_size; + return 0; } if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE) && @@ -164,6 +163,17 @@ return 0; } +static int imapc_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r) +{ + struct index_mail *mail = (struct index_mail *)_mail; + struct index_mail_data *data = &mail->data; + + if (imapc_mail_get_physical_size(_mail, size_r) < 0) + return -1; + data->virtual_size = data->physical_size; + return 0; +} + static int imapc_mail_get_header_stream(struct mail *_mail, struct mailbox_header_lookup_ctx *headers, @@ -302,7 +312,8 @@ data->save_date = data->received_date; } } - if ((data->wanted_fields & MAIL_FETCH_PHYSICAL_SIZE) != 0) { + if ((data->wanted_fields & (MAIL_FETCH_PHYSICAL_SIZE | + MAIL_FETCH_VIRTUAL_SIZE)) != 0) { if (index_mail_get_physical_size(_mail, &size) < 0 && !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) data->access_part |= READ_HDR | READ_BODY; @@ -532,7 +543,7 @@ index_mail_get_date, imapc_mail_get_received_date, imapc_mail_get_save_date, - imapc_mail_get_physical_size, + imapc_mail_get_virtual_size, imapc_mail_get_physical_size, imapc_mail_get_first_header, imapc_mail_get_headers, From dovecot at dovecot.org Mon May 4 15:46:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 04 May 2015 15:46:28 +0000 Subject: dovecot-2.2: imap: FETCH BODY.PEEK[HEADER.FIELDS (..)] didn't se... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8c49fb6d789b changeset: 18513:8c49fb6d789b user: Timo Sirainen date: Mon May 04 18:44:29 2015 +0300 description: imap: FETCH BODY.PEEK[HEADER.FIELDS (..)] didn't set wanted_headers optimization. Primarily this fixes imapc's prefetching. diffstat: src/imap/imap-fetch-body.c | 1 + src/lib-imap-storage/imap-msgpart.c | 12 ++++++++++++ src/lib-imap-storage/imap-msgpart.h | 4 ++++ 3 files changed, 17 insertions(+), 0 deletions(-) diffs (47 lines): diff -r d350a23207c2 -r 8c49fb6d789b src/imap/imap-fetch-body.c --- a/src/imap/imap-fetch-body.c Mon May 04 18:31:27 2015 +0300 +++ b/src/imap/imap-fetch-body.c Mon May 04 18:44:29 2015 +0300 @@ -346,6 +346,7 @@ } ctx->fetch_ctx->fetch_data |= imap_msgpart_get_fetch_data(body->msgpart); + imap_msgpart_get_wanted_headers(body->msgpart, &ctx->fetch_ctx->all_headers); if (body_parse_partial(body, p, &error) < 0) { ctx->error = p_strdup_printf(ctx->pool, diff -r d350a23207c2 -r 8c49fb6d789b src/lib-imap-storage/imap-msgpart.c --- a/src/lib-imap-storage/imap-msgpart.c Mon May 04 18:31:27 2015 +0300 +++ b/src/lib-imap-storage/imap-msgpart.c Mon May 04 18:44:29 2015 +0300 @@ -333,6 +333,18 @@ return msgpart->wanted_fields; } +void imap_msgpart_get_wanted_headers(struct imap_msgpart *msgpart, + ARRAY_TYPE(const_string) *headers) +{ + unsigned int i; + + if (msgpart->fetch_type != FETCH_HEADER_FIELDS) + return; + + for (i = 0; msgpart->headers[i] != NULL; i++) + array_append(headers, &msgpart->headers[i], 1); +} + static int imap_msgpart_get_partial_header(struct mail *mail, struct istream *mail_input, const struct imap_msgpart *msgpart, diff -r d350a23207c2 -r 8c49fb6d789b src/lib-imap-storage/imap-msgpart.h --- a/src/lib-imap-storage/imap-msgpart.h Mon May 04 18:31:27 2015 +0300 +++ b/src/lib-imap-storage/imap-msgpart.h Mon May 04 18:44:29 2015 +0300 @@ -36,6 +36,10 @@ uoff_t imap_msgpart_get_partial_size(struct imap_msgpart *msgpart); /* Return wanted_fields mask. */ enum mail_fetch_field imap_msgpart_get_fetch_data(struct imap_msgpart *msgpart); +/* Append all the specifically requested headers to the headers array + (no deduplication is done) */ +void imap_msgpart_get_wanted_headers(struct imap_msgpart *msgpart, + ARRAY_TYPE(const_string) *headers); /* Open message part refenced by IMAP section as istream. Returns 0 if successful, -1 if storage error. Returned istream is initially referenced, From dovecot at dovecot.org Mon May 4 16:08:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 04 May 2015 16:08:20 +0000 Subject: dovecot-2.2: lib-fs: Added fs_ref() and fs_unref() for reference... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0b297bc94746 changeset: 18514:0b297bc94746 user: Timo Sirainen date: Mon May 04 19:06:11 2015 +0300 description: lib-fs: Added fs_ref() and fs_unref() for reference counting. diffstat: src/lib-fs/fs-api-private.h | 1 + src/lib-fs/fs-api.c | 21 +++++++++++++++++++-- src/lib-fs/fs-api.h | 4 ++++ 3 files changed, 24 insertions(+), 2 deletions(-) diffs (72 lines): diff -r 8c49fb6d789b -r 0b297bc94746 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Mon May 04 18:44:29 2015 +0300 +++ b/src/lib-fs/fs-api-private.h Mon May 04 19:06:11 2015 +0300 @@ -69,6 +69,7 @@ const char *name; struct fs_vfuncs v; char *temp_path_prefix; + int refcount; char *username, *session_id; diff -r 8c49fb6d789b -r 0b297bc94746 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Mon May 04 18:44:29 2015 +0300 +++ b/src/lib-fs/fs-api.c Mon May 04 19:06:11 2015 +0300 @@ -38,7 +38,7 @@ data stack. */ *error_r = t_strdup_printf("%s: %s", fs_class->name, fs_last_error(fs)); - fs_deinit(&fs); + fs_unref(&fs); return -1; } fs->username = i_strdup(set->username); @@ -147,14 +147,31 @@ return 0; } -void fs_deinit(struct fs **_fs) +void fs_deinit(struct fs **fs) +{ + fs_unref(fs); +} + +void fs_ref(struct fs *fs) +{ + i_assert(fs->refcount > 0); + + fs->refcount++; +} + +void fs_unref(struct fs **_fs) { struct fs *fs = *_fs; string_t *last_error = fs->last_error; struct array module_contexts_arr = fs->module_contexts.arr; + i_assert(fs->refcount > 0); + *_fs = NULL; + if (--fs->refcount > 0) + return; + if (fs->files_open_count > 0) { i_panic("fs-%s: %u files still open (first = %s)", fs->name, fs->files_open_count, fs_file_path(fs->files)); diff -r 8c49fb6d789b -r 0b297bc94746 src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Mon May 04 18:44:29 2015 +0300 +++ b/src/lib-fs/fs-api.h Mon May 04 19:06:11 2015 +0300 @@ -166,8 +166,12 @@ int fs_init(const char *driver, const char *args, const struct fs_settings *set, struct fs **fs_r, const char **error_r); +/* same as fs_unref() */ void fs_deinit(struct fs **fs); +void fs_ref(struct fs *fs); +void fs_unref(struct fs **fs); + /* Returns the parent filesystem (if this is a wrapper fs) or NULL if there's no parent. */ struct fs *fs_get_parent(struct fs *fs); From dovecot at dovecot.org Mon May 4 16:16:58 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 04 May 2015 16:16:58 +0000 Subject: dovecot-2.2: lib-fs: Forgot to initialize refcount in previous c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6403ccd2f5f9 changeset: 18515:6403ccd2f5f9 user: Timo Sirainen date: Mon May 04 19:15:10 2015 +0300 description: lib-fs: Forgot to initialize refcount in previous commit diffstat: src/lib-fs/fs-api.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 0b297bc94746 -r 6403ccd2f5f9 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Mon May 04 19:06:11 2015 +0300 +++ b/src/lib-fs/fs-api.c Mon May 04 19:15:10 2015 +0300 @@ -26,6 +26,7 @@ int ret; fs = fs_class->v.alloc(); + fs->refcount = 1; fs->last_error = str_new(default_pool, 64); i_array_init(&fs->module_contexts, 5); From dovecot at dovecot.org Mon May 4 16:56:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 04 May 2015 16:56:39 +0000 Subject: dovecot-2.2: mail-storage: Moved mailbox attributes API to separ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5f43c88e2c87 changeset: 18516:5f43c88e2c87 user: Stephan Bosch date: Mon May 04 19:53:53 2015 +0300 description: mail-storage: Moved mailbox attributes API to separate module. diffstat: src/lib-storage/Makefile.am | 3 + src/lib-storage/mail-storage-private.h | 8 +- src/lib-storage/mail-storage.c | 97 ------------------------- src/lib-storage/mail-storage.h | 67 +----------------- src/lib-storage/mailbox-attribute-private.h | 12 +++ src/lib-storage/mailbox-attribute.c | 108 ++++++++++++++++++++++++++++ src/lib-storage/mailbox-attribute.h | 77 +++++++++++++++++++ 7 files changed, 202 insertions(+), 170 deletions(-) diffs (truncated from 459 to 300 lines): diff -r 6403ccd2f5f9 -r 5f43c88e2c87 src/lib-storage/Makefile.am --- a/src/lib-storage/Makefile.am Mon May 04 19:15:10 2015 +0300 +++ b/src/lib-storage/Makefile.am Mon May 04 19:53:53 2015 +0300 @@ -43,6 +43,7 @@ mail-storage-settings.c \ mail-thread.c \ mail-user.c \ + mailbox-attribute.c \ mailbox-get.c \ mailbox-guid-cache.c \ mailbox-header.c \ @@ -70,6 +71,8 @@ mail-storage-service.h \ mail-storage-settings.h \ mail-user.h \ + mailbox-attribute.h \ + mailbox-attribute-private.h \ mailbox-guid-cache.h \ mailbox-list.h \ mailbox-list-iter.h \ diff -r 6403ccd2f5f9 -r 5f43c88e2c87 src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Mon May 04 19:15:10 2015 +0300 +++ b/src/lib-storage/mail-storage-private.h Mon May 04 19:53:53 2015 +0300 @@ -7,6 +7,7 @@ #include "mail-storage.h" #include "mail-storage-hooks.h" #include "mail-storage-settings.h" +#include "mailbox-attribute-private.h" #include "mail-index-private.h" /* Default prefix for indexes */ @@ -602,10 +603,6 @@ unsigned int *idx; }; -struct mailbox_attribute_iter { - struct mailbox *box; -}; - /* Modules should use do "my_id = mail_storage_module_id++" and use objects' module_contexts[id] for their own purposes. */ extern struct mail_storage_module_register mail_storage_module_register; @@ -678,9 +675,6 @@ unsigned int mail_storage_get_lock_timeout(struct mail_storage *storage, unsigned int secs); void mail_storage_free_binary_cache(struct mail_storage *storage); -int mailbox_attribute_value_to_string(struct mail_storage *storage, - const struct mail_attribute_value *value, - const char **str_r); enum mail_index_open_flags mail_storage_settings_to_index_flags(const struct mail_storage_settings *set); diff -r 6403ccd2f5f9 -r 5f43c88e2c87 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Mon May 04 19:15:10 2015 +0300 +++ b/src/lib-storage/mail-storage.c Mon May 04 19:53:53 2015 +0300 @@ -1633,103 +1633,6 @@ return 0; } -int mailbox_attribute_set(struct mailbox_transaction_context *t, - enum mail_attribute_type type, const char *key, - const struct mail_attribute_value *value) -{ - return t->box->v.attribute_set(t, type, key, value); -} - -int mailbox_attribute_unset(struct mailbox_transaction_context *t, - enum mail_attribute_type type, const char *key) -{ - struct mail_attribute_value value; - - memset(&value, 0, sizeof(value)); - return t->box->v.attribute_set(t, type, key, &value); -} - -int mailbox_attribute_value_to_string(struct mail_storage *storage, - const struct mail_attribute_value *value, - const char **str_r) -{ - string_t *str; - const unsigned char *data; - size_t size; - - if (value->value_stream == NULL) { - *str_r = value->value; - return 0; - } - str = t_str_new(128); - i_stream_seek(value->value_stream, 0); - while (i_stream_read_data(value->value_stream, &data, &size, 0) > 0) { - if (memchr(data, '\0', size) != NULL) { - mail_storage_set_error(storage, MAIL_ERROR_PARAMS, - "Attribute string value has NULs"); - return -1; - } - str_append_n(str, data, size); - i_stream_skip(value->value_stream, size); - } - if (value->value_stream->stream_errno != 0) { - mail_storage_set_critical(storage, "read(%s) failed: %s", - i_stream_get_name(value->value_stream), - i_stream_get_error(value->value_stream)); - return -1; - } - i_assert(value->value_stream->eof); - *str_r = str_c(str); - return 0; -} - -int mailbox_attribute_get(struct mailbox_transaction_context *t, - enum mail_attribute_type type, const char *key, - struct mail_attribute_value *value_r) -{ - int ret; - - memset(value_r, 0, sizeof(*value_r)); - if ((ret = t->box->v.attribute_get(t, type, key, value_r)) <= 0) - return ret; - i_assert(value_r->value != NULL); - return 1; -} - -int mailbox_attribute_get_stream(struct mailbox_transaction_context *t, - enum mail_attribute_type type, const char *key, - struct mail_attribute_value *value_r) -{ - int ret; - - memset(value_r, 0, sizeof(*value_r)); - value_r->flags |= MAIL_ATTRIBUTE_VALUE_FLAG_INT_STREAMS; - if ((ret = t->box->v.attribute_get(t, type, key, value_r)) <= 0) - return ret; - i_assert(value_r->value != NULL || value_r->value_stream != NULL); - return 1; -} - -struct mailbox_attribute_iter * -mailbox_attribute_iter_init(struct mailbox *box, enum mail_attribute_type type, - const char *prefix) -{ - return box->v.attribute_iter_init(box, type, prefix); -} - -const char *mailbox_attribute_iter_next(struct mailbox_attribute_iter *iter) -{ - return iter->box->v.attribute_iter_next(iter); -} - -int mailbox_attribute_iter_deinit(struct mailbox_attribute_iter **_iter) -{ - struct mailbox_attribute_iter *iter = *_iter; - - *_iter = NULL; - return iter->box->v.attribute_iter_deinit(iter); -} - struct mailbox_sync_context * mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) { diff -r 6403ccd2f5f9 -r 5f43c88e2c87 src/lib-storage/mail-storage.h --- a/src/lib-storage/mail-storage.h Mon May 04 19:15:10 2015 +0300 +++ b/src/lib-storage/mail-storage.h Mon May 04 19:53:53 2015 +0300 @@ -10,6 +10,7 @@ #include "mail-error.h" #include "mail-namespace.h" #include "mailbox-list.h" +#include "mailbox-attribute.h" /* If some operation is taking long, call notify_ok every n seconds. */ #define MAIL_STORAGE_STAYALIVE_SECS 15 @@ -205,42 +206,6 @@ MAILBOX_SYNC_TYPE_MODSEQ = 0x04 }; -/* RFC 5464 specifies that this is vendor//. The registered - vendor-tokens always begin with "vendor." so there's some redundancy.. */ -#define MAILBOX_ATTRIBUTE_PREFIX_DOVECOT "vendor/vendor.dovecot/" -/* Prefix used for attributes reserved for Dovecot's internal use. Normal - users cannot access these in any way. */ -#define MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT \ - MAILBOX_ATTRIBUTE_PREFIX_DOVECOT"pvt/" -/* Prefix used for server attributes in INBOX. INBOX deletion won't delete - any attributes under this prefix. */ -#define MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER \ - MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT"server/" - -enum mail_attribute_type { - MAIL_ATTRIBUTE_TYPE_PRIVATE, - MAIL_ATTRIBUTE_TYPE_SHARED -}; -enum mail_attribute_value_flags { - MAIL_ATTRIBUTE_VALUE_FLAG_READONLY = 0x01, - MAIL_ATTRIBUTE_VALUE_FLAG_INT_STREAMS = 0x02 -}; - -struct mail_attribute_value { - /* mailbox_attribute_set() can set either value or value_stream. - mailbox_attribute_get() returns only values, but - mailbox_attribute_get_stream() may return either value or - value_stream. The caller must unreference the returned streams. */ - const char *value; - struct istream *value_stream; - - /* Last time the attribute was changed (0 = unknown). This may be - returned even for values that don't exist anymore. */ - time_t last_change; - - enum mail_attribute_value_flags flags; -}; - struct message_part; struct mail_namespace; struct mail_storage; @@ -583,36 +548,6 @@ (as opposed to flags shared between users). */ enum mail_flags mailbox_get_private_flags_mask(struct mailbox *box); -/* Set mailbox attribute key to value. The key should be compatible with - IMAP METADATA, so for Dovecot-specific keys use - MAILBOX_ATTRIBUTE_PREFIX_DOVECOT. */ -int mailbox_attribute_set(struct mailbox_transaction_context *t, - enum mail_attribute_type type, const char *key, - const struct mail_attribute_value *value); -/* Delete mailbox attribute key. This is just a wrapper to - mailbox_attribute_set() with value->value=NULL. */ -int mailbox_attribute_unset(struct mailbox_transaction_context *t, - enum mail_attribute_type type, const char *key); -/* Returns value for mailbox attribute key. Returns 1 if value was returned, - 0 if value wasn't found (set to NULL), -1 if error */ -int mailbox_attribute_get(struct mailbox_transaction_context *t, - enum mail_attribute_type type, const char *key, - struct mail_attribute_value *value_r); -/* Same as mailbox_attribute_get(), but the returned value may be either an - input stream or a string. */ -int mailbox_attribute_get_stream(struct mailbox_transaction_context *t, - enum mail_attribute_type type, const char *key, - struct mail_attribute_value *value_r); - -/* Iterate through mailbox attributes of the given type. The prefix can be used - to restrict what attributes are returned. */ -struct mailbox_attribute_iter * -mailbox_attribute_iter_init(struct mailbox *box, enum mail_attribute_type type, - const char *prefix); -/* Returns the attribute key or NULL if there are no more attributes. */ -const char *mailbox_attribute_iter_next(struct mailbox_attribute_iter *iter); -int mailbox_attribute_iter_deinit(struct mailbox_attribute_iter **iter); - /* Synchronize the mailbox. */ struct mailbox_sync_context * mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); diff -r 6403ccd2f5f9 -r 5f43c88e2c87 src/lib-storage/mailbox-attribute-private.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/mailbox-attribute-private.h Mon May 04 19:53:53 2015 +0300 @@ -0,0 +1,12 @@ +#ifndef MAILBOX_ATTRIBUTE_PRIVATE_H +#define MAILBOX_ATTRIBUTE_PRIVATE_H + +struct mailbox_attribute_iter { + struct mailbox *box; +}; + +int mailbox_attribute_value_to_string(struct mail_storage *storage, + const struct mail_attribute_value *value, + const char **str_r); + +#endif diff -r 6403ccd2f5f9 -r 5f43c88e2c87 src/lib-storage/mailbox-attribute.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/mailbox-attribute.c Mon May 04 19:53:53 2015 +0300 @@ -0,0 +1,108 @@ +/* Copyright (c) 2003-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "istream.h" +#include "mail-storage-private.h" + +/* + * Attribute API + */ + +int mailbox_attribute_set(struct mailbox_transaction_context *t, + enum mail_attribute_type type, const char *key, + const struct mail_attribute_value *value) +{ + return t->box->v.attribute_set(t, type, key, value); +} + +int mailbox_attribute_unset(struct mailbox_transaction_context *t, + enum mail_attribute_type type, const char *key) +{ + struct mail_attribute_value value; + + memset(&value, 0, sizeof(value)); + return t->box->v.attribute_set(t, type, key, &value); +} + +int mailbox_attribute_value_to_string(struct mail_storage *storage, + const struct mail_attribute_value *value, + const char **str_r) From dovecot at dovecot.org Mon May 4 20:29:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 04 May 2015 20:29:34 +0000 Subject: dovecot-2.2: lib-storage: Fixed setting/getting server metadata. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/986cbdfb887d changeset: 18517:986cbdfb887d user: Timo Sirainen date: Mon May 04 23:27:42 2015 +0300 description: lib-storage: Fixed setting/getting server metadata. Also added a MAILBOX_ATTRIBUTE_KEY_IS_USER_ACCESSIBLE() macro to make it a bit easier to check if a key should be accessible to a user or not. diffstat: src/lib-storage/index/index-attribute.c | 6 ++---- src/lib-storage/mailbox-attribute.h | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diffs (51 lines): diff -r 5f43c88e2c87 -r 986cbdfb887d src/lib-storage/index/index-attribute.c --- a/src/lib-storage/index/index-attribute.c Mon May 04 19:53:53 2015 +0300 +++ b/src/lib-storage/index/index-attribute.c Mon May 04 23:27:42 2015 +0300 @@ -186,8 +186,7 @@ time_t ts = value->last_change != 0 ? value->last_change : ioloop_time; int ret = 0; - if (strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT, - strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT)) == 0) { + if (!MAILBOX_ATTRIBUTE_KEY_IS_USER_ACCESSIBLE(key)) { mail_storage_set_error(t->box->storage, MAIL_ERROR_PARAMS, "Internal attributes cannot be changed directly"); return -1; @@ -227,8 +226,7 @@ memset(value_r, 0, sizeof(*value_r)); - if (strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT, - strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT)) == 0) + if (!MAILBOX_ATTRIBUTE_KEY_IS_USER_ACCESSIBLE(key)) return 0; if (index_storage_get_dict(t->box, type, &dict, &mailbox_prefix) < 0) diff -r 5f43c88e2c87 -r 986cbdfb887d src/lib-storage/mailbox-attribute.h --- a/src/lib-storage/mailbox-attribute.h Mon May 04 19:53:53 2015 +0300 +++ b/src/lib-storage/mailbox-attribute.h Mon May 04 23:27:42 2015 +0300 @@ -11,11 +11,22 @@ users cannot access these in any way. */ #define MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT \ MAILBOX_ATTRIBUTE_PREFIX_DOVECOT"pvt/" -/* Prefix used for server attributes in INBOX. INBOX deletion won't delete - any attributes under this prefix. */ +/* Server attributes are currently stored in INBOX under this private prefix. + They're under the pvt/ prefix so they won't be listed as regular INBOX + attributes, but unlike other pvt/ attributes it's actually possible to + access these attributes as regular users. + + If INBOX is deleted, attributes under this prefix are preserved. */ #define MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER \ MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT"server/" +/* User can get/set all non-pvt/ attributes and also pvt/server/ attributes. */ +#define MAILBOX_ATTRIBUTE_KEY_IS_USER_ACCESSIBLE(key) \ + (strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT, \ + strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT)) != 0 || \ + strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER, \ + strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER)) == 0) + enum mail_attribute_type { MAIL_ATTRIBUTE_TYPE_PRIVATE, MAIL_ATTRIBUTE_TYPE_SHARED From pigeonhole at rename-it.nl Tue May 5 08:41:53 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 05 May 2015 10:41:53 +0200 Subject: dovecot-2.2-pigeonhole: lib-smtp: util: program client: Fixed ha... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/d71cfa2a1141 changeset: 2042:d71cfa2a1141 user: Stephan Bosch date: Tue May 05 10:41:45 2015 +0200 description: lib-smtp: util: program client: Fixed handing of situation when no input is provided to external program used as a command. Previous fixes still omitted a few important aspects: Connect return result wasn't actually passed up to program_client_run(). Default exit code was wrong. diffstat: src/lib-sieve/util/program-client-remote.c | 2 +- src/lib-sieve/util/program-client.c | 8 +++++--- tests/plugins/extprograms/execute/execute.svtest | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diffs (70 lines): diff -r 1eb0362461f0 -r d71cfa2a1141 src/lib-sieve/util/program-client-remote.c --- a/src/lib-sieve/util/program-client-remote.c Mon May 04 00:26:58 2015 +0200 +++ b/src/lib-sieve/util/program-client-remote.c Tue May 05 10:41:45 2015 +0200 @@ -208,7 +208,7 @@ program_client_fail(pclient, PROGRAM_CLIENT_ERROR_IO); return; } - + (void)program_client_connected(pclient); } diff -r 1eb0362461f0 -r d71cfa2a1141 src/lib-sieve/util/program-client.c --- a/src/lib-sieve/util/program-client.c Mon May 04 00:26:58 2015 +0200 +++ b/src/lib-sieve/util/program-client.c Tue May 05 10:41:45 2015 +0200 @@ -29,17 +29,19 @@ static int program_client_connect(struct program_client *pclient) { + int ret; + if (pclient->set.client_connect_timeout_msecs != 0) { pclient->to = timeout_add (pclient->set.client_connect_timeout_msecs, program_client_connect_timeout, pclient); } - if ( pclient->connect(pclient) < 0 ) { + if ( (ret=pclient->connect(pclient)) < 0 ) { program_client_fail(pclient, PROGRAM_CLIENT_ERROR_IO); return -1; } - return 1; + return ret; } static int program_client_close_output(struct program_client *pclient) @@ -487,7 +489,7 @@ /* reset */ pclient->disconnected = FALSE; - pclient->exit_code = 0; + pclient->exit_code = 1; pclient->error = PROGRAM_CLIENT_ERROR_NONE; pclient->ioloop = io_loop_create(); diff -r 1eb0362461f0 -r d71cfa2a1141 tests/plugins/extprograms/execute/execute.svtest --- a/tests/plugins/extprograms/execute/execute.svtest Mon May 04 00:26:58 2015 +0200 +++ b/tests/plugins/extprograms/execute/execute.svtest Tue May 05 10:41:45 2015 +0200 @@ -23,6 +23,20 @@ } test_result_reset; +test "Execute - i/-" { + execute :input "FROP" "frame"; +} + +test_result_reset; +test "Execute - -/o" { + execute :output "out" "frame"; + + if not string "${out}" "FRAMED { }" { + test_fail "wrong string returned: ${out}"; + } +} + +test_result_reset; test "Execute - i/o" { execute :input "FROP" :output "out" "frame"; From pigeonhole at rename-it.nl Tue May 5 08:50:16 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 05 May 2015 10:50:16 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: util: program client: Fixed h... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/89e0cef5b264 changeset: 2042:89e0cef5b264 user: Stephan Bosch date: Tue May 05 10:41:45 2015 +0200 description: lib-sieve: util: program client: Fixed handing of situation when no input is provided to external program used as a command. Previous fixes still omitted a few important aspects: Connect return result wasn't actually passed up to program_client_run(). Default exit code was wrong. diffstat: src/lib-sieve/util/program-client-remote.c | 2 +- src/lib-sieve/util/program-client.c | 8 +++++--- tests/plugins/extprograms/execute/execute.svtest | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diffs (70 lines): diff -r 1eb0362461f0 -r 89e0cef5b264 src/lib-sieve/util/program-client-remote.c --- a/src/lib-sieve/util/program-client-remote.c Mon May 04 00:26:58 2015 +0200 +++ b/src/lib-sieve/util/program-client-remote.c Tue May 05 10:41:45 2015 +0200 @@ -208,7 +208,7 @@ program_client_fail(pclient, PROGRAM_CLIENT_ERROR_IO); return; } - + (void)program_client_connected(pclient); } diff -r 1eb0362461f0 -r 89e0cef5b264 src/lib-sieve/util/program-client.c --- a/src/lib-sieve/util/program-client.c Mon May 04 00:26:58 2015 +0200 +++ b/src/lib-sieve/util/program-client.c Tue May 05 10:41:45 2015 +0200 @@ -29,17 +29,19 @@ static int program_client_connect(struct program_client *pclient) { + int ret; + if (pclient->set.client_connect_timeout_msecs != 0) { pclient->to = timeout_add (pclient->set.client_connect_timeout_msecs, program_client_connect_timeout, pclient); } - if ( pclient->connect(pclient) < 0 ) { + if ( (ret=pclient->connect(pclient)) < 0 ) { program_client_fail(pclient, PROGRAM_CLIENT_ERROR_IO); return -1; } - return 1; + return ret; } static int program_client_close_output(struct program_client *pclient) @@ -487,7 +489,7 @@ /* reset */ pclient->disconnected = FALSE; - pclient->exit_code = 0; + pclient->exit_code = 1; pclient->error = PROGRAM_CLIENT_ERROR_NONE; pclient->ioloop = io_loop_create(); diff -r 1eb0362461f0 -r 89e0cef5b264 tests/plugins/extprograms/execute/execute.svtest --- a/tests/plugins/extprograms/execute/execute.svtest Mon May 04 00:26:58 2015 +0200 +++ b/tests/plugins/extprograms/execute/execute.svtest Tue May 05 10:41:45 2015 +0200 @@ -23,6 +23,20 @@ } test_result_reset; +test "Execute - i/-" { + execute :input "FROP" "frame"; +} + +test_result_reset; +test "Execute - -/o" { + execute :output "out" "frame"; + + if not string "${out}" "FRAMED { }" { + test_fail "wrong string returned: ${out}"; + } +} + +test_result_reset; test "Execute - i/o" { execute :input "FROP" :output "out" "frame"; From dovecot at dovecot.org Tue May 5 10:15:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 05 May 2015 10:15:30 +0000 Subject: dovecot-2.2: doveadm fs delete: Recent changes broke it completely. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5d67814a596d changeset: 18518:5d67814a596d user: Timo Sirainen date: Tue May 05 13:13:42 2015 +0300 description: doveadm fs delete: Recent changes broke it completely. diffstat: src/doveadm/doveadm-fs.c | 29 ++++++++++++++--------------- 1 files changed, 14 insertions(+), 15 deletions(-) diffs (75 lines): diff -r 986cbdfb887d -r 5d67814a596d src/doveadm/doveadm-fs.c --- a/src/doveadm/doveadm-fs.c Mon May 04 23:27:42 2015 +0300 +++ b/src/doveadm/doveadm-fs.c Tue May 05 13:13:42 2015 +0300 @@ -224,7 +224,7 @@ struct fs_delete_ctx { unsigned int files_count; - struct fs_delete_file **files; + struct fs_delete_file *files; }; static bool cmd_fs_delete_ctx_run(struct fs_delete_ctx *ctx) @@ -233,20 +233,20 @@ bool ret = FALSE; for (i = 0; i < ctx->files_count; i++) { - if (ctx->files[i]->file == NULL || ctx->files[i]->finished) + if (ctx->files[i].file == NULL || ctx->files[i].finished) ; - else if (fs_delete(ctx->files[i]->file) == 0) { - fs_file_deinit(&ctx->files[i]->file); - ctx->files[i]->finished = TRUE; + else if (fs_delete(ctx->files[i].file) == 0) { + fs_file_deinit(&ctx->files[i].file); + ctx->files[i].finished = TRUE; } else if (errno == EAGAIN) ret = TRUE; else { i_error("fs_delete(%s) failed: %s", - fs_file_path(ctx->files[i]->file), - fs_file_last_error(ctx->files[i]->file)); + fs_file_path(ctx->files[i].file), + fs_file_last_error(ctx->files[i].file)); doveadm_exit_code = EX_TEMPFAIL; - fs_file_deinit(&ctx->files[i]->file); - ctx->files[i]->finished = TRUE; + fs_file_deinit(&ctx->files[i].file); + ctx->files[i].finished = TRUE; } } return ret; @@ -264,7 +264,7 @@ memset(&ctx, 0, sizeof(ctx)); ctx.files_count = I_MAX(async_count, 1); - ctx.files = t_new(struct fs_delete_file *, ctx.files_count); + ctx.files = t_new(struct fs_delete_file, ctx.files_count); /* delete subdirs first. all fs backends can't handle recursive lookups, so save the list first. */ @@ -309,11 +309,10 @@ fname = *fnamep; retry: for (i = 0; i < ctx.files_count; i++) { - if (ctx.files[i]->file != NULL || - ctx.files[i]->finished) + if (ctx.files[i].file != NULL || ctx.files[i].finished) continue; - ctx.files[i]->file = fs_file_init(fs, + ctx.files[i].file = fs_file_init(fs, t_strdup_printf("%s%s", path_prefix, fname), FS_OPEN_MODE_READONLY | FS_OPEN_FLAG_ASYNC | FS_OPEN_FLAG_ASYNC_NOQUEUE); @@ -338,8 +337,8 @@ } } for (i = 0; i < ctx.files_count; i++) { - if (ctx.files[i]->file != NULL) - fs_file_deinit(&ctx.files[i]->file); + if (ctx.files[i].file != NULL) + fs_file_deinit(&ctx.files[i].file); } } From dovecot at dovecot.org Tue May 5 10:37:42 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 05 May 2015 10:37:42 +0000 Subject: dovecot-2.2: doveadm fs delete: Reverted last two patches - they... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9b2b3c609367 changeset: 18519:9b2b3c609367 user: Timo Sirainen date: Tue May 05 13:30:38 2015 +0300 description: doveadm fs delete: Reverted last two patches - they were all completely wrong.. diffstat: src/doveadm/doveadm-fs.c | 32 ++++++++++++-------------------- 1 files changed, 12 insertions(+), 20 deletions(-) diffs (79 lines): diff -r 5d67814a596d -r 9b2b3c609367 src/doveadm/doveadm-fs.c --- a/src/doveadm/doveadm-fs.c Tue May 05 13:13:42 2015 +0300 +++ b/src/doveadm/doveadm-fs.c Tue May 05 13:30:38 2015 +0300 @@ -217,14 +217,9 @@ fs_deinit(&fs); } -struct fs_delete_file { - struct fs_file *file; - bool finished; -}; - struct fs_delete_ctx { unsigned int files_count; - struct fs_delete_file *files; + struct fs_file **files; }; static bool cmd_fs_delete_ctx_run(struct fs_delete_ctx *ctx) @@ -233,20 +228,17 @@ bool ret = FALSE; for (i = 0; i < ctx->files_count; i++) { - if (ctx->files[i].file == NULL || ctx->files[i].finished) + if (ctx->files[i] == NULL) ; - else if (fs_delete(ctx->files[i].file) == 0) { - fs_file_deinit(&ctx->files[i].file); - ctx->files[i].finished = TRUE; - } else if (errno == EAGAIN) + else if (fs_delete(ctx->files[i]) == 0) + fs_file_deinit(&ctx->files[i]); + else if (errno == EAGAIN) ret = TRUE; else { i_error("fs_delete(%s) failed: %s", - fs_file_path(ctx->files[i].file), - fs_file_last_error(ctx->files[i].file)); + fs_file_path(ctx->files[i]), + fs_file_last_error(ctx->files[i])); doveadm_exit_code = EX_TEMPFAIL; - fs_file_deinit(&ctx->files[i].file); - ctx->files[i].finished = TRUE; } } return ret; @@ -264,7 +256,7 @@ memset(&ctx, 0, sizeof(ctx)); ctx.files_count = I_MAX(async_count, 1); - ctx.files = t_new(struct fs_delete_file, ctx.files_count); + ctx.files = t_new(struct fs_file *, ctx.files_count); /* delete subdirs first. all fs backends can't handle recursive lookups, so save the list first. */ @@ -309,10 +301,10 @@ fname = *fnamep; retry: for (i = 0; i < ctx.files_count; i++) { - if (ctx.files[i].file != NULL || ctx.files[i].finished) + if (ctx.files[i] != NULL) continue; - ctx.files[i].file = fs_file_init(fs, + ctx.files[i] = fs_file_init(fs, t_strdup_printf("%s%s", path_prefix, fname), FS_OPEN_MODE_READONLY | FS_OPEN_FLAG_ASYNC | FS_OPEN_FLAG_ASYNC_NOQUEUE); @@ -337,8 +329,8 @@ } } for (i = 0; i < ctx.files_count; i++) { - if (ctx.files[i].file != NULL) - fs_file_deinit(&ctx.files[i].file); + if (ctx.files[i] != NULL) + fs_file_deinit(&ctx.files[i]); } } From dovecot at dovecot.org Tue May 5 10:37:42 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 05 May 2015 10:37:42 +0000 Subject: dovecot-2.2: doveadm fs delete: Another attempt at fixing recurs... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a46620d6e0ff changeset: 18520:a46620d6e0ff user: Timo Sirainen date: Tue May 05 13:35:52 2015 +0300 description: doveadm fs delete: Another attempt at fixing recursive deletion. I'm not entirely sure anymore what the original infinite looping was, but this fixes all the potential problems that I see. diffstat: src/doveadm/doveadm-fs.c | 18 +++++++++++------- 1 files changed, 11 insertions(+), 7 deletions(-) diffs (60 lines): diff -r 9b2b3c609367 -r a46620d6e0ff src/doveadm/doveadm-fs.c --- a/src/doveadm/doveadm-fs.c Tue May 05 13:30:38 2015 +0300 +++ b/src/doveadm/doveadm-fs.c Tue May 05 13:35:52 2015 +0300 @@ -225,20 +225,22 @@ static bool cmd_fs_delete_ctx_run(struct fs_delete_ctx *ctx) { unsigned int i; - bool ret = FALSE; + int ret = 0; for (i = 0; i < ctx->files_count; i++) { if (ctx->files[i] == NULL) ; else if (fs_delete(ctx->files[i]) == 0) fs_file_deinit(&ctx->files[i]); - else if (errno == EAGAIN) - ret = TRUE; - else { + else if (errno == EAGAIN) { + if (ret == 0) + ret = 1; + } else { i_error("fs_delete(%s) failed: %s", fs_file_path(ctx->files[i]), fs_file_last_error(ctx->files[i])); doveadm_exit_code = EX_TEMPFAIL; + ret = -1; } } return ret; @@ -253,6 +255,7 @@ struct fs_delete_ctx ctx; const char *fname, *const *fnamep; unsigned int i; + int ret; memset(&ctx, 0, sizeof(ctx)); ctx.files_count = I_MAX(async_count, 1); @@ -311,9 +314,10 @@ fname = NULL; break; } - cmd_fs_delete_ctx_run(&ctx); + if ((ret = cmd_fs_delete_ctx_run(&ctx)) < 0) + break; if (fname != NULL) { - if (fs_wait_async(fs) < 0) { + if (ret > 0 && fs_wait_async(fs) < 0) { i_error("fs_wait_async() failed: %s", fs_last_error(fs)); doveadm_exit_code = EX_TEMPFAIL; break; @@ -321,7 +325,7 @@ goto retry; } } T_END; - while (doveadm_exit_code == 0 && cmd_fs_delete_ctx_run(&ctx)) { + while (doveadm_exit_code == 0 && cmd_fs_delete_ctx_run(&ctx) != 0) { if (fs_wait_async(fs) < 0) { i_error("fs_wait_async() failed: %s", fs_last_error(fs)); doveadm_exit_code = EX_TEMPFAIL; From dovecot at dovecot.org Tue May 5 11:18:21 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 05 May 2015 11:18:21 +0000 Subject: dovecot-2.2: auth: If passdb has non-matching allow_nets, don't ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5e445c659f89 changeset: 18521:5e445c659f89 user: Timo Sirainen date: Tue May 05 14:16:31 2015 +0300 description: auth: If passdb has non-matching allow_nets, don't fail the other passdb lookups also. We might want to use e.g.: passdb { driver = static args = password=secretmasterpass allow_nets=10.1.2.3 } passdb { ... } If allow_nets didn't match in the first passdb, we should just ignore it and continue to the next passdb. diffstat: src/auth/auth-request.c | 45 ++++++++++++++++++++++++++------------------- 1 files changed, 26 insertions(+), 19 deletions(-) diffs (67 lines): diff -r a46620d6e0ff -r 5e445c659f89 src/auth/auth-request.c --- a/src/auth/auth-request.c Tue May 05 13:35:52 2015 +0300 +++ b/src/auth/auth-request.c Tue May 05 14:16:31 2015 +0300 @@ -552,6 +552,15 @@ } return TRUE; } + if (request->failed) { + /* The passdb didn't fail, but something inside it failed + (e.g. allow_nets mismatch). Make sure we'll fail this + lookup, but reset the failure so the next passdb can + succeed. */ + if (*result == PASSDB_RESULT_OK) + *result = PASSDB_RESULT_USER_UNKNOWN; + request->failed = FALSE; + } /* users that exist but can't log in are special. we don't try to match any of the success/failure rules to them. they'll always fail. */ @@ -618,30 +627,28 @@ auth_request_want_skip_passdb(request, next_passdb)) next_passdb = next_passdb->next; + if (*result == PASSDB_RESULT_OK) { + /* this passdb lookup succeeded, preserve its extra fields */ + auth_fields_snapshot(request->extra_fields); + request->snapshot_have_userdb_prefetch_set = + request->userdb_prefetch_set; + if (request->userdb_reply != NULL) + auth_fields_snapshot(request->userdb_reply); + } else { + /* this passdb lookup failed, remove any extra fields it set */ + auth_fields_rollback(request->extra_fields); + if (request->userdb_reply != NULL) { + auth_fields_rollback(request->userdb_reply); + request->userdb_prefetch_set = + request->snapshot_have_userdb_prefetch_set; + } + } + if (passdb_continue && next_passdb != NULL) { /* try next passdb. */ request->passdb = next_passdb; request->passdb_password = NULL; - if (*result == PASSDB_RESULT_OK) { - /* this passdb lookup succeeded, preserve its extra - fields */ - auth_fields_snapshot(request->extra_fields); - request->snapshot_have_userdb_prefetch_set = - request->userdb_prefetch_set; - if (request->userdb_reply != NULL) - auth_fields_snapshot(request->userdb_reply); - } else { - /* this passdb lookup failed, remove any extra fields - it set */ - auth_fields_rollback(request->extra_fields); - if (request->userdb_reply != NULL) { - auth_fields_rollback(request->userdb_reply); - request->userdb_prefetch_set = - request->snapshot_have_userdb_prefetch_set; - } - } - if (*result == PASSDB_RESULT_USER_UNKNOWN) { /* remember that we did at least one successful passdb lookup */ From pigeonhole at rename-it.nl Tue May 5 11:44:33 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 05 May 2015 13:44:33 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: Erroneously decoded mime-enco... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/a3d26f12c2cd changeset: 2043:a3d26f12c2cd user: Stephan Bosch date: Tue May 05 13:42:38 2015 +0200 description: lib-sieve: Erroneously decoded mime-encoded words in address headers. diffstat: src/lib-sieve/plugins/date/tst-date.c | 2 +- src/lib-sieve/plugins/index/tag-index.c | 3 ++- src/lib-sieve/sieve-message.c | 9 +++++---- src/lib-sieve/sieve-message.h | 4 ++-- src/lib-sieve/tst-address.c | 2 +- src/lib-sieve/tst-exists.c | 2 +- src/lib-sieve/tst-header.c | 2 +- tests/test-address.svtest | 25 +++++++++++++++++++++++++ 8 files changed, 38 insertions(+), 11 deletions(-) diffs (164 lines): diff -r 89e0cef5b264 -r a3d26f12c2cd src/lib-sieve/plugins/date/tst-date.c --- a/src/lib-sieve/plugins/date/tst-date.c Tue May 05 10:41:45 2015 +0200 +++ b/src/lib-sieve/plugins/date/tst-date.c Tue May 05 13:42:38 2015 +0200 @@ -471,7 +471,7 @@ /* Get header */ sieve_runtime_trace_descend(renv); if ( (ret=sieve_message_get_header_fields - (renv, hdr_list, &svmos, &hdr_value_list)) <= 0 ) + (renv, hdr_list, &svmos, FALSE, &hdr_value_list)) <= 0 ) return ret; sieve_runtime_trace_ascend(renv); diff -r 89e0cef5b264 -r a3d26f12c2cd src/lib-sieve/plugins/index/tag-index.c --- a/src/lib-sieve/plugins/index/tag-index.c Tue May 05 10:41:45 2015 +0200 +++ b/src/lib-sieve/plugins/index/tag-index.c Tue May 05 13:42:38 2015 +0200 @@ -63,7 +63,7 @@ static int svmo_index_header_override (const struct sieve_message_override *svmo, const struct sieve_runtime_env *renv, - struct sieve_stringlist **headers); + bool mime_decode, struct sieve_stringlist **headers); const struct sieve_message_override_def index_header_override = { SIEVE_OBJECT("index", &index_operand, 0), @@ -256,6 +256,7 @@ static int svmo_index_header_override (const struct sieve_message_override *svmo, const struct sieve_runtime_env *renv, + bool mime_decode ATTR_UNUSED, struct sieve_stringlist **headers) { struct svmo_index_context *ctx = diff -r 89e0cef5b264 -r a3d26f12c2cd src/lib-sieve/sieve-message.c --- a/src/lib-sieve/sieve-message.c Tue May 05 10:41:45 2015 +0200 +++ b/src/lib-sieve/sieve-message.c Tue May 05 13:42:38 2015 +0200 @@ -733,7 +733,7 @@ (const struct sieve_runtime_env *renv, struct sieve_stringlist *field_names, ARRAY_TYPE(sieve_message_override) *svmos, - struct sieve_stringlist **fields_r) + bool mime_decode, struct sieve_stringlist **fields_r) { const struct sieve_message_override *svmo; unsigned int count, i; @@ -742,7 +742,7 @@ if ( svmos == NULL || !array_is_created(svmos) || array_count(svmos) == 0 ) { *fields_r = sieve_message_header_stringlist_create - (renv, field_names, TRUE); + (renv, field_names, mime_decode); return SIEVE_EXEC_OK; } @@ -752,12 +752,13 @@ *fields_r = field_names; } else { *fields_r = sieve_message_header_stringlist_create - (renv, field_names, TRUE); + (renv, field_names, mime_decode); } for ( i = 0; i < count; i++ ) { if ( svmo[i].def->header_override != NULL && - (ret=svmo[i].def->header_override(&svmo[i], renv, fields_r)) <= 0 ) + (ret=svmo[i].def->header_override + (&svmo[i], renv, mime_decode, fields_r)) <= 0 ) return ret; } return SIEVE_EXEC_OK; diff -r 89e0cef5b264 -r a3d26f12c2cd src/lib-sieve/sieve-message.h --- a/src/lib-sieve/sieve-message.h Tue May 05 10:41:45 2015 +0200 +++ b/src/lib-sieve/sieve-message.h Tue May 05 13:42:38 2015 +0200 @@ -102,7 +102,7 @@ int (*header_override) (const struct sieve_message_override *svmo, const struct sieve_runtime_env *renv, - struct sieve_stringlist **headers); + bool mime_decode, struct sieve_stringlist **headers); }; struct sieve_message_override { @@ -164,6 +164,6 @@ (const struct sieve_runtime_env *renv, struct sieve_stringlist *field_names, ARRAY_TYPE(sieve_message_override) *svmos, - struct sieve_stringlist **fields_r); + bool mime_decode, struct sieve_stringlist **fields_r); #endif /* __SIEVE_MESSAGE_H */ diff -r 89e0cef5b264 -r a3d26f12c2cd src/lib-sieve/tst-address.c --- a/src/lib-sieve/tst-address.c Tue May 05 10:41:45 2015 +0200 +++ b/src/lib-sieve/tst-address.c Tue May 05 13:42:38 2015 +0200 @@ -261,7 +261,7 @@ /* Get header */ sieve_runtime_trace_descend(renv); if ( (ret=sieve_message_get_header_fields - (renv, hdr_list, &svmos, &hdr_value_list)) <= 0 ) + (renv, hdr_list, &svmos, FALSE, &hdr_value_list)) <= 0 ) return ret; sieve_runtime_trace_ascend(renv); diff -r 89e0cef5b264 -r a3d26f12c2cd src/lib-sieve/tst-exists.c --- a/src/lib-sieve/tst-exists.c Tue May 05 10:41:45 2015 +0200 +++ b/src/lib-sieve/tst-exists.c Tue May 05 13:42:38 2015 +0200 @@ -151,7 +151,7 @@ /* Get header */ if ( (ret=sieve_message_get_header_fields (renv, sieve_single_stringlist_create(renv, hdr_item, FALSE), - &svmos, &value_list) <= 0) ) + &svmos, FALSE, &value_list) <= 0) ) return ret; if ( (ret=sieve_stringlist_next_item(value_list, &dummy)) < 0) diff -r 89e0cef5b264 -r a3d26f12c2cd src/lib-sieve/tst-header.c --- a/src/lib-sieve/tst-header.c Tue May 05 10:41:45 2015 +0200 +++ b/src/lib-sieve/tst-header.c Tue May 05 13:42:38 2015 +0200 @@ -191,7 +191,7 @@ /* Get header */ sieve_runtime_trace_descend(renv); if ( (ret=sieve_message_get_header_fields - (renv, hdr_list, &svmos, &value_list)) <= 0 ) + (renv, hdr_list, &svmos, TRUE, &value_list)) <= 0 ) return ret; sieve_runtime_trace_ascend(renv); diff -r 89e0cef5b264 -r a3d26f12c2cd tests/test-address.svtest --- a/tests/test-address.svtest Tue May 05 10:41:45 2015 +0200 +++ b/tests/test-address.svtest Tue May 05 13:42:38 2015 +0200 @@ -372,6 +372,30 @@ } /* + * TEST: Mime encoding of '@' in display name + */ + +test_set "message" text: +From: "Frop " +To: =?UTF-8?B?RnJpZXBAZnJvcA0K?= + +Subject: Test + +Frop! +. +; + + +test "Mime encoding of '@' in display name" { + # Relevant sieve rule: + + if not address :is "To" + ["friep at example.com"] { + test_fail "Invalid address extracted"; + } +} + +/* * TEST: Erroneous mime encoding */ @@ -394,3 +418,4 @@ } } + From dovecot at dovecot.org Tue May 5 13:20:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 05 May 2015 13:20:15 +0000 Subject: dovecot-2.2: lib-storage: mailbox_list_fs_get_list() shouldn't c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/164d4d499069 changeset: 18522:164d4d499069 user: Timo Sirainen date: Tue May 05 16:14:07 2015 +0300 description: lib-storage: mailbox_list_fs_get_list() shouldn't crash if fs wasn't created by mailbox_list_init_fs() diffstat: src/lib-storage/mailbox-list.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (10 lines): diff -r 5e445c659f89 -r 164d4d499069 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Tue May 05 14:16:31 2015 +0300 +++ b/src/lib-storage/mailbox-list.c Tue May 05 16:14:07 2015 +0300 @@ -1872,5 +1872,5 @@ fs = fs->parent; ctx = MAILBOX_LIST_FS_CONTEXT(fs); - return ctx->list; + return ctx == NULL ? NULL : ctx->list; } From dovecot at dovecot.org Tue May 5 13:20:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 05 May 2015 13:20:15 +0000 Subject: dovecot-2.2: fts: Initialize fts backend in mail_namespaces_adde... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/eee937dc9438 changeset: 18523:eee937dc9438 user: Timo Sirainen date: Tue May 05 16:17:14 2015 +0300 description: fts: Initialize fts backend in mail_namespaces_added(), not in mailbox_list_created() This way the storage has already been created by the time fts initialization starts, which simplifies things. diffstat: src/plugins/fts/fts-plugin.c | 2 +- src/plugins/fts/fts-storage.c | 28 +++++++++++++++++++--------- src/plugins/fts/fts-storage.h | 2 +- 3 files changed, 21 insertions(+), 11 deletions(-) diffs (70 lines): diff -r 164d4d499069 -r eee937dc9438 src/plugins/fts/fts-plugin.c --- a/src/plugins/fts/fts-plugin.c Tue May 05 16:14:07 2015 +0300 +++ b/src/plugins/fts/fts-plugin.c Tue May 05 16:17:14 2015 +0300 @@ -13,7 +13,7 @@ const char *fts_plugin_version = DOVECOT_ABI_VERSION; static struct mail_storage_hooks fts_mail_storage_hooks = { - .mailbox_list_created = fts_mailbox_list_created, + .mail_namespaces_added = fts_mail_namespaces_added, .mailbox_allocated = fts_mailbox_allocated, .mail_allocated = fts_mail_allocated }; diff -r 164d4d499069 -r eee937dc9438 src/plugins/fts/fts-storage.c --- a/src/plugins/fts/fts-storage.c Tue May 05 16:14:07 2015 +0300 +++ b/src/plugins/fts/fts-storage.c Tue May 05 16:17:14 2015 +0300 @@ -756,17 +756,11 @@ return 0; } -void fts_mailbox_list_created(struct mailbox_list *list) +static void +fts_mailbox_list_init(struct mailbox_list *list, const char *name) { struct fts_backend *backend; - const char *name, *path, *error; - - name = mail_user_plugin_getenv(list->ns->user, "fts"); - if (name == NULL) { - if (list->mail_set->mail_debug) - i_debug("fts: No fts setting - plugin disabled"); - return; - } + const char *path, *error; if (!mailbox_list_get_root_path(list, MAILBOX_LIST_PATH_TYPE_INDEX, &path)) { if (list->mail_set->mail_debug) { @@ -797,6 +791,22 @@ } } +void fts_mail_namespaces_added(struct mail_namespace *namespaces) +{ + struct mail_namespace *ns; + const char *name; + + name = mail_user_plugin_getenv(namespaces->user, "fts"); + if (name == NULL) { + if (namespaces->user->mail_debug) + i_debug("fts: No fts setting - plugin disabled"); + return; + } + + for (ns = namespaces; ns != NULL; ns = ns->next) + fts_mailbox_list_init(ns->list, name); +} + struct fts_backend *fts_mailbox_backend(struct mailbox *box) { struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list); diff -r 164d4d499069 -r eee937dc9438 src/plugins/fts/fts-storage.h --- a/src/plugins/fts/fts-storage.h Tue May 05 16:14:07 2015 +0300 +++ b/src/plugins/fts/fts-storage.h Tue May 05 16:17:14 2015 +0300 @@ -52,6 +52,6 @@ void fts_mail_allocated(struct mail *mail); void fts_mailbox_allocated(struct mailbox *box); -void fts_mailbox_list_created(struct mailbox_list *list); +void fts_mail_namespaces_added(struct mail_namespace *namespaces); #endif From dovecot at dovecot.org Tue May 5 13:20:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 05 May 2015 13:20:25 +0000 Subject: dovecot-2.2: fts-lucene: Removed lazy initialization code. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/25f29613908f changeset: 18524:25f29613908f user: Timo Sirainen date: Tue May 05 16:18:21 2015 +0300 description: fts-lucene: Removed lazy initialization code. This is no longer needed after the previous fts commit. diffstat: src/plugins/fts-lucene/fts-backend-lucene.c | 49 +++++++--------------------- 1 files changed, 13 insertions(+), 36 deletions(-) diffs (113 lines): diff -r eee937dc9438 -r 25f29613908f src/plugins/fts-lucene/fts-backend-lucene.c --- a/src/plugins/fts-lucene/fts-backend-lucene.c Tue May 05 16:17:14 2015 +0300 +++ b/src/plugins/fts-lucene/fts-backend-lucene.c Tue May 05 16:18:21 2015 +0300 @@ -126,35 +126,14 @@ return &backend->backend; } -static void fts_backend_lucene_real_init(struct lucene_fts_backend *backend) -{ - struct fts_lucene_user *fuser = - FTS_LUCENE_USER_CONTEXT(backend->backend.ns->user); - const char *path; - - if (backend->index != NULL) - return; - - /* initialize this path lazily, because with mbox format the get_path() - is overridden by the mbox code, but it hasn't had a chance to do - that yet in fts_backend_lucene_init(). */ - path = mailbox_list_get_root_forced(backend->backend.ns->list, - MAILBOX_LIST_PATH_TYPE_INDEX); - - backend->dir_path = i_strconcat(path, "/"LUCENE_INDEX_DIR_NAME, NULL); - backend->index = lucene_index_init(backend->dir_path, - backend->backend.ns->list, - &fuser->set); - - path = t_strconcat(backend->dir_path, "/"LUCENE_EXPUNGE_LOG_NAME, NULL); - backend->expunge_log = fts_expunge_log_init(path); -} - static int fts_backend_lucene_init(struct fts_backend *_backend, const char **error_r) { + struct lucene_fts_backend *backend = + (struct lucene_fts_backend *)_backend; struct fts_lucene_user *fuser = FTS_LUCENE_USER_CONTEXT(_backend->ns->user); + const char *path; if (fuser == NULL) { /* invalid settings */ @@ -168,6 +147,16 @@ _backend->flags &= ~FTS_BACKEND_FLAG_FUZZY_SEARCH; _backend->flags |= FTS_BACKEND_FLAG_TOKENIZED_INPUT; } + path = mailbox_list_get_root_forced(_backend->ns->list, + MAILBOX_LIST_PATH_TYPE_INDEX); + + backend->dir_path = i_strconcat(path, "/"LUCENE_INDEX_DIR_NAME, NULL); + backend->index = lucene_index_init(backend->dir_path, + _backend->ns->list, + &fuser->set); + + path = t_strconcat(backend->dir_path, "/"LUCENE_EXPUNGE_LOG_NAME, NULL); + backend->expunge_log = fts_expunge_log_init(path); return 0; } @@ -195,8 +184,6 @@ struct fts_index_header hdr; uint32_t set_checksum; - fts_backend_lucene_real_init(backend); - if (fts_index_get_header(box, &hdr)) { set_checksum = fts_lucene_settings_checksum(&fuser->set); if (!fts_index_have_compatible_settings(_backend->ns->list, @@ -231,8 +218,6 @@ i_assert(!backend->updating); - fts_backend_lucene_real_init(backend); - ctx = i_new(struct lucene_fts_backend_update_context, 1); ctx->ctx.backend = _backend; ctx->mime_parts = fuser->set.mime_parts; @@ -468,8 +453,6 @@ struct lucene_fts_backend *backend = (struct lucene_fts_backend *)_backend; - fts_backend_lucene_real_init(backend); - if (lucene_index_rescan(backend->index) < 0) return -1; return lucene_index_optimize(backend->index); @@ -481,8 +464,6 @@ (struct lucene_fts_backend *)_backend; int ret; - fts_backend_lucene_real_init(backend); - ret = lucene_index_expunge_from_log(backend->index, backend->expunge_log); if (ret == 0) { @@ -504,8 +485,6 @@ (struct lucene_fts_backend *)_backend; int ret; - fts_backend_lucene_real_init(backend); - if (fts_backend_select(backend, box) < 0) return -1; T_BEGIN { @@ -578,8 +557,6 @@ (struct lucene_fts_backend *)_backend; int ret; - fts_backend_lucene_real_init(backend); - T_BEGIN { HASH_TABLE_TYPE(wguid_result) guids; From dovecot at dovecot.org Tue May 5 13:25:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 05 May 2015 13:25:56 +0000 Subject: dovecot-2.2: lib-storage: After mailbox_list_init_fs() is finish... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5c80c7309635 changeset: 18525:5c80c7309635 user: Timo Sirainen date: Tue May 05 16:24:07 2015 +0300 description: lib-storage: After mailbox_list_init_fs() is finished, notify fs about it. Creating a separate fs_init_finish() would perhaps have been clearner, but it's a lot more work and usually isn't even necessary for most backends. So I simply chose to use fs_get_properties() which is a fast call in all fs backends and the few ones that actually care about the initialization finish can then do their work in there. diffstat: src/lib-storage/mailbox-list.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r 25f29613908f -r 5c80c7309635 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Tue May 05 16:18:21 2015 +0300 +++ b/src/lib-storage/mailbox-list.c Tue May 05 16:24:07 2015 +0300 @@ -1861,6 +1861,10 @@ ctx = p_new(list->pool, struct mailbox_list_fs_context, 1); ctx->list = list; MODULE_CONTEXT_SET(parent_fs, mailbox_list_fs_module, ctx); + + /* a bit kludgy notification to the fs that we're now finished setting + up the module context. */ + (void)fs_get_properties(*fs_r); return 0; } From dovecot at dovecot.org Tue May 5 20:08:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 05 May 2015 20:08:52 +0000 Subject: dovecot-2.2: doveadm fs delete: Fixed function return type Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2a2b3897dfe1 changeset: 18526:2a2b3897dfe1 user: Timo Sirainen date: Tue May 05 23:07:03 2015 +0300 description: doveadm fs delete: Fixed function return type diffstat: src/doveadm/doveadm-fs.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 5c80c7309635 -r 2a2b3897dfe1 src/doveadm/doveadm-fs.c --- a/src/doveadm/doveadm-fs.c Tue May 05 16:24:07 2015 +0300 +++ b/src/doveadm/doveadm-fs.c Tue May 05 23:07:03 2015 +0300 @@ -222,7 +222,7 @@ struct fs_file **files; }; -static bool cmd_fs_delete_ctx_run(struct fs_delete_ctx *ctx) +static int cmd_fs_delete_ctx_run(struct fs_delete_ctx *ctx) { unsigned int i; int ret = 0; From dovecot at dovecot.org Wed May 6 08:11:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 May 2015 08:11:59 +0000 Subject: dovecot-2.2: script: Updated comment to describe the current scr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cf9e0cbf55a9 changeset: 18527:cf9e0cbf55a9 user: Timo Sirainen date: Wed May 06 11:10:09 2015 +0300 description: script: Updated comment to describe the current script protocol. diffstat: src/util/script.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (23 lines): diff -r 2a2b3897dfe1 -r cf9e0cbf55a9 src/util/script.c --- a/src/util/script.c Tue May 05 23:07:03 2015 +0300 +++ b/src/util/script.c Wed May 06 11:10:09 2015 +0300 @@ -78,14 +78,17 @@ /* Input contains: VERSION .. - [timeout=] - | "-" + [alarm= ] + [noreply ] arg 1 arg 2 ... DATA + + It could be thought of either as a feature or a bug that alarm and + noreply settings are mixed together with regular args.. */ alarm(SCRIPT_READ_TIMEOUT_SECS); scanpos = 1; From dovecot at dovecot.org Wed May 6 11:15:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 May 2015 11:15:59 +0000 Subject: dovecot-2.2: script: Another comment update about protocol - the... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/956e152467de changeset: 18528:956e152467de user: Timo Sirainen date: Wed May 06 14:14:09 2015 +0300 description: script: Another comment update about protocol - the previous one was wrong. diffstat: src/util/script.c | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diffs (24 lines): diff -r cf9e0cbf55a9 -r 956e152467de src/util/script.c --- a/src/util/script.c Wed May 06 11:10:09 2015 +0300 +++ b/src/util/script.c Wed May 06 14:14:09 2015 +0300 @@ -79,7 +79,7 @@ VERSION .. [alarm= ] - [noreply ] + "noreply" | "-" (or anything really) arg 1 arg 2 @@ -87,8 +87,9 @@ DATA - It could be thought of either as a feature or a bug that alarm and - noreply settings are mixed together with regular args.. + This is quite a horrible protocol. If alarm is specified, it MUST be + before "noreply". If "noreply" isn't given, something other string + (typically "-") must be given which is eaten away. */ alarm(SCRIPT_READ_TIMEOUT_SECS); scanpos = 1; From dovecot at dovecot.org Wed May 6 12:47:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 May 2015 12:47:40 +0000 Subject: dovecot-2.2: doveadm-server: Invalid parameters for some command... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5fce0eaa75d6 changeset: 18529:5fce0eaa75d6 user: Timo Sirainen date: Wed May 06 15:42:31 2015 +0300 description: doveadm-server: Invalid parameters for some commands caused crash at deinit handlers. deinit() can be called even if init() was never called. diffstat: src/doveadm/doveadm-mail-mailbox-status.c | 3 ++- src/doveadm/doveadm-mail-mailbox.c | 3 ++- src/doveadm/doveadm-mail.h | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diffs (39 lines): diff -r 956e152467de -r 5fce0eaa75d6 src/doveadm/doveadm-mail-mailbox-status.c --- a/src/doveadm/doveadm-mail-mailbox-status.c Wed May 06 14:14:09 2015 +0300 +++ b/src/doveadm/doveadm-mail-mailbox-status.c Wed May 06 15:42:31 2015 +0300 @@ -214,7 +214,8 @@ { struct status_cmd_context *ctx = (struct status_cmd_context *)_ctx; - mail_search_args_unref(&ctx->search_args); + if (ctx->search_args != NULL) + mail_search_args_unref(&ctx->search_args); } static bool diff -r 956e152467de -r 5fce0eaa75d6 src/doveadm/doveadm-mail-mailbox.c --- a/src/doveadm/doveadm-mail-mailbox.c Wed May 06 14:14:09 2015 +0300 +++ b/src/doveadm/doveadm-mail-mailbox.c Wed May 06 15:42:31 2015 +0300 @@ -182,7 +182,8 @@ { struct list_cmd_context *ctx = (struct list_cmd_context *)_ctx; - mail_search_args_unref(&ctx->search_args); + if (ctx->search_args != NULL) + mail_search_args_unref(&ctx->search_args); } static struct doveadm_mail_cmd_context *cmd_mailbox_list_alloc(void) diff -r 956e152467de -r 5fce0eaa75d6 src/doveadm/doveadm-mail.h --- a/src/doveadm/doveadm-mail.h Wed May 06 14:14:09 2015 +0300 +++ b/src/doveadm/doveadm-mail.h Wed May 06 15:42:31 2015 +0300 @@ -45,7 +45,8 @@ command. This is called once per each user. */ int (*run)(struct doveadm_mail_cmd_context *ctx, struct mail_user *mail_user); - /* Deinitialize the command. Called once at the end. */ + /* Deinitialize the command. Called once at the end - even if + preinit() or init() was never called. */ void (*deinit)(struct doveadm_mail_cmd_context *ctx); }; From dovecot at dovecot.org Wed May 6 12:47:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 May 2015 12:47:40 +0000 Subject: dovecot-2.2: doveadm: Added -U parameter for executing th... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/14eae320322c changeset: 18530:14eae320322c user: Timo Sirainen date: Wed May 06 15:45:43 2015 +0300 description: doveadm: Added -U parameter for executing the command for all the users in the file. This is similar to -A parameter, but instead of getting the list of users from userdb they are read from the file. The file contains one username per line. diffstat: src/doveadm/client-connection.c | 3 ++- src/doveadm/doveadm-mail.c | 23 +++++++++++++++++++++-- src/doveadm/doveadm-mail.h | 1 + 3 files changed, 24 insertions(+), 3 deletions(-) diffs (81 lines): diff -r 5fce0eaa75d6 -r 14eae320322c src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Wed May 06 15:42:31 2015 +0300 +++ b/src/doveadm/client-connection.c Wed May 06 15:45:43 2015 +0300 @@ -93,10 +93,11 @@ ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_DEBUG; optind = 1; - getopt_args = t_strconcat("AS:u:", ctx->getopt_args, NULL); + getopt_args = t_strconcat("AS:u:U:", ctx->getopt_args, NULL); while ((c = getopt(argc, argv, getopt_args)) > 0) { switch (c) { case 'A': + case 'U': add_username_header = TRUE; break; case 'S': diff -r 5fce0eaa75d6 -r 14eae320322c src/doveadm/doveadm-mail.c --- a/src/doveadm/doveadm-mail.c Wed May 06 15:42:31 2015 +0300 +++ b/src/doveadm/doveadm-mail.c Wed May 06 15:45:43 2015 +0300 @@ -515,7 +515,17 @@ doveadm_mail_cmd_get_next_user(struct doveadm_mail_cmd_context *ctx, const char **username_r) { - return mail_storage_service_all_next(ctx->storage_service, username_r); + if (ctx->users_list_input == NULL) + return mail_storage_service_all_next(ctx->storage_service, username_r); + + *username_r = i_stream_read_next_line(ctx->users_list_input); + if (ctx->users_list_input->stream_errno != 0) { + i_error("read(%s) failed: %s", + i_stream_get_name(ctx->users_list_input), + i_stream_get_error(ctx->users_list_input)); + return -1; + } + return *username_r != NULL ? 1 : 0; } static void @@ -557,7 +567,7 @@ if (doveadm_debug) ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_DEBUG; - getopt_args = "AS:u:"; + getopt_args = "AS:u:U:"; /* keep context's getopt_args first in case it contains '+' */ if (ctx->getopt_args != NULL) getopt_args = t_strconcat(ctx->getopt_args, getopt_args, NULL); @@ -583,6 +593,13 @@ ctx->cur_username = NULL; } break; + case 'U': + ctx->service_flags |= + MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP; + wildcard_user = "*"; + ctx->users_list_input = + i_stream_create_file(optarg, 1024); + break; default: if (ctx->v.parse_arg == NULL || !ctx->v.parse_arg(ctx, c)) @@ -633,6 +650,8 @@ /* service deinit unloads mail plugins, so do it late */ mail_storage_service_deinit(&ctx->storage_service); + if (ctx->users_list_input != NULL) + i_stream_unref(&ctx->users_list_input); if (ctx->cmd_input != NULL) i_stream_unref(&ctx->cmd_input); if (ctx->exit_code != 0) diff -r 5fce0eaa75d6 -r 14eae320322c src/doveadm/doveadm-mail.h --- a/src/doveadm/doveadm-mail.h Wed May 06 15:42:31 2015 +0300 +++ b/src/doveadm/doveadm-mail.h Wed May 06 15:45:43 2015 +0300 @@ -75,6 +75,7 @@ struct mail_storage_service_input storage_service_input; /* search args aren't set for all mail commands */ struct mail_search_args *search_args; + struct istream *users_list_input; struct ip_addr cur_client_ip; const char *cur_username; From dovecot at dovecot.org Wed May 6 13:03:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 May 2015 13:03:36 +0000 Subject: dovecot-2.2: doveadm: Changed -U to -F Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/13c933ed6371 changeset: 18531:13c933ed6371 user: Timo Sirainen date: Wed May 06 16:01:45 2015 +0300 description: doveadm: Changed -U to -F Otherwise -U collides with doveadm sync -U parameter. -F isn't currently used by anything. diffstat: src/doveadm/client-connection.c | 4 ++-- src/doveadm/doveadm-mail.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diffs (38 lines): diff -r 14eae320322c -r 13c933ed6371 src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Wed May 06 15:45:43 2015 +0300 +++ b/src/doveadm/client-connection.c Wed May 06 16:01:45 2015 +0300 @@ -93,11 +93,11 @@ ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_DEBUG; optind = 1; - getopt_args = t_strconcat("AS:u:U:", ctx->getopt_args, NULL); + getopt_args = t_strconcat("AF:S:u:", ctx->getopt_args, NULL); while ((c = getopt(argc, argv, getopt_args)) > 0) { switch (c) { case 'A': - case 'U': + case 'F': add_username_header = TRUE; break; case 'S': diff -r 14eae320322c -r 13c933ed6371 src/doveadm/doveadm-mail.c --- a/src/doveadm/doveadm-mail.c Wed May 06 15:45:43 2015 +0300 +++ b/src/doveadm/doveadm-mail.c Wed May 06 16:01:45 2015 +0300 @@ -567,7 +567,7 @@ if (doveadm_debug) ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_DEBUG; - getopt_args = "AS:u:U:"; + getopt_args = "AF:S:u:"; /* keep context's getopt_args first in case it contains '+' */ if (ctx->getopt_args != NULL) getopt_args = t_strconcat(ctx->getopt_args, getopt_args, NULL); @@ -593,7 +593,7 @@ ctx->cur_username = NULL; } break; - case 'U': + case 'F': ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP; wildcard_user = "*"; From dovecot at dovecot.org Wed May 6 13:18:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 May 2015 13:18:11 +0000 Subject: dovecot-2.2: dsync: Don't try to rename namespace roots. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6612d20f2fb8 changeset: 18532:6612d20f2fb8 user: Timo Sirainen date: Wed May 06 16:16:11 2015 +0300 description: dsync: Don't try to rename namespace roots. It'll just cause an assert-crash. diffstat: src/doveadm/dsync/dsync-mailbox-tree-sync.c | 38 ++++++++++++++--------- src/doveadm/dsync/test-dsync-mailbox-tree-sync.c | 26 +++++++++++++++- 2 files changed, 48 insertions(+), 16 deletions(-) diffs (126 lines): diff -r 13c933ed6371 -r 6612d20f2fb8 src/doveadm/dsync/dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c Wed May 06 16:01:45 2015 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Wed May 06 16:16:11 2015 +0300 @@ -363,6 +363,9 @@ const char *full_name; unsigned int prefix_len = node->ns == NULL ? 0 : node->ns->prefix_len; + if (strcmp(node->name, "INBOX") == 0 && node->parent == &tree->root) + return TRUE; + if (prefix_len == 0) return FALSE; @@ -585,6 +588,16 @@ return ts; } +static bool sync_node_is_namespace_root(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node) +{ + if (node == NULL) + return FALSE; + if (node == &tree->root) + return TRUE; + return sync_node_is_namespace_prefix(tree, node); +} + static bool ATTR_NULL(3, 4) sync_rename_lower_ts(struct dsync_mailbox_tree_sync_ctx *ctx, struct dsync_mailbox_node *local_node1, @@ -604,6 +617,16 @@ at all. Note that node1 and node2 may be the same node pointers. */ i_assert(strcmp(local_node1->name, remote_node2->name) == 0); + if (sync_node_is_namespace_root(ctx->remote_tree, remote_node1) || + sync_node_is_namespace_root(ctx->remote_tree, remote_node2) || + sync_node_is_namespace_root(ctx->local_tree, local_node1) || + sync_node_is_namespace_root(ctx->local_tree, local_node2)) { + local_node1->sync_delayed_guid_change = TRUE; + remote_node2->sync_delayed_guid_change = TRUE; + *reason_r = "Can't rename namespace prefixes - will be merged later"; + return FALSE; + } + local_ts = nodes_get_timestamp(local_node1, local_node2); remote_ts = nodes_get_timestamp(remote_node1, remote_node2); @@ -791,14 +814,6 @@ return NULL; } -static bool sync_node_is_namespace_root(struct dsync_mailbox_tree *tree, - struct dsync_mailbox_node *node) -{ - if (node == &tree->root) - return TRUE; - return sync_node_is_namespace_prefix(tree, node); -} - static bool sync_rename_directory(struct dsync_mailbox_tree_sync_ctx *ctx, struct dsync_mailbox_node *local_node1, struct dsync_mailbox_node *remote_node2, @@ -821,13 +836,6 @@ *reason_r = "Directory name paths are equal"; return FALSE; } - if (sync_node_is_namespace_root(ctx->remote_tree, remote_node1) || - sync_node_is_namespace_root(ctx->remote_tree, remote_node2) || - sync_node_is_namespace_root(ctx->local_tree, local_node1) || - sync_node_is_namespace_root(ctx->local_tree, local_node2)) { - *reason_r = "Directory is part of namespace prefix"; - return FALSE; - } return sync_rename_lower_ts(ctx, local_node1, remote_node1, local_node2, remote_node2, reason_r); diff -r 13c933ed6371 -r 6612d20f2fb8 src/doveadm/dsync/test-dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c Wed May 06 16:01:45 2015 +0300 +++ b/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c Wed May 06 16:16:11 2015 +0300 @@ -181,7 +181,8 @@ dsync_mailbox_tree_build_guid_hash(tree1, &dup_node1, &dup_node2); dsync_mailbox_tree_build_guid_hash(tree2, &dup_node1, &dup_node2); ctx = dsync_mailbox_trees_sync_init(tree1, tree2, - DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY, 0); + DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY, + DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG); while ((change = dsync_mailbox_trees_sync_next(ctx)) != NULL) { } dsync_mailbox_trees_sync_deinit(&ctx); @@ -681,6 +682,28 @@ test_end(); } +static void test_dsync_mailbox_tree_sync_renames21(void) +{ +#if 0 + /* FIXME: we can't currently test this without crashing */ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 21"); + tree1 = dsync_mailbox_tree_init('/', '_'); + tree2 = dsync_mailbox_tree_init('/', '_'); + + node_create(tree1, 1, "INBOX", 0); + node_create(tree1, 2, "foo", 0); + /* swap INBOX and foo - the INBOX name is important since it's + treated specially */ + node_create(tree2, 1, "foo", 0); + node_create(tree2, 2, "INBOX", 1); + + test_trees(tree1, tree2); + test_end(); +#endif +} + static void test_dsync_mailbox_tree_sync_random(void) { struct dsync_mailbox_tree *tree1, *tree2; @@ -717,6 +740,7 @@ test_dsync_mailbox_tree_sync_renames18, test_dsync_mailbox_tree_sync_renames19, test_dsync_mailbox_tree_sync_renames20, + test_dsync_mailbox_tree_sync_renames21, test_dsync_mailbox_tree_sync_random, NULL }; From dovecot at dovecot.org Wed May 6 16:52:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 May 2015 16:52:14 +0000 Subject: dovecot-2.2: imap: Don't advertise SPECIAL-USE if there are no s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/18b71db74d31 changeset: 18533:18b71db74d31 user: Timo Sirainen date: Wed May 06 19:50:23 2015 +0300 description: imap: Don't advertise SPECIAL-USE if there are no such mailboxes configured. diffstat: configure.ac | 2 +- src/imap/imap-client.c | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletions(-) diffs (45 lines): diff -r 6612d20f2fb8 -r 18b71db74d31 configure.ac --- a/configure.ac Wed May 06 16:16:11 2015 +0300 +++ b/configure.ac Wed May 06 19:50:23 2015 +0300 @@ -2834,7 +2834,7 @@ dnl IDLE doesn't really belong to banner. It's there just to make Blackberries dnl happy, because otherwise BIS server disables push email. capability_banner="IMAP4rev1 LITERAL+ SASL-IR LOGIN-REFERRALS ID ENABLE IDLE" -capability="$capability_banner SORT SORT=DISPLAY THREAD=REFERENCES THREAD=REFS THREAD=ORDEREDSUBJECT MULTIAPPEND URL-PARTIAL CATENATE UNSELECT CHILDREN NAMESPACE UIDPLUS LIST-EXTENDED I18NLEVEL=1 CONDSTORE QRESYNC ESEARCH ESORT SEARCHRES WITHIN CONTEXT=SEARCH LIST-STATUS SPECIAL-USE BINARY MOVE" +capability="$capability_banner SORT SORT=DISPLAY THREAD=REFERENCES THREAD=REFS THREAD=ORDEREDSUBJECT MULTIAPPEND URL-PARTIAL CATENATE UNSELECT CHILDREN NAMESPACE UIDPLUS LIST-EXTENDED I18NLEVEL=1 CONDSTORE QRESYNC ESEARCH ESORT SEARCHRES WITHIN CONTEXT=SEARCH LIST-STATUS BINARY MOVE" AC_DEFINE_UNQUOTED(CAPABILITY_STRING, "$capability", IMAP capabilities) AC_DEFINE_UNQUOTED(CAPABILITY_BANNER_STRING, "$capability_banner", IMAP capabilities advertised in banner) diff -r 6612d20f2fb8 -r 18b71db74d31 src/imap/imap-client.c --- a/src/imap/imap-client.c Wed May 06 16:16:11 2015 +0300 +++ b/src/imap/imap-client.c Wed May 06 19:50:23 2015 +0300 @@ -55,6 +55,17 @@ client->urlauth_ctx = imap_urlauth_init(client->user, &config); } +static bool user_has_special_use_mailboxes(struct mail_user *user) +{ + struct mail_namespace *ns; + + for (ns = user->namespaces; ns != NULL; ns = ns->next) { + if (ns->special_use_mailboxes) + return TRUE; + } + return FALSE; +} + struct client *client_create(int fd_in, int fd_out, const char *session_id, struct mail_user *user, struct mail_storage_service_user *service_user, @@ -143,6 +154,11 @@ if (!explicit_capability) str_append(client->capability_string, " METADATA"); } + if (!explicit_capability && user_has_special_use_mailboxes(user)) { + /* Advertise SPECIAL-USE only if there are actually some + SPECIAL-USE flags in mailbox configuration. */ + str_append(client->capability_string, " SPECIAL-USE"); + } ident = mail_user_get_anvil_userip_ident(client->user); if (ident != NULL) { From dovecot at dovecot.org Wed May 6 21:03:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 May 2015 21:03:07 +0000 Subject: dovecot-2.2: mbox: Fixed crash/corruption in some situations whe... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/94bd895721d8 changeset: 18534:94bd895721d8 user: Timo Sirainen date: Thu May 07 00:01:08 2015 +0300 description: mbox: Fixed crash/corruption in some situations when the first mail was expunged. This could be reproduced with default mbox settings, IMAP session that does - STORE 1 +FLAGS \DELETED - EXPUNGE With mbox containing: === >From root at example.com Tue Jan 13 10:18:16 2015 a >From root at example.com Tue Jan 13 10:18:20 2015 a === diffstat: src/lib-storage/index/mbox/mbox-sync.c | 11 +++++++++-- 1 files changed, 9 insertions(+), 2 deletions(-) diffs (38 lines): diff -r 18b71db74d31 -r 94bd895721d8 src/lib-storage/index/mbox/mbox-sync.c --- a/src/lib-storage/index/mbox/mbox-sync.c Wed May 06 19:50:23 2015 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync.c Thu May 07 00:01:08 2015 +0300 @@ -640,7 +640,7 @@ static int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx) { struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; - uoff_t orig_from_offset; + uoff_t orig_from_offset, postlf_from_offset = (uoff_t)-1; off_t move_diff; int ret; @@ -657,6 +657,7 @@ if (sync_ctx->first_mail_crlf_expunged) mail_ctx->mail.from_offset++; } + postlf_from_offset = mail_ctx->mail.from_offset; /* read the From-line before rewriting overwrites it */ if (mbox_read_from_line(mail_ctx) < 0) @@ -710,10 +711,16 @@ /* create dummy message to describe the expunged data */ struct mbox_sync_mail mail; + /* if this is going to be the first mail, increase the + from_offset to point to the beginning of the + From-line, because the previous [CR]LF is already + covered by expunged_space. */ + i_assert(postlf_from_offset != (uoff_t)-1); + mail_ctx->mail.from_offset = postlf_from_offset; + memset(&mail, 0, sizeof(mail)); mail.expunged = TRUE; mail.offset = mail.from_offset = - (sync_ctx->dest_first_mail ? 1 : 0) + mail_ctx->mail.from_offset - sync_ctx->expunged_space; mail.space = sync_ctx->expunged_space; From dovecot at dovecot.org Wed May 6 21:03:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 May 2015 21:03:07 +0000 Subject: dovecot-2.2: mbox: Added an extra assert Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/34b5abf6b9b7 changeset: 18535:34b5abf6b9b7 user: Timo Sirainen date: Thu May 07 00:01:16 2015 +0300 description: mbox: Added an extra assert diffstat: src/lib-storage/index/mbox/mbox-sync-rewrite.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 94bd895721d8 -r 34b5abf6b9b7 src/lib-storage/index/mbox/mbox-sync-rewrite.c --- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c Thu May 07 00:01:08 2015 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c Thu May 07 00:01:16 2015 +0300 @@ -435,6 +435,7 @@ if (first_nonexpunged && expunged_space > 0) { /* move From-line (after parsing headers so we don't overwrite them) */ + i_assert(mails[idx].from_offset >= expunged_space); if (mbox_move(sync_ctx, mails[idx].from_offset - expunged_space, mails[idx].from_offset, mails[idx].offset - mails[idx].from_offset) < 0) From pigeonhole at rename-it.nl Wed May 6 22:05:10 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 07 May 2015 00:05:10 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: storage: Implemented magic to... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/70a3e6c5bb1f changeset: 2044:70a3e6c5bb1f user: Stephan Bosch date: Wed May 06 23:58:57 2015 +0200 description: lib-sieve: storage: Implemented magic to make sieve_default script visible in main storage (e.g. from ManageSieve). diffstat: INSTALL | 49 ++- src/lib-sieve/sieve-script.c | 142 ++++++- src/lib-sieve/sieve-script.h | 16 +- src/lib-sieve/sieve-storage-private.h | 23 +- src/lib-sieve/sieve-storage.c | 370 +++++++++++++++++- src/lib-sieve/sieve-storage.h | 9 + src/lib-sieve/storage/file/sieve-file-storage-save.c | 74 ++- src/lib-sieve/storage/file/sieve-file-storage.c | 1 + src/lib-sieve/storage/file/sieve-file-storage.h | 10 +- src/plugins/lda-sieve/lda-sieve-plugin.c | 117 +----- 10 files changed, 627 insertions(+), 184 deletions(-) diffs (truncated from 1281 to 300 lines): diff -r a3d26f12c2cd -r 70a3e6c5bb1f INSTALL --- a/INSTALL Tue May 05 13:42:38 2015 +0200 +++ b/INSTALL Wed May 06 23:58:57 2015 +0200 @@ -126,6 +126,11 @@ Multiple mail users can share a single script directory if the script location is the same and all users share the same system credentials (uid, gid). + + default= + The name by which the default Sieve script (see `sieve_default=' setting + below) is visible to ManageSieve clients. Normally, it is not visible at + all. See "Visible Default Script" section below for more information. Sieve Interpreter - Basic Configuration --------------------------------------- @@ -166,7 +171,9 @@ /var/lib/dovecot/default.sieve. This is usually a global script, so be sure to pre-compile this script manually using the sievec command line tool, as explained in the README file. This setting used to be called - `sieve_global_path', but that name is now deprecated. + `sieve_global_path', but that name is now deprecated. See the "Visible + Default Script" section below for information on how to make the default + script visible from ManageSieve. sieve_global = Location for :global include scripts for the Sieve include extension. This @@ -388,6 +395,46 @@ to store the compiled binaries. In that case, be sure to manually pre-compile those scripts using the sievec tool, as explained in the README file. +Sieve Interpreter - Visible Default Script +------------------------------------------ + +The `sieve_default=' setting specifies the location of a default script that +is executed when the user has no active personal script. Normally, this +default script is invisible to the user; i.e., it is not listed in ManageSieve. +To give the user the ability to base a custom personal script on the default +script, it is possible to make it visible under a specific configurable name. + +ManageSieve will magically list the default script under that name, even though +it does not actually exist in the user's normal script storage location. This +way, the ManageSieve client can see that it exists and it can retrieve its +contents. If no normal script is active, the default is always listed as active. +The user can replace the default with a custom script, by uploading it under the +default script's name. If that custom script is ever deleted, the default script +will reappear from the shadows implicitly. + +This way, ManageSieve clients will not need any special handling for this +feature. If the name of the default script is equal to the name the client uses +for the main script, it will initially see and read the default script when the +user account is freshly created. The user can edit the script, and when the +edited script is saved through the ManageSieve client, it will will override the +default script. If the user ever wants to revert to the default, the user only +needs to delete the edited script and the default will reappear. + +To enable this feature, the the `;default=' option must be +specified for the `sieve=' setting. It configures the name by which the default +script will be known. Of course, the `sieve_default=' setting needs to point to +a valid script location as well for this to work. If the default script does not +exist at the indicated location, it is not shown. + +For example: + +plugin { +... + sieve = file:~/sieve;active=~/.dovecot.sieve;default=roundcube + + sieve_default = /var/lib/dovecot/sieve/default.sieve +} + Sieve Interpreter - Extension Configuration ------------------------------------------- diff -r a3d26f12c2cd -r 70a3e6c5bb1f src/lib-sieve/sieve-script.c --- a/src/lib-sieve/sieve-script.c Tue May 05 13:42:38 2015 +0200 +++ b/src/lib-sieve/sieve-script.c Wed May 06 23:58:57 2015 +0200 @@ -219,6 +219,24 @@ return script; } +int sieve_script_check +(struct sieve_instance *svinst, const char *location, const char *name, + enum sieve_error *error_r) +{ + struct sieve_script *script; + enum sieve_error error; + + if (error_r == NULL) + error_r = &error; + + script = sieve_script_create_open(svinst, location, name, error_r); + if (script == NULL) + return ( *error_r == SIEVE_ERROR_NOT_FOUND ? 0 : -1); + + sieve_script_unref(&script); + return 1; +} + /* * Properties */ @@ -261,6 +279,11 @@ return script->open; } +bool sieve_script_is_default(const struct sieve_script *script) +{ + return script->storage->is_default; +} + /* * Stream management */ @@ -525,15 +548,65 @@ return -1; } - i_assert( (script->storage->flags & SIEVE_STORAGE_FLAG_READWRITE) != 0 ); i_assert( script->open ); // FIXME: auto-open? - i_assert( script->v.rename != NULL ); - ret = script->v.rename(script, newname); + if ( storage->default_for == NULL ) { + i_assert( (storage->flags & SIEVE_STORAGE_FLAG_READWRITE) != 0 ); - /* rename INBOX mailbox attribute */ - if ( ret >= 0 && oldname != NULL ) - (void)sieve_storage_sync_script_rename(storage, oldname, newname); + /* rename script */ + i_assert( script->v.rename != NULL ); + ret = script->v.rename(script, newname); + + /* rename INBOX mailbox attribute */ + if ( ret >= 0 && oldname != NULL ) + (void)sieve_storage_sync_script_rename(storage, oldname, newname); + + } else if ( sieve_storage_check_script + (storage->default_for, newname, NULL) > 0 ) { + sieve_script_set_error(script, SIEVE_ERROR_EXISTS, + "A sieve script with that name already exists."); + sieve_storage_copy_error(storage->default_for, storage); + ret = -1; + + } else { + struct istream *input; + + /* copy from default */ + if ( (ret=sieve_script_open(script, NULL)) >= 0 && + (ret=sieve_script_get_stream(script, &input, NULL)) >= 0 ) { + ret = sieve_storage_save_as + (storage->default_for, input, newname); + + if ( ret < 0 ) { + sieve_storage_copy_error(storage, storage->default_for); + + } else if ( sieve_script_is_active(script) > 0 ) { + struct sieve_script *newscript; + enum sieve_error error; + + newscript = sieve_storage_open_script + (storage->default_for, newname, &error); + if ( newscript == NULL ) { + /* Somehow not actually saved */ + ret = ( error == SIEVE_ERROR_NOT_FOUND ? 0 : -1 ); + } else if ( sieve_script_activate(newscript, (time_t)-1) < 0 ) { + /* Failed to activate; roll back */ + ret = -1; + (void)sieve_script_delete(newscript); + sieve_script_unref(&newscript); + } + + if (ret < 0) { + sieve_storage_sys_error(storage, + "Failed to implicitly activate script `%s' " + "after rename", newname); + sieve_storage_copy_error(storage->default_for, storage); + } + } + } else { + sieve_storage_copy_error(storage->default_for, storage); + } + } return ret; } @@ -543,19 +616,28 @@ struct sieve_storage *storage = script->storage; int ret = 0; - i_assert( (script->storage->flags & SIEVE_STORAGE_FLAG_READWRITE) != 0 ); i_assert( script->open ); // FIXME: auto-open? /* Is the requested script active? */ - if ( sieve_script_is_active(script) ) { + if ( sieve_script_is_active(script) > 0 ) { sieve_script_set_error(script, SIEVE_ERROR_ACTIVE, "Cannot delete the active Sieve script."); - ret = -1; - } else { - i_assert( script->v.delete != NULL ); - ret = script->v.delete(script); + if (storage->default_for != NULL) + sieve_storage_copy_error(storage->default_for, storage); + return -1; } + /* Trying to delete the default script? */ + if ( storage->is_default ) { + /* ignore */ + return 0; + } + + i_assert( (script->storage->flags & SIEVE_STORAGE_FLAG_READWRITE) != 0 ); + + i_assert( script->v.delete != NULL ); + ret = script->v.delete(script); + /* unset INBOX mailbox attribute */ if ( ret >= 0 ) (void)sieve_storage_sync_script_delete(storage, script->name); @@ -564,6 +646,17 @@ int sieve_script_is_active(struct sieve_script *script) { + struct sieve_storage *storage = script->storage; + + /* Special handling if this is a default script */ + if ( storage->default_for != NULL ) { + int ret = sieve_storage_active_script_is_default + (storage->default_for); + if (ret < 0) + sieve_storage_copy_error(storage, storage->default_for); + return ret; + } + if ( script->v.is_active == NULL ) return 0; return script->v.is_active(script); @@ -572,17 +665,28 @@ int sieve_script_activate(struct sieve_script *script, time_t mtime) { struct sieve_storage *storage = script->storage; - int ret; + int ret = 0; - i_assert( (storage->flags & SIEVE_STORAGE_FLAG_READWRITE) != 0 ); i_assert( script->open ); // FIXME: auto-open? - i_assert( script->v.activate != NULL ); - ret = script->v.activate(script); + if (storage->default_for == NULL) { + i_assert( (storage->flags & SIEVE_STORAGE_FLAG_READWRITE) != 0 ); + + i_assert( script->v.activate != NULL ); + ret = script->v.activate(script); - if (ret >= 0) { - sieve_storage_set_modified(storage, mtime); - (void)sieve_storage_sync_script_activate(storage); + if (ret >= 0) { + sieve_storage_set_modified(storage, mtime); + (void)sieve_storage_sync_script_activate(storage); + } + + } else { + /* Activating the default script is equal to deactivating + the storage */ + ret = sieve_storage_deactivate + (storage->default_for, (time_t)-1); + if (ret < 0) + sieve_storage_copy_error(storage, storage->default_for); } return ret; diff -r a3d26f12c2cd -r 70a3e6c5bb1f src/lib-sieve/sieve-script.h --- a/src/lib-sieve/sieve-script.h Tue May 05 13:42:38 2015 +0200 +++ b/src/lib-sieve/sieve-script.h Wed May 06 23:58:57 2015 +0200 @@ -50,13 +50,17 @@ (struct sieve_script *script, enum sieve_error *error_r) ATTR_NULL(2); int sieve_script_open_as - (struct sieve_script *script, const char *name, enum sieve_error *error_r) - ATTR_NULL(3); + (struct sieve_script *script, const char *name, + enum sieve_error *error_r) ATTR_NULL(3); struct sieve_script *sieve_script_create_open - (struct sieve_instance *svinst, const char *location, const char *name, - enum sieve_error *error_r) - ATTR_NULL(3,4); + (struct sieve_instance *svinst, const char *location, + const char *name, enum sieve_error *error_r) + ATTR_NULL(3, 4); +int sieve_script_check + (struct sieve_instance *svinst, const char *location, + const char *name, enum sieve_error *error_r) + ATTR_NULL(3, 4); /* * Binary From dovecot at dovecot.org Thu May 7 08:29:49 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 07 May 2015 08:29:49 +0000 Subject: dovecot-2.2: ldap auth: If password is already verified (e.g. ma... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5dc00179dd60 changeset: 18536:5dc00179dd60 user: Timo Sirainen date: Thu May 07 11:21:33 2015 +0300 description: ldap auth: If password is already verified (e.g. master user login), skip LDAP auth binding. This happens only if auth_bind_userdn isn't set, i.e. it only makes sense if the LDAP DN lookup also returns some extra fields. diffstat: src/auth/passdb-ldap.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (16 lines): diff -r 34b5abf6b9b7 -r 5dc00179dd60 src/auth/passdb-ldap.c --- a/src/auth/passdb-ldap.c Thu May 07 00:01:16 2015 +0300 +++ b/src/auth/passdb-ldap.c Thu May 07 11:21:33 2015 +0300 @@ -260,6 +260,12 @@ } else if (res == NULL || passdb_ldap_request->entries != 1) { /* failure */ ldap_bind_lookup_dn_fail(auth_request, passdb_ldap_request, res); + } else if (auth_request->skip_password_check) { + /* we've already verified that the password matched - + we just wanted to get any extra fields */ + passdb_ldap_request->callback. + verify_plain(PASSDB_RESULT_OK, auth_request); + auth_request_unref(&auth_request); } else { /* create a new bind request */ brequest = p_new(auth_request->pool, From dovecot at dovecot.org Thu May 7 08:29:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 07 May 2015 08:29:50 +0000 Subject: dovecot-2.2: auth: Fixed credentials lookups via auth-workers wh... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8a3da4ef590f changeset: 18537:8a3da4ef590f user: Timo Sirainen date: Thu May 07 11:27:55 2015 +0300 description: auth: Fixed credentials lookups via auth-workers when no actual password was returned. For example LDAP lookup with auth_bind=yes should still return any extra fields. diffstat: src/auth/auth-worker-client.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r 5dc00179dd60 -r 8a3da4ef590f src/auth/auth-worker-client.c --- a/src/auth/auth-worker-client.c Thu May 07 11:21:33 2015 +0300 +++ b/src/auth/auth-worker-client.c Thu May 07 11:27:55 2015 +0300 @@ -241,7 +241,9 @@ else { str_append(str, "OK\t"); str_append_tabescaped(str, request->user); - str_printfa(str, "\t{%s.b64}", request->credentials_scheme); + str_append_c(str, '\t'); + if (request->credentials_scheme[0] != '\0') + str_printfa(str, "{%s.b64}", request->credentials_scheme); base64_encode(credentials, size, str); reply_append_extra_fields(str, request); } From dovecot at dovecot.org Thu May 7 14:20:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 07 May 2015 14:20:36 +0000 Subject: dovecot-2.2: auth: Added assert to make sure previous change is ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d3332ee1d26a changeset: 18538:d3332ee1d26a user: Timo Sirainen date: Thu May 07 17:18:44 2015 +0300 description: auth: Added assert to make sure previous change is correct. diffstat: src/auth/auth-worker-client.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 8a3da4ef590f -r d3332ee1d26a src/auth/auth-worker-client.c --- a/src/auth/auth-worker-client.c Thu May 07 11:27:55 2015 +0300 +++ b/src/auth/auth-worker-client.c Thu May 07 17:18:44 2015 +0300 @@ -242,9 +242,12 @@ str_append(str, "OK\t"); str_append_tabescaped(str, request->user); str_append_c(str, '\t'); - if (request->credentials_scheme[0] != '\0') + if (request->credentials_scheme[0] != '\0') { str_printfa(str, "{%s.b64}", request->credentials_scheme); - base64_encode(credentials, size, str); + base64_encode(credentials, size, str); + } else { + i_assert(size == 0); + } reply_append_extra_fields(str, request); } str_append_c(str, '\n'); From dovecot at dovecot.org Thu May 7 14:30:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 07 May 2015 14:30:52 +0000 Subject: dovecot-2.2: exmaple-config: Added lmtp_hdr_delivery_address set... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6417e972a602 changeset: 18539:6417e972a602 user: Timo Sirainen date: Thu May 07 17:29:00 2015 +0300 description: exmaple-config: Added lmtp_hdr_delivery_address setting. diffstat: doc/example-config/conf.d/20-lmtp.conf | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diffs (20 lines): diff -r d3332ee1d26a -r 6417e972a602 doc/example-config/conf.d/20-lmtp.conf --- a/doc/example-config/conf.d/20-lmtp.conf Thu May 07 17:18:44 2015 +0300 +++ b/doc/example-config/conf.d/20-lmtp.conf Thu May 07 17:29:00 2015 +0300 @@ -13,8 +13,14 @@ # Verify quota before replying to RCPT TO. This adds a small overhead. #lmtp_rcpt_check_quota = no +# Which recipient address to use for Delivered-To: header and Recipient: +# header. The default is "final", which is the same as the one given to +# RCPT TO command. "original" uses the address given in RCPT TO's ORCPT +# parameter, "none" uses nothing. Note that "none" is currently always used +# when a mail has multiple recipients. +#lmtp_hdr_delivery_address = final + protocol lmtp { # Space separated list of plugins to load (default is global mail_plugins). #mail_plugins = $mail_plugins } - \ No newline at end of file From dovecot at dovecot.org Thu May 7 14:31:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 07 May 2015 14:31:50 +0000 Subject: dovecot-2.2: example-config: s/Recipient:/Received:/ Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7a972218b15e changeset: 18540:7a972218b15e user: Timo Sirainen date: Thu May 07 17:29:55 2015 +0300 description: example-config: s/Recipient:/Received:/ diffstat: doc/example-config/conf.d/20-lmtp.conf | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 6417e972a602 -r 7a972218b15e doc/example-config/conf.d/20-lmtp.conf --- a/doc/example-config/conf.d/20-lmtp.conf Thu May 07 17:29:00 2015 +0300 +++ b/doc/example-config/conf.d/20-lmtp.conf Thu May 07 17:29:55 2015 +0300 @@ -13,7 +13,7 @@ # Verify quota before replying to RCPT TO. This adds a small overhead. #lmtp_rcpt_check_quota = no -# Which recipient address to use for Delivered-To: header and Recipient: +# Which recipient address to use for Delivered-To: header and Received: # header. The default is "final", which is the same as the one given to # RCPT TO command. "original" uses the address given in RCPT TO's ORCPT # parameter, "none" uses nothing. Note that "none" is currently always used From dovecot at dovecot.org Thu May 7 15:06:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 07 May 2015 15:06:33 +0000 Subject: dovecot-2.2: director: Fixed "doveadm director status " lo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/04ac1a15e6e4 changeset: 18541:04ac1a15e6e4 user: Timo Sirainen date: Thu May 07 18:04:33 2015 +0300 description: director: Fixed "doveadm director status " lookup to return "Current" correctly. diffstat: src/director/doveadm-connection.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 7a972218b15e -r 04ac1a15e6e4 src/director/doveadm-connection.c --- a/src/director/doveadm-connection.c Thu May 07 17:29:55 2015 +0300 +++ b/src/director/doveadm-connection.c Thu May 07 18:04:33 2015 +0300 @@ -366,7 +366,7 @@ tag = args[1] != NULL ? args[1] : ""; } if (str_to_uint(username, &username_hash) < 0) - username_hash = user_directory_get_username_hash(conn->dir->users, line); + username_hash = user_directory_get_username_hash(conn->dir->users, username); /* get user's current host */ user = user_directory_lookup(conn->dir->users, username_hash); From dovecot at dovecot.org Thu May 7 15:23:27 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 07 May 2015 15:23:27 +0000 Subject: dovecot-2.2: lib-storage: Fixed LAYOUT=imapdir to work again Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c45fe041ccfb changeset: 18542:c45fe041ccfb user: Timo Sirainen date: Thu May 07 18:21:35 2015 +0300 description: lib-storage: Fixed LAYOUT=imapdir to work again Patch by Jason Gunthorpe diffstat: src/lib-storage/list/mailbox-list-maildir.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 04ac1a15e6e4 -r c45fe041ccfb src/lib-storage/list/mailbox-list-maildir.c --- a/src/lib-storage/list/mailbox-list-maildir.c Thu May 07 18:04:33 2015 +0300 +++ b/src/lib-storage/list/mailbox-list-maildir.c Thu May 07 18:21:35 2015 +0300 @@ -46,6 +46,7 @@ list = p_new(pool, struct maildir_mailbox_list, 1); list->list = imapdir_mailbox_list; list->list.pool = pool; + list->sep = '.'; list->global_temp_prefix = IMAPDIR_GLOBAL_TEMP_PREFIX; list->temp_prefix = p_strconcat(pool, list->global_temp_prefix, From dovecot at dovecot.org Thu May 7 17:27:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 07 May 2015 17:27:39 +0000 Subject: dovecot-2.2: lib-fs: Added fs-compress wrapper. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4c8fc477d68a changeset: 18543:4c8fc477d68a user: Timo Sirainen date: Thu May 07 20:25:44 2015 +0300 description: lib-fs: Added fs-compress wrapper. Future TODO could include automatically detecting the format of the input file, but this should be optional. diffstat: src/lib-fs/Makefile.am | 10 + src/lib-fs/fs-api-private.h | 1 + src/lib-fs/fs-api.c | 1 + src/lib-fs/fs-compress.c | 416 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 428 insertions(+), 0 deletions(-) diffs (truncated from 470 to 300 lines): diff -r c45fe041ccfb -r 4c8fc477d68a src/lib-fs/Makefile.am --- a/src/lib-fs/Makefile.am Thu May 07 18:21:35 2015 +0300 +++ b/src/lib-fs/Makefile.am Thu May 07 20:25:44 2015 +0300 @@ -1,7 +1,12 @@ noinst_LTLIBRARIES = libfs.la +fs_moduledir = $(moduledir) +fs_module_LTLIBRARIES = \ + libfs_compress.la + AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-compression \ -I$(top_srcdir)/src/lib-ssl-iostream \ -DMODULE_DIR=\""$(moduledir)"\" @@ -17,6 +22,11 @@ ostream-metawrap.c \ ostream-cmp.c +libfs_compress_la_SOURCES = fs-compress.c +libfs_compress_la_LIBADD = ../lib-compression/libdovecot-compression.la +libfs_compress_la_DEPS = ../lib-compression/libdovecot-compression.la +libfs_compress_la_LDFLAGS = -module -avoid-version + headers = \ fs-api.h \ fs-api-private.h \ diff -r c45fe041ccfb -r 4c8fc477d68a src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Thu May 07 18:21:35 2015 +0300 +++ b/src/lib-fs/fs-api-private.h Thu May 07 20:25:44 2015 +0300 @@ -134,6 +134,7 @@ void *async_context; }; +extern const struct fs fs_class_compress; extern const struct fs fs_class_posix; extern const struct fs fs_class_metawrap; extern const struct fs fs_class_sis; diff -r c45fe041ccfb -r 4c8fc477d68a src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Thu May 07 18:21:35 2015 +0300 +++ b/src/lib-fs/fs-api.c Thu May 07 20:25:44 2015 +0300 @@ -63,6 +63,7 @@ static void fs_classes_init(void) { i_array_init(&fs_classes, 8); + fs_class_register(&fs_class_compress); fs_class_register(&fs_class_posix); fs_class_register(&fs_class_metawrap); fs_class_register(&fs_class_sis); diff -r c45fe041ccfb -r 4c8fc477d68a src/lib-fs/fs-compress.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fs/fs-compress.c Thu May 07 20:25:44 2015 +0300 @@ -0,0 +1,416 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "ostream.h" +#include "iostream-temp.h" +#include "compression.h" +#include "fs-api-private.h" + +struct compress_fs { + struct fs fs; + const struct compression_handler *handler; + unsigned int compress_level; +}; + +struct compress_fs_file { + struct fs_file file; + struct compress_fs *fs; + struct fs_file *super, *super_read; + enum fs_open_mode open_mode; + struct istream *input; + + struct ostream *super_output; + struct ostream *temp_output; +}; + +struct compress_fs_iter { + struct fs_iter iter; + struct fs_iter *super; +}; + +static struct fs *fs_compress_alloc(void) +{ + struct compress_fs *fs; + + fs = i_new(struct compress_fs, 1); + fs->fs = fs_class_compress; + return &fs->fs; +} + +static int +fs_compress_init(struct fs *_fs, const char *args, const + struct fs_settings *set) +{ + struct compress_fs *fs = (struct compress_fs *)_fs; + const char *p, *compression_name, *level_str, *error; + const char *parent_name, *parent_args; + + /* get compression handler name */ + p = strchr(args, ':'); + if (p == NULL) { + fs_set_error(_fs, "Compression method not given as parameter"); + return -1; + } + compression_name = t_strdup_until(args, p++); + args = p; + + /* get compression level */ + p = strchr(args, ':'); + if (p == NULL || p[1] == '\0') { + fs_set_error(_fs, "Parent filesystem not given as parameter"); + return -1; + } + + level_str = t_strdup_until(args, p++); + if (str_to_uint(level_str, &fs->compress_level) < 0 || + fs->compress_level < 1 || fs->compress_level > 9) { + fs_set_error(_fs, "Invalid compression level parameter '%s'", level_str); + return -1; + } + args = p; + + fs->handler = compression_lookup_handler(compression_name); + if (fs->handler == NULL) { + fs_set_error(_fs, "Compression method '%s' not support", compression_name); + return -1; + } + + parent_args = strchr(args, ':'); + if (parent_args == NULL) { + parent_name = args; + parent_args = ""; + } else { + parent_name = t_strdup_until(args, parent_args); + parent_args++; + } + if (fs_init(parent_name, parent_args, set, &_fs->parent, &error) < 0) { + fs_set_error(_fs, "%s: %s", parent_name, error); + return -1; + } + return 0; +} + +static void fs_compress_deinit(struct fs *_fs) +{ + struct compress_fs *fs = (struct compress_fs *)_fs; + + if (_fs->parent != NULL) + fs_deinit(&_fs->parent); + i_free(fs); +} + +static enum fs_properties fs_compress_get_properties(struct fs *_fs) +{ + return fs_get_properties(_fs->parent); +} + +static struct fs_file * +fs_compress_file_init(struct fs *_fs, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags) +{ + struct compress_fs *fs = (struct compress_fs *)_fs; + struct compress_fs_file *file; + + file = i_new(struct compress_fs_file, 1); + file->file.fs = _fs; + file->file.path = i_strdup(path); + file->fs = fs; + file->open_mode = mode; + + /* avoid unnecessarily creating two seekable streams */ + flags &= ~FS_OPEN_FLAG_SEEKABLE; + + file->super = fs_file_init(_fs->parent, path, mode | flags); + if (mode == FS_OPEN_MODE_READONLY && + (flags & FS_OPEN_FLAG_ASYNC) == 0) { + /* use async stream for super, so fs_read_stream() won't create + another seekable stream unneededly */ + file->super_read = fs_file_init(_fs->parent, path, mode | flags | + FS_OPEN_FLAG_ASYNC); + } else { + file->super_read = file->super; + } + return &file->file; +} + +static void fs_compress_file_deinit(struct fs_file *_file) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + if (file->super_read != file->super && file->super_read != NULL) + fs_file_deinit(&file->super_read); + fs_file_deinit(&file->super); + i_free(file->file.path); + i_free(file); +} + +static void fs_compress_file_close(struct fs_file *_file) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + if (file->input != NULL) + i_stream_unref(&file->input); + if (file->super_read != NULL) + fs_file_close(file->super_read); + if (file->super != NULL) + fs_file_close(file->super); +} + +static const char *fs_compress_file_get_path(struct fs_file *_file) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + return fs_file_path(file->super); +} + +static void +fs_compress_set_async_callback(struct fs_file *_file, + fs_file_async_callback_t *callback, + void *context) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + fs_file_set_async_callback(file->super, callback, context); +} + +static int fs_compress_wait_async(struct fs *_fs) +{ + return fs_wait_async(_fs->parent); +} + +static void +fs_compress_set_metadata(struct fs_file *_file, const char *key, + const char *value) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + fs_set_metadata(file->super, key, value); +} + +static int +fs_compress_get_metadata(struct fs_file *_file, + const ARRAY_TYPE(fs_metadata) **metadata_r) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + return fs_get_metadata(file->super, metadata_r); +} + +static bool fs_compress_prefetch(struct fs_file *_file, uoff_t length) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + return fs_prefetch(file->super, length); +} + +static struct istream * +fs_compress_read_stream(struct fs_file *_file, size_t max_buffer_size) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + struct istream *input; + + if (file->input != NULL) { + i_stream_ref(file->input); + i_stream_seek(file->input, 0); + return file->input; + } + + input = fs_read_stream(file->super_read, max_buffer_size); + file->input = file->fs->handler->create_istream(input, FALSE); + i_stream_unref(&input); + i_stream_ref(file->input); + return file->input; +} + +static void fs_compress_write_stream(struct fs_file *_file) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + i_assert(_file->output == NULL); + + file->temp_output = + iostream_temp_create_named(_file->fs->temp_path_prefix, + IOSTREAM_TEMP_FLAG_TRY_FD_DUP, + fs_file_path(_file)); + _file->output = file->fs->handler-> + create_ostream(file->temp_output, file->fs->compress_level); +} + +static int fs_compress_write_stream_finish(struct fs_file *_file, bool success) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + struct istream *input; + int ret; + From dovecot at dovecot.org Thu May 7 17:37:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 07 May 2015 17:37:25 +0000 Subject: dovecot-2.2: lib-fs: Fixed fs-compress code to actually build an... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7516211ff7e6 changeset: 18544:7516211ff7e6 user: Timo Sirainen date: Thu May 07 20:35:23 2015 +0300 description: lib-fs: Fixed fs-compress code to actually build and run. It's a plugin (because it depends on extra compression libraries). We want to register it only when it's explicitly attempted to be used, not before. diffstat: src/lib-fs/fs-api-private.h | 1 - src/lib-fs/fs-api.c | 1 - src/lib-fs/fs-compress.c | 2 ++ 3 files changed, 2 insertions(+), 2 deletions(-) diffs (34 lines): diff -r 4c8fc477d68a -r 7516211ff7e6 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Thu May 07 20:25:44 2015 +0300 +++ b/src/lib-fs/fs-api-private.h Thu May 07 20:35:23 2015 +0300 @@ -134,7 +134,6 @@ void *async_context; }; -extern const struct fs fs_class_compress; extern const struct fs fs_class_posix; extern const struct fs fs_class_metawrap; extern const struct fs fs_class_sis; diff -r 4c8fc477d68a -r 7516211ff7e6 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Thu May 07 20:25:44 2015 +0300 +++ b/src/lib-fs/fs-api.c Thu May 07 20:35:23 2015 +0300 @@ -63,7 +63,6 @@ static void fs_classes_init(void) { i_array_init(&fs_classes, 8); - fs_class_register(&fs_class_compress); fs_class_register(&fs_class_posix); fs_class_register(&fs_class_metawrap); fs_class_register(&fs_class_sis); diff -r 4c8fc477d68a -r 7516211ff7e6 src/lib-fs/fs-compress.c --- a/src/lib-fs/fs-compress.c Thu May 07 20:25:44 2015 +0300 +++ b/src/lib-fs/fs-compress.c Thu May 07 20:35:23 2015 +0300 @@ -30,6 +30,8 @@ struct fs_iter *super; }; +extern const struct fs fs_class_compress; + static struct fs *fs_compress_alloc(void) { struct compress_fs *fs; From pigeonhole at rename-it.nl Thu May 7 19:10:55 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 07 May 2015 21:10:55 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: storage: Changed configuratio... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/d4aa9ddf3efd changeset: 2045:d4aa9ddf3efd user: Stephan Bosch date: Thu May 07 21:10:37 2015 +0200 description: lib-sieve: storage: Changed configuration of default script visibility feature. Using a location option for the default name makes no sense if its only used for the main personal script. diffstat: INSTALL | 31 +++++++++++++++---------------- src/lib-sieve/sieve-storage.c | 36 ++++++++++++++++-------------------- 2 files changed, 31 insertions(+), 36 deletions(-) diffs (168 lines): diff -r 70a3e6c5bb1f -r d4aa9ddf3efd INSTALL --- a/INSTALL Wed May 06 23:58:57 2015 +0200 +++ b/INSTALL Thu May 07 21:10:37 2015 +0200 @@ -126,11 +126,6 @@ Multiple mail users can share a single script directory if the script location is the same and all users share the same system credentials (uid, gid). - - default= - The name by which the default Sieve script (see `sieve_default=' setting - below) is visible to ManageSieve clients. Normally, it is not visible at - all. See "Visible Default Script" section below for more information. Sieve Interpreter - Basic Configuration --------------------------------------- @@ -171,9 +166,12 @@ /var/lib/dovecot/default.sieve. This is usually a global script, so be sure to pre-compile this script manually using the sievec command line tool, as explained in the README file. This setting used to be called - `sieve_global_path', but that name is now deprecated. See the "Visible - Default Script" section below for information on how to make the default - script visible from ManageSieve. + `sieve_global_path', but that name is now deprecated. + + sieve_default_name = + The name by which the default Sieve script is visible to ManageSieve + clients. Normally, it is not visible at all. See "Visible Default Script" + section below for more information. sieve_global = Location for :global include scripts for the Sieve include extension. This @@ -401,8 +399,9 @@ The `sieve_default=' setting specifies the location of a default script that is executed when the user has no active personal script. Normally, this default script is invisible to the user; i.e., it is not listed in ManageSieve. -To give the user the ability to base a custom personal script on the default -script, it is possible to make it visible under a specific configurable name. +To give the user the ability to see and read the default script, it is possible +to make it visible under a specific configurable name using the +`sieve_default_name' setting. ManageSieve will magically list the default script under that name, even though it does not actually exist in the user's normal script storage location. This @@ -420,19 +419,19 @@ default script. If the user ever wants to revert to the default, the user only needs to delete the edited script and the default will reappear. -To enable this feature, the the `;default=' option must be -specified for the `sieve=' setting. It configures the name by which the default -script will be known. Of course, the `sieve_default=' setting needs to point to -a valid script location as well for this to work. If the default script does not -exist at the indicated location, it is not shown. +The name by which the default script will be known is configured using the +`sieve_default_name' setting. Of course, the `sieve_default' setting needs to +point to a valid script location as well for this to work. If the default script +does not exist at the indicated location, it is not shown. For example: plugin { ... - sieve = file:~/sieve;active=~/.dovecot.sieve;default=roundcube + sieve = file:~/sieve;active=~/.dovecot.sieve sieve_default = /var/lib/dovecot/sieve/default.sieve + sieve_default_name = roundcube } Sieve Interpreter - Extension Configuration diff -r 70a3e6c5bb1f -r d4aa9ddf3efd src/lib-sieve/sieve-storage.c --- a/src/lib-sieve/sieve-storage.c Wed May 06 23:58:57 2015 +0200 +++ b/src/lib-sieve/sieve-storage.c Thu May 07 21:10:37 2015 +0200 @@ -199,15 +199,6 @@ if ( storage->script_name == NULL ) storage->script_name = p_strdup(storage->pool, option+5); - } else if ( strncasecmp(option, "default=", 8) == 0 ) { - if ( option[8] == '\0' ) { - /* skip if empty */ - continue; - } - - if ( storage->default_name == NULL ) - storage->default_name = p_strdup(storage->pool, option+8); - } else if ( strncasecmp(option, "bindir=", 7) == 0 ) { const char *bin_dir = option+7; @@ -479,7 +470,7 @@ enum sieve_storage_flags flags, enum sieve_error *error_r) { struct sieve_storage *storage; - const char *set_sieve_default; + const char *set_default, *set_default_name; enum sieve_error error; if ( error_r != NULL ) @@ -488,21 +479,26 @@ error_r = &error; /* Determine location for default script */ - set_sieve_default = + set_default = sieve_setting_get(svinst, "sieve_default"); - if ( set_sieve_default == NULL ) { + if ( set_default == NULL ) { /* For backwards compatibility */ - set_sieve_default = + set_default = sieve_setting_get(svinst, "sieve_global_path"); } + set_default_name = + sieve_setting_get(svinst, "sieve_default_name"); /* Attempt to locate user's main storage */ storage = sieve_storage_do_create_main(svinst, user, flags, error_r); + storage->default_name = + p_strdup_empty(storage->pool, set_default_name); + if ( storage != NULL ) { /* Success; record default script location for later use */ storage->default_location = - p_strdup_empty(storage->pool, set_sieve_default); + p_strdup_empty(storage->pool, set_default); if (storage->default_location != NULL && storage->default_name != NULL) { @@ -516,33 +512,33 @@ /* Failed; try using default script location (not for temporary failures, read/write access, or dsync) */ - if ( set_sieve_default == NULL ) { + if ( set_default == NULL ) { sieve_sys_debug(svinst, "storage: " "No default script location configured"); } else { sieve_sys_debug(svinst, "storage: " "Trying default script location `%s'", - set_sieve_default); + set_default); storage = sieve_storage_create - (svinst, set_sieve_default, 0, error_r); + (svinst, set_default, 0, error_r); if ( storage == NULL ) { switch ( *error_r ) { case SIEVE_ERROR_NOT_FOUND: sieve_sys_debug(svinst, "storage: " "Default script location `%s' not found", - set_sieve_default); + set_default); break; case SIEVE_ERROR_TEMP_FAILURE: sieve_sys_error(svinst, "storage: " "Failed to access default script location `%s' " "(temporary failure)", - set_sieve_default); + set_default); break; default: sieve_sys_error(svinst, "storage: " "Failed to access default script location `%s'", - set_sieve_default); + set_default); break; } } From pigeonhole at rename-it.nl Thu May 7 19:17:59 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 07 May 2015 21:17:59 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: file storage: The sieve_scrip... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/eeef7010b52e changeset: 2046:eeef7010b52e user: Stephan Bosch date: Thu May 07 21:17:40 2015 +0200 description: lib-sieve: file storage: The sieve_script_is_active() function returns int rather than bool. diffstat: src/lib-sieve/storage/file/sieve-file-script.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d4aa9ddf3efd -r eeef7010b52e src/lib-sieve/storage/file/sieve-file-script.c --- a/src/lib-sieve/storage/file/sieve-file-script.c Thu May 07 21:10:37 2015 +0200 +++ b/src/lib-sieve/storage/file/sieve-file-script.c Thu May 07 21:17:40 2015 +0200 @@ -686,7 +686,7 @@ ret = link(fscript->path, newpath); if ( ret >= 0 ) { /* Is the requested script active? */ - if ( sieve_script_is_active(script) ) { + if ( sieve_script_is_active(script) > 0 ) { /* Active; make active link point to the new copy */ i_assert( fstorage->link_path != NULL ); link_path = t_strconcat From pigeonhole at rename-it.nl Thu May 7 19:49:43 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 07 May 2015 21:49:43 +0200 Subject: dovecot-2.2-pigeonhole: LDA Sieve plugin: Fixed spurious error c... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/1414446ef704 changeset: 2048:1414446ef704 user: Stephan Bosch date: Thu May 07 21:46:57 2015 +0200 description: LDA Sieve plugin: Fixed spurious error caused by earlier changes. diffstat: src/plugins/lda-sieve/lda-sieve-plugin.c | 5 ++--- 1 files changed, 2 insertions(+), 3 deletions(-) diffs (22 lines): diff -r eb001d7bbabf -r 1414446ef704 src/plugins/lda-sieve/lda-sieve-plugin.c --- a/src/plugins/lda-sieve/lda-sieve-plugin.c Thu May 07 21:41:24 2015 +0200 +++ b/src/plugins/lda-sieve/lda-sieve-plugin.c Thu May 07 21:46:57 2015 +0200 @@ -645,7 +645,7 @@ srctx->main_script = sieve_storage_active_script_open(main_storage, &error); - if ( srctx->user_script == NULL ) { + if ( srctx->main_script == NULL ) { switch ( error ) { case SIEVE_ERROR_NOT_FOUND: sieve_sys_debug(svinst, @@ -661,8 +661,7 @@ break; default: sieve_sys_error(svinst, - "Failed to access active Sieve script in user storage `%s' " - "(trying default script location instead)", + "Failed to access active Sieve script in user storage `%s'", sieve_storage_location(main_storage)); break; } From pigeonhole at rename-it.nl Thu May 7 19:49:43 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 07 May 2015 21:49:43 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: storage: Check Sieve script n... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/eb001d7bbabf changeset: 2047:eb001d7bbabf user: Stephan Bosch date: Thu May 07 21:41:24 2015 +0200 description: lib-sieve: storage: Check Sieve script name validity in configuration. diffstat: src/lib-sieve/sieve-storage.c | 20 +++++++++++++++++--- 1 files changed, 17 insertions(+), 3 deletions(-) diffs (42 lines): diff -r eeef7010b52e -r eb001d7bbabf src/lib-sieve/sieve-storage.c --- a/src/lib-sieve/sieve-storage.c Thu May 07 21:17:40 2015 +0200 +++ b/src/lib-sieve/sieve-storage.c Thu May 07 21:41:24 2015 +0200 @@ -196,8 +196,16 @@ return -1; } - if ( storage->script_name == NULL ) + if ( storage->script_name == NULL ) { + if ( !sieve_script_name_is_valid(option+5) ) { + sieve_storage_sys_error(storage, + "Failed to parse storage location: " + "Invalid script name `%s'.", + str_sanitize(option+5, 80)); + return -1; + } storage->script_name = p_strdup(storage->pool, option+5); + } } else if ( strncasecmp(option, "bindir=", 7) == 0 ) { const char *bin_dir = option+7; @@ -486,12 +494,18 @@ set_default = sieve_setting_get(svinst, "sieve_global_path"); } - set_default_name = - sieve_setting_get(svinst, "sieve_default_name"); /* Attempt to locate user's main storage */ storage = sieve_storage_do_create_main(svinst, user, flags, error_r); + set_default_name = + sieve_setting_get(svinst, "sieve_default_name"); + if ( !sieve_script_name_is_valid(set_default_name) ) { + sieve_storage_sys_error(storage, + "Invalid script name `%s' for `sieve_default_name' setting.", + str_sanitize(set_default_name, 80)); + set_default_name = NULL; + } storage->default_name = p_strdup_empty(storage->pool, set_default_name); From pigeonhole at rename-it.nl Thu May 7 20:05:45 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 07 May 2015 22:05:45 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: storage: Forgot to check whet... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/f050c1de127c changeset: 2049:f050c1de127c user: Stephan Bosch date: Thu May 07 22:05:23 2015 +0200 description: lib-sieve: storage: Forgot to check whether sieve_default_name setting exists. This caused a segfault in the test suite. diffstat: src/lib-sieve/sieve-storage.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 1414446ef704 -r f050c1de127c src/lib-sieve/sieve-storage.c --- a/src/lib-sieve/sieve-storage.c Thu May 07 21:46:57 2015 +0200 +++ b/src/lib-sieve/sieve-storage.c Thu May 07 22:05:23 2015 +0200 @@ -500,7 +500,8 @@ set_default_name = sieve_setting_get(svinst, "sieve_default_name"); - if ( !sieve_script_name_is_valid(set_default_name) ) { + if ( set_default_name != NULL && *set_default_name != '\0' && + !sieve_script_name_is_valid(set_default_name) ) { sieve_storage_sys_error(storage, "Invalid script name `%s' for `sieve_default_name' setting.", str_sanitize(set_default_name, 80)); From dovecot at dovecot.org Fri May 8 07:46:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 08 May 2015 07:46:11 +0000 Subject: dovecot-2.2: Moved fs-compress from lib-fs to lib-compression. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f2a8e1793718 changeset: 18545:f2a8e1793718 user: Timo Sirainen date: Fri May 08 10:44:18 2015 +0300 description: Moved fs-compress from lib-fs to lib-compression. This solves build ordering issues. diffstat: src/lib-compression/Makefile.am | 10 + src/lib-compression/fs-compress.c | 418 ++++++++++++++++++++++++++++++++++++++ src/lib-fs/Makefile.am | 10 - src/lib-fs/fs-compress.c | 418 -------------------------------------- 4 files changed, 428 insertions(+), 428 deletions(-) diffs (truncated from 900 to 300 lines): diff -r 7516211ff7e6 -r f2a8e1793718 src/lib-compression/Makefile.am --- a/src/lib-compression/Makefile.am Thu May 07 20:35:23 2015 +0300 +++ b/src/lib-compression/Makefile.am Fri May 08 10:44:18 2015 +0300 @@ -1,7 +1,12 @@ noinst_LTLIBRARIES = libcompression.la +fs_moduledir = $(moduledir) +fs_module_LTLIBRARIES = \ + libfs_compress.la + AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-fs \ -I$(top_srcdir)/src/lib-test libcompression_la_SOURCES = \ @@ -30,6 +35,11 @@ libdovecot_compression_la_DEPENDENCIES = libcompression.la ../lib-dovecot/libdovecot.la libdovecot_compression_la_LDFLAGS = -export-dynamic +libfs_compress_la_SOURCES = fs-compress.c +libfs_compress_la_LIBADD = libdovecot-compression.la +libfs_compress_la_DEPS = libdovecot-compression.la +libfs_compress_la_LDFLAGS = -module -avoid-version + test_programs = \ test-compression diff -r 7516211ff7e6 -r f2a8e1793718 src/lib-compression/fs-compress.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-compression/fs-compress.c Fri May 08 10:44:18 2015 +0300 @@ -0,0 +1,418 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "ostream.h" +#include "iostream-temp.h" +#include "compression.h" +#include "fs-api-private.h" + +struct compress_fs { + struct fs fs; + const struct compression_handler *handler; + unsigned int compress_level; +}; + +struct compress_fs_file { + struct fs_file file; + struct compress_fs *fs; + struct fs_file *super, *super_read; + enum fs_open_mode open_mode; + struct istream *input; + + struct ostream *super_output; + struct ostream *temp_output; +}; + +struct compress_fs_iter { + struct fs_iter iter; + struct fs_iter *super; +}; + +extern const struct fs fs_class_compress; + +static struct fs *fs_compress_alloc(void) +{ + struct compress_fs *fs; + + fs = i_new(struct compress_fs, 1); + fs->fs = fs_class_compress; + return &fs->fs; +} + +static int +fs_compress_init(struct fs *_fs, const char *args, const + struct fs_settings *set) +{ + struct compress_fs *fs = (struct compress_fs *)_fs; + const char *p, *compression_name, *level_str, *error; + const char *parent_name, *parent_args; + + /* get compression handler name */ + p = strchr(args, ':'); + if (p == NULL) { + fs_set_error(_fs, "Compression method not given as parameter"); + return -1; + } + compression_name = t_strdup_until(args, p++); + args = p; + + /* get compression level */ + p = strchr(args, ':'); + if (p == NULL || p[1] == '\0') { + fs_set_error(_fs, "Parent filesystem not given as parameter"); + return -1; + } + + level_str = t_strdup_until(args, p++); + if (str_to_uint(level_str, &fs->compress_level) < 0 || + fs->compress_level < 1 || fs->compress_level > 9) { + fs_set_error(_fs, "Invalid compression level parameter '%s'", level_str); + return -1; + } + args = p; + + fs->handler = compression_lookup_handler(compression_name); + if (fs->handler == NULL) { + fs_set_error(_fs, "Compression method '%s' not support", compression_name); + return -1; + } + + parent_args = strchr(args, ':'); + if (parent_args == NULL) { + parent_name = args; + parent_args = ""; + } else { + parent_name = t_strdup_until(args, parent_args); + parent_args++; + } + if (fs_init(parent_name, parent_args, set, &_fs->parent, &error) < 0) { + fs_set_error(_fs, "%s: %s", parent_name, error); + return -1; + } + return 0; +} + +static void fs_compress_deinit(struct fs *_fs) +{ + struct compress_fs *fs = (struct compress_fs *)_fs; + + if (_fs->parent != NULL) + fs_deinit(&_fs->parent); + i_free(fs); +} + +static enum fs_properties fs_compress_get_properties(struct fs *_fs) +{ + return fs_get_properties(_fs->parent); +} + +static struct fs_file * +fs_compress_file_init(struct fs *_fs, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags) +{ + struct compress_fs *fs = (struct compress_fs *)_fs; + struct compress_fs_file *file; + + file = i_new(struct compress_fs_file, 1); + file->file.fs = _fs; + file->file.path = i_strdup(path); + file->fs = fs; + file->open_mode = mode; + + /* avoid unnecessarily creating two seekable streams */ + flags &= ~FS_OPEN_FLAG_SEEKABLE; + + file->super = fs_file_init(_fs->parent, path, mode | flags); + if (mode == FS_OPEN_MODE_READONLY && + (flags & FS_OPEN_FLAG_ASYNC) == 0) { + /* use async stream for super, so fs_read_stream() won't create + another seekable stream unneededly */ + file->super_read = fs_file_init(_fs->parent, path, mode | flags | + FS_OPEN_FLAG_ASYNC); + } else { + file->super_read = file->super; + } + return &file->file; +} + +static void fs_compress_file_deinit(struct fs_file *_file) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + if (file->super_read != file->super && file->super_read != NULL) + fs_file_deinit(&file->super_read); + fs_file_deinit(&file->super); + i_free(file->file.path); + i_free(file); +} + +static void fs_compress_file_close(struct fs_file *_file) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + if (file->input != NULL) + i_stream_unref(&file->input); + if (file->super_read != NULL) + fs_file_close(file->super_read); + if (file->super != NULL) + fs_file_close(file->super); +} + +static const char *fs_compress_file_get_path(struct fs_file *_file) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + return fs_file_path(file->super); +} + +static void +fs_compress_set_async_callback(struct fs_file *_file, + fs_file_async_callback_t *callback, + void *context) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + fs_file_set_async_callback(file->super, callback, context); +} + +static int fs_compress_wait_async(struct fs *_fs) +{ + return fs_wait_async(_fs->parent); +} + +static void +fs_compress_set_metadata(struct fs_file *_file, const char *key, + const char *value) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + fs_set_metadata(file->super, key, value); +} + +static int +fs_compress_get_metadata(struct fs_file *_file, + const ARRAY_TYPE(fs_metadata) **metadata_r) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + return fs_get_metadata(file->super, metadata_r); +} + +static bool fs_compress_prefetch(struct fs_file *_file, uoff_t length) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + return fs_prefetch(file->super, length); +} + +static struct istream * +fs_compress_read_stream(struct fs_file *_file, size_t max_buffer_size) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + struct istream *input; + + if (file->input != NULL) { + i_stream_ref(file->input); + i_stream_seek(file->input, 0); + return file->input; + } + + input = fs_read_stream(file->super_read, max_buffer_size); + file->input = file->fs->handler->create_istream(input, FALSE); + i_stream_unref(&input); + i_stream_ref(file->input); + return file->input; +} + +static void fs_compress_write_stream(struct fs_file *_file) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + + i_assert(_file->output == NULL); + + file->temp_output = + iostream_temp_create_named(_file->fs->temp_path_prefix, + IOSTREAM_TEMP_FLAG_TRY_FD_DUP, + fs_file_path(_file)); + _file->output = file->fs->handler-> + create_ostream(file->temp_output, file->fs->compress_level); +} + +static int fs_compress_write_stream_finish(struct fs_file *_file, bool success) +{ + struct compress_fs_file *file = (struct compress_fs_file *)_file; + struct istream *input; + int ret; + + if (_file->output != NULL) { + if (_file->output->closed) + success = FALSE; + if (_file->output == file->super_output) + _file->output = NULL; + else + o_stream_unref(&_file->output); + } + if (!success) { + if (file->temp_output != NULL) + o_stream_destroy(&file->temp_output); + if (file->super_output != NULL) + fs_write_stream_abort(file->super, &file->super_output); + return -1; + } + + if (file->super_output != NULL) { + i_assert(file->temp_output == NULL); + return fs_write_stream_finish(file->super, &file->super_output); + } From dovecot at dovecot.org Fri May 8 11:49:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 08 May 2015 11:49:15 +0000 Subject: dovecot-2.2: dsync: Stop running if SIGINT/SIGTERM is received. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/60e48bb43f19 changeset: 18546:60e48bb43f19 user: Timo Sirainen date: Fri May 08 14:47:26 2015 +0300 description: dsync: Stop running if SIGINT/SIGTERM is received. diffstat: src/doveadm/doveadm-dsync.c | 7 +++++-- src/doveadm/doveadm-mail.c | 5 +++++ src/doveadm/doveadm-mail.h | 2 ++ 3 files changed, 12 insertions(+), 2 deletions(-) diffs (51 lines): diff -r f2a8e1793718 -r 60e48bb43f19 src/doveadm/doveadm-dsync.c --- a/src/doveadm/doveadm-dsync.c Fri May 08 10:44:18 2015 +0300 +++ b/src/doveadm/doveadm-dsync.c Fri May 08 14:47:26 2015 +0300 @@ -383,7 +383,8 @@ changed1 = changed2 = TRUE; while (brain1_running || brain2_running) { if (dsync_brain_has_failed(brain) || - dsync_brain_has_failed(brain2)) + dsync_brain_has_failed(brain2) || + doveadm_is_killed()) break; i_assert(changed1 || changed2); @@ -391,7 +392,9 @@ brain2_running = dsync_brain_run(brain2, &changed2); } *changes_during_sync_r = dsync_brain_has_unexpected_changes(brain2); - return dsync_brain_deinit(&brain2, mail_error_r); + if (dsync_brain_deinit(&brain2, mail_error_r) < 0) + return -1; + return doveadm_is_killed() ? -1 : 0; } static void cmd_dsync_wait_remote(struct dsync_cmd_context *ctx, diff -r f2a8e1793718 -r 60e48bb43f19 src/doveadm/doveadm-mail.c --- a/src/doveadm/doveadm-mail.c Fri May 08 10:44:18 2015 +0300 +++ b/src/doveadm/doveadm-mail.c Fri May 08 14:47:26 2015 +0300 @@ -40,6 +40,11 @@ static int killed_signo = 0; +bool doveadm_is_killed(void) +{ + return killed_signo != 0; +} + void doveadm_mail_failed_error(struct doveadm_mail_cmd_context *ctx, enum mail_error error) { diff -r f2a8e1793718 -r 60e48bb43f19 src/doveadm/doveadm-mail.h --- a/src/doveadm/doveadm-mail.h Fri May 08 10:44:18 2015 +0300 +++ b/src/doveadm/doveadm-mail.h Fri May 08 14:47:26 2015 +0300 @@ -111,6 +111,8 @@ extern struct doveadm_mail_cmd_module_register doveadm_mail_cmd_module_register; extern char doveadm_mail_cmd_hide; +bool doveadm_is_killed(void); + bool doveadm_mail_try_run(const char *cmd_name, int argc, char *argv[]); void doveadm_mail_register_cmd(const struct doveadm_mail_cmd *cmd); const struct doveadm_mail_cmd *doveadm_mail_cmd_find(const char *cmd_name); From dovecot at dovecot.org Fri May 8 13:02:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 08 May 2015 13:02:08 +0000 Subject: dovecot-2.2: fts: Fixed handling NOT when using lib-fts Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6ec2073bbf5c changeset: 18547:6ec2073bbf5c user: Timo Sirainen date: Fri May 08 16:00:16 2015 +0300 description: fts: Fixed handling NOT when using lib-fts diffstat: src/plugins/fts/fts-search-args.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 60e48bb43f19 -r 6ec2073bbf5c src/plugins/fts/fts-search-args.c --- a/src/plugins/fts/fts-search-args.c Fri May 08 14:47:26 2015 +0300 +++ b/src/plugins/fts/fts-search-args.c Fri May 08 16:00:16 2015 +0300 @@ -43,6 +43,7 @@ array_foreach(tokens, tokenp) { arg = p_new(pool, struct mail_search_arg, 1); *arg = *orig_arg; + arg->match_not = FALSE; /* we copied this to the parent SUB */ arg->next = NULL; arg->value.str = p_strdup(pool, *tokenp); From dovecot at dovecot.org Sat May 9 08:32:02 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 08:32:02 +0000 Subject: dovecot-2.2: fts: Create filters earlier Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/abbd71252175 changeset: 18548:abbd71252175 user: Teemu Huovila date: Sat May 09 10:53:25 2015 +0300 description: fts: Create filters earlier This builds on the assumption that early initialization will be made less costly, in a coming change. diffstat: src/plugins/fts/fts-build-mail.c | 10 +++------- src/plugins/fts/fts-search-args.c | 8 -------- src/plugins/fts/fts-user.c | 38 +++++++++++++++++++------------------- src/plugins/fts/fts-user.h | 8 +++----- 4 files changed, 25 insertions(+), 39 deletions(-) diffs (152 lines): diff -r 6ec2073bbf5c -r abbd71252175 src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Fri May 08 16:00:16 2015 +0300 +++ b/src/plugins/fts/fts-build-mail.c Sat May 09 10:53:25 2015 +0300 @@ -299,7 +299,6 @@ { struct mail_user *user = ctx->update_ctx->backend->ns->user; const struct fts_language *lang; - const char *error; int ret; if (ctx->cur_user_lang != NULL) { @@ -310,12 +309,9 @@ /* wait for more data */ return 0; } else { - if (fts_user_language_get(user, lang, &ctx->cur_user_lang, - &error) < 0) { - i_error("fts: Can't index input because of invalid language '%s' config: %s", - lang->name, error); - return -1; - } + ctx->cur_user_lang = fts_user_language_find(user, lang); + i_assert(ctx->cur_user_lang != NULL); + if (ctx->pending_input->used > 0) { if (fts_build_add_tokens_with_filter(ctx, ctx->pending_input->data, diff -r 6ec2073bbf5c -r abbd71252175 src/plugins/fts/fts-search-args.c --- a/src/plugins/fts/fts-search-args.c Fri May 08 16:00:16 2015 +0300 +++ b/src/plugins/fts/fts-search-args.c Sat May 09 10:53:25 2015 +0300 @@ -151,15 +151,7 @@ int fts_search_args_expand(struct fts_backend *backend, struct mail_search_args *args) { - const char *error; - /* we need to know all the possible languages for building the - search query. each search word queried by passing it through each - language's filters. */ - if (fts_user_languages_fill_all(backend->ns->user, &error) < 0) { - i_error("fts_dovecot: Failed to initialize languages: %s", error); - return -1; - } fts_search_args_expand_tree(backend, args->pool, &args->args); /* we'll need to re-simplify the args if we changed anything */ diff -r 6ec2073bbf5c -r abbd71252175 src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Fri May 08 16:00:16 2015 +0300 +++ b/src/plugins/fts/fts-user.c Sat May 09 10:53:25 2015 +0300 @@ -114,12 +114,13 @@ return 0; } -static struct fts_user_language * -fts_user_language_find(struct fts_user *fuser, +struct fts_user_language * +fts_user_language_find(struct mail_user *user, const struct fts_language *lang) { struct fts_user_language *const *user_langp; - + struct fts_user *fuser = FTS_USER_CONTEXT(user); + array_foreach(&fuser->languages, user_langp) { if (strcmp((*user_langp)->lang->name, lang->name) == 0) return *user_langp; @@ -127,19 +128,13 @@ return NULL; } -int fts_user_language_get(struct mail_user *user, - const struct fts_language *lang, - struct fts_user_language **user_lang_r, - const char **error_r) +static int fts_user_language_create(struct mail_user *user, + struct fts_user *fuser, + const struct fts_language *lang, + const char **error_r) { - struct fts_user *fuser = FTS_USER_CONTEXT(user); + struct fts_filter *filter; struct fts_user_language *user_lang; - struct fts_filter *filter; - - *user_lang_r = fts_user_language_find(fuser, lang); - if (*user_lang_r != NULL) - return 0; - if (fts_user_create_filters(user, lang, &filter, error_r) < 0) return -1; @@ -148,18 +143,17 @@ user_lang->filter = filter; array_append(&fuser->languages, &user_lang, 1); - *user_lang_r = user_lang; return 0; } -int fts_user_languages_fill_all(struct mail_user *user, const char **error_r) +static int fts_user_languages_fill_all(struct mail_user *user, + struct fts_user *fuser, + const char **error_r) { - struct fts_user *fuser = FTS_USER_CONTEXT(user); const struct fts_language *const *langp; - struct fts_user_language *user_lang; array_foreach(fts_language_list_get_all(fuser->lang_list), langp) { - if (fts_user_language_get(user, *langp, &user_lang, error_r) < 0) + if (fts_user_language_create(user, fuser, *langp, error_r) < 0) return -1; } return 0; @@ -196,6 +190,7 @@ int fts_mail_user_init(struct mail_user *user, const char **error_r) { struct fts_user *fuser; + const char *error; fuser = p_new(user->pool, struct fts_user, 1); p_array_init(&fuser->languages, user->pool, 4); @@ -204,6 +199,11 @@ fts_user_free(fuser); return -1; } + if (fts_user_languages_fill_all(user, fuser, &error) < 0) { + i_error("fts_dovecot: Failed to initialize languages: %s", error); + fts_user_free(fuser); + return -1; + } MODULE_CONTEXT_SET(user, fts_user_module, fuser); return 0; } diff -r 6ec2073bbf5c -r abbd71252175 src/plugins/fts/fts-user.h --- a/src/plugins/fts/fts-user.h Fri May 08 16:00:16 2015 +0300 +++ b/src/plugins/fts/fts-user.h Sat May 09 10:53:25 2015 +0300 @@ -7,11 +7,9 @@ }; ARRAY_DEFINE_TYPE(fts_user_language, struct fts_user_language *); -int fts_user_language_get(struct mail_user *user, - const struct fts_language *lang, - struct fts_user_language **user_lang_r, - const char **error_r); -int fts_user_languages_fill_all(struct mail_user *user, const char **error_r); +struct fts_user_language * +fts_user_language_find(struct mail_user *user, + const struct fts_language *lang); struct fts_language_list *fts_user_get_language_list(struct mail_user *user); const ARRAY_TYPE(fts_user_language) * From dovecot at dovecot.org Sat May 9 08:32:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 08:32:07 +0000 Subject: dovecot-2.2: fts: Create tokenizers differently Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ae0458c63761 changeset: 18549:ae0458c63761 user: Teemu Huovila date: Sat May 09 11:02:22 2015 +0300 description: fts: Create tokenizers differently Create tokenizers earlier. Create separate tokenizers for search and indexing. Enable configuration of tokenizers. Add some helpers in fts-tokenizer.h api. Change tokenizer unit tests to match those changes. lib-fts: Refactor lib-fts settings a bit Turned address tokenizer settings into "boolean" values. Changed have_parent to "no_parent" and added "search" setting. Added documentation in fts-tokenizer.h. Change unit tests accordingly. diffstat: src/lib-fts/fts-tokenizer-address.c | 35 ++++++----- src/lib-fts/fts-tokenizer.c | 19 ++++++ src/lib-fts/fts-tokenizer.h | 32 ++++++++-- src/lib-fts/test-fts-tokenizer.c | 22 +++---- src/plugins/fts/fts-api-private.h | 2 - src/plugins/fts/fts-build-mail.c | 6 +- src/plugins/fts/fts-plugin.c | 3 + src/plugins/fts/fts-search-args.c | 7 +- src/plugins/fts/fts-storage.c | 14 ---- src/plugins/fts/fts-user.c | 106 ++++++++++++++++++++++++++++++++++- src/plugins/fts/fts-user.h | 3 +- 11 files changed, 188 insertions(+), 61 deletions(-) diffs (truncated from 599 to 300 lines): diff -r abbd71252175 -r ae0458c63761 src/lib-fts/fts-tokenizer-address.c --- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 10:53:25 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 11:02:22 2015 +0300 @@ -5,8 +5,8 @@ #include "buffer.h" #include "fts-tokenizer-private.h" -/* Return not only our tokens, but also data for parent to process.*/ -#define FTS_DEFAULT_HAVE_PARENT 1 +#define FTS_DEFAULT_NO_PARENT FALSE +#define FTS_DEFAULT_SEARCH FALSE enum email_address_parser_state { EMAIL_ADDRESS_PARSER_STATE_NONE = 0, @@ -21,8 +21,8 @@ string_t *last_word; string_t *parent_data; /* Copy of input data between tokens. TODO: could be buffer_t maybe */ - unsigned int have_parent; /* Setting for stand-alone usage. - Might be superfluous. */ + bool no_parent; + bool search; }; /* @@ -85,18 +85,17 @@ const char **error_r) { struct email_address_fts_tokenizer *tok; - unsigned int have_parent = FTS_DEFAULT_HAVE_PARENT; + bool no_parent = FTS_DEFAULT_NO_PARENT; + bool search = FTS_DEFAULT_SEARCH; unsigned int i; for (i = 0; settings[i] != NULL; i += 2) { - const char *key = settings[i], *value = settings[i+1]; + const char *key = settings[i]; - if (strcmp(key, "have_parent") == 0) { - if (str_to_uint(value, &have_parent) < 0 ) { - *error_r = t_strdup_printf( - "Invalid parent setting: %s", value); - return -1; - } + if (strcmp(key, "no_parent") == 0) { + no_parent = TRUE; + }else if (strcmp(key, "search") == 0) { + search = TRUE; } else { *error_r = t_strdup_printf("Unknown setting: %s", key); return -1; @@ -107,7 +106,8 @@ tok->tokenizer = *fts_tokenizer_email_address; tok->last_word = str_new(default_pool, 128); tok->parent_data = str_new(default_pool, 128); - tok->have_parent = have_parent; + tok->no_parent = no_parent; + tok->search = search; *tokenizer_r = &tok->tokenizer; return 0; } @@ -134,6 +134,9 @@ fts_tokenizer_address_parent_data(struct email_address_fts_tokenizer *tok) { const char *ret; + /* TODO: search option removes address from data here. */ + if (tok->search && tok->state >= EMAIL_ADDRESS_PARSER_STATE_DOMAIN) + i_debug("Would remove current token"); ret = t_strdup(str_c(tok->parent_data)); str_truncate(tok->parent_data, 0); @@ -250,7 +253,7 @@ fts_tokenizer_address_update_parent(struct email_address_fts_tokenizer *tok, const unsigned char *data, size_t size) { - if (tok->have_parent > 0) + if (!tok->no_parent) str_append_n(tok->parent_data, data, size); } static const char * @@ -273,7 +276,7 @@ /* end of data, output lingering tokens. first the parents data, then possibly our token, if complete enough */ if (size == 0) { - if (tok->have_parent > 0 && str_len(tok->parent_data) > 0) + if (!tok->no_parent && str_len(tok->parent_data) > 0) return fts_tokenizer_address_parent_data(tok); if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN @@ -328,7 +331,7 @@ *skip_r = pos + local_skip; fts_tokenizer_address_update_parent(tok, data+pos, local_skip); - if (tok->have_parent > 0) + if (!tok->no_parent) return fts_tokenizer_address_parent_data(tok); else { return fts_tokenizer_address_current_token(tok); diff -r abbd71252175 -r ae0458c63761 src/lib-fts/fts-tokenizer.c --- a/src/lib-fts/fts-tokenizer.c Sat May 09 10:53:25 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.c Sat May 09 11:02:22 2015 +0300 @@ -10,6 +10,20 @@ ARRAY(struct fts_tokenizer) fts_tokenizer_classes; +void fts_tokenizers_init(void) +{ + if (!array_is_created(&fts_tokenizer_classes)) { + fts_tokenizer_register(fts_tokenizer_generic); + fts_tokenizer_register(fts_tokenizer_email_address); + } +} + +void fts_tokenizers_deinit(void) +{ + if (array_is_created(&fts_tokenizer_classes)) + array_free(&fts_tokenizer_classes); +} + /* private */ void fts_tokenizer_register(const struct fts_tokenizer *tok_class) { @@ -47,6 +61,11 @@ return NULL; } +const char *fts_tokenizer_name(const struct fts_tokenizer *tok) +{ + return tok->name; +} + int fts_tokenizer_create(const struct fts_tokenizer *tok_class, struct fts_tokenizer *parent, const char *const *settings, diff -r abbd71252175 -r ae0458c63761 src/lib-fts/fts-tokenizer.h --- a/src/lib-fts/fts-tokenizer.h Sat May 09 10:53:25 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.h Sat May 09 11:02:22 2015 +0300 @@ -3,7 +3,9 @@ /* Settings are given in the form of a const char * const *settings = - {"key, "value", "key2", "value2", NULL} array of string pairs. + {"key, "value", "key2", "value2", NULL} array of string pairs. Some + keys, like "no_parent" and "search" are a sort of boolean and the + value does not matter, just mentioning the key enables the functionality. The array has to be NULL terminated. */ /* Email address header tokenizer that returns "user at domain.org" input as @@ -13,15 +15,21 @@ allows doing an explicit "user at domain" search, which returns only mails matching that exact address (instead of e.g. a mail with both user at domain2 and user2 at domain words). */ -/* Settings: "have_parent", Return not only our tokens, but also data - for parent to process. Defaults to 1. Should normally not need to - be changed. */ +/* Settings: + "no_parent", Return only our tokens, no data for parent to process. + Defaults to disabled. Should normally not be needed. + + "search" Remove addresses from parent data stream, so they are not processed + further. Defaults to disabled. Enable by defining the keyword (and any + value). */ extern const struct fts_tokenizer *fts_tokenizer_email_address; #define FTS_TOKENIZER_EMAIL_ADDRESS_NAME "email-address" /* Generic email content tokenizer. Cuts text into tokens. */ -/* Settings: "maxlen" Maximum length of token, before an arbitary cut - off is made. Defaults to FTS_DEFAULT_TOKEN_MAX_LENGTH. +/* Settings: + "maxlen" Maximum length of token, before an arbitary cut off is made. + Defaults to FTS_DEFAULT_TOKEN_MAX_LENGTH. + "algorithm", accepted values are "simple" or "tr29". Defines the method for looking for word boundaries. Simple is faster and will work for many texts, especially those using latin alphabets, but @@ -35,9 +43,18 @@ extern const struct fts_tokenizer *fts_tokenizer_generic; #define FTS_TOKENIZER_GENERIC_NAME "generic" +/* + Tokenizing workflow, find --> create --> filter --> destroy. + Do init before first use and deinit after all done. + */ + +/* Register all built-in tokenizers. */ +void fts_tokenizers_init(void); +void fts_tokenizers_deinit(void); + const struct fts_tokenizer *fts_tokenizer_find(const char *name); -/* Create a new tokenizer. The settings is an array of key,value pairs. */ +/* Create a new tokenizer. The settings are described above. */ int fts_tokenizer_create(const struct fts_tokenizer *tok_class, struct fts_tokenizer *parent, const char *const *settings, @@ -57,4 +74,5 @@ fts_tokenizer_next(struct fts_tokenizer *tok, const unsigned char *data, size_t size); +const char *fts_tokenizer_name(const struct fts_tokenizer *tok); #endif diff -r abbd71252175 -r ae0458c63761 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Sat May 09 10:53:25 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Sat May 09 11:02:22 2015 +0300 @@ -25,7 +25,7 @@ const char *token, *error; test_begin("fts tokenizer generic simple"); - fts_tokenizer_register(fts_tokenizer_generic); + fts_tokenizers_init(); tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME); test_assert(fts_tokenizer_create(tok_class, NULL, NULL, &tok, &error) == 0); while ((token = fts_tokenizer_next(tok, input, sizeof(input)-1)) != NULL) { @@ -38,7 +38,7 @@ } test_assert(*eopp == NULL); fts_tokenizer_unref(&tok); - fts_tokenizer_unregister(fts_tokenizer_generic); + fts_tokenizers_deinit(); test_end(); } @@ -267,7 +267,7 @@ "abc at example.com", "bar at example.org", "foo at domain", "foo at domain", "bar at example.org", NULL }; - const char *const settings[] = {"have_parent", "0", NULL}; + const char *const settings[] = {"no_parent", "foo", NULL}; struct fts_tokenizer *tok; const char * const *eopp = expected_output; const char *token, *error; @@ -305,7 +305,7 @@ "abc at example.com", "bar at example.org", "foo at domain", NULL }; - const char *const settings[] = {"have_parent", "0", NULL}; + const char *const settings[] = {"no_parent", "0", NULL}; struct fts_tokenizer *tok; const char * const *eopp = expected_output; const char *token, *error; @@ -346,7 +346,7 @@ struct fts_tokenizer *tok; const char * const *eopp = expected_output; const char *token, *error; - const char *const settings[] = {"have_parent", "0", NULL}; + const char *const settings[] = {"no_parent", "abc", NULL}; unsigned int i, step, step_max = 10; test_begin("fts tokenizer email address, input random length"); @@ -390,8 +390,7 @@ unsigned int i; test_begin("fts tokenizer email address + parent, input one character at a time"); - fts_tokenizer_register(fts_tokenizer_generic); - fts_tokenizer_register(fts_tokenizer_email_address); + fts_tokenizers_init(); test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &gen_tok, &error) == 0); test_assert(fts_tokenizer_create(fts_tokenizer_email_address, gen_tok, NULL, &tok, &error) == 0); @@ -411,8 +410,7 @@ test_assert(*eopp == NULL); fts_tokenizer_unref(&tok); fts_tokenizer_unref(&gen_tok); - fts_tokenizer_unregister(fts_tokenizer_generic); - fts_tokenizer_unregister(fts_tokenizer_email_address); + fts_tokenizers_deinit(); test_end(); } @@ -437,8 +435,7 @@ unsigned int i; test_begin("fts tokenizer email address + parent, input one line at a time"); - fts_tokenizer_register(fts_tokenizer_generic); - fts_tokenizer_register(fts_tokenizer_email_address); + fts_tokenizers_init(); test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &gen_tok, &error) == 0); test_assert(fts_tokenizer_create(fts_tokenizer_email_address, gen_tok, NULL, &tok, &error) == 0); @@ -457,8 +454,7 @@ test_assert(*eopp == NULL); fts_tokenizer_unref(&tok); fts_tokenizer_unref(&gen_tok); - fts_tokenizer_unregister(fts_tokenizer_generic); - fts_tokenizer_unregister(fts_tokenizer_email_address); + fts_tokenizers_deinit(); test_end(); } diff -r abbd71252175 -r ae0458c63761 src/plugins/fts/fts-api-private.h --- a/src/plugins/fts/fts-api-private.h Sat May 09 10:53:25 2015 +0300 +++ b/src/plugins/fts/fts-api-private.h Sat May 09 11:02:22 2015 +0300 @@ -76,8 +76,6 @@ struct fts_backend_vfuncs v; struct mail_namespace *ns; - struct fts_tokenizer *tokenizer; - unsigned int updating:1; }; From dovecot at dovecot.org Sat May 9 08:32:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 08:32:09 +0000 Subject: dovecot-2.2: fts: Replace '-' with '_' in tokenizer-specific fts... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cebe8be92034 changeset: 18550:cebe8be92034 user: Teemu Huovila date: Sat May 09 11:03:21 2015 +0300 description: fts: Replace '-' with '_' in tokenizer-specific fts_tokenizers_* settings diffstat: src/plugins/fts/fts-user.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (22 lines): diff -r ae0458c63761 -r cebe8be92034 src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Sat May 09 11:02:22 2015 +0300 +++ b/src/plugins/fts/fts-user.c Sat May 09 11:03:21 2015 +0300 @@ -123,7 +123,7 @@ { const struct fts_tokenizer *tokenizer_class; struct fts_tokenizer *tokenizer = NULL, *parent = NULL; - const char *tokenizers_key, *const *tokenizers; + const char *tokenizers_key, *const *tokenizers, *tokenizer_set_name; const char *str, *error, *set_key, *const *settings; unsigned int i; int ret = 0; @@ -144,7 +144,8 @@ break; } - set_key = t_strdup_printf("fts_tokenizers_%s", tokenizers[i]); + tokenizer_set_name = t_str_replace(tokenizers[i], '-', '_'); + set_key = t_strdup_printf("fts_tokenizers_%s", tokenizer_set_name); str = mail_user_plugin_getenv(user, set_key); /* If the email-address tokenizer is included in the search From dovecot at dovecot.org Sat May 9 08:32:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 08:32:19 +0000 Subject: dovecot-2.2: fts: Change tokenizer API to be able to return errors Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7fe766887394 changeset: 18551:7fe766887394 user: Teemu Huovila date: Sat May 09 11:05:04 2015 +0300 description: fts: Change tokenizer API to be able to return errors Modify fts_tokenizer_next() to return integer status codes. It returns 1 if a token was returned in *token_r, 0 if more input is needed and -1 on error. diffstat: src/lib-fts/fts-tokenizer-address.c | 36 +++++++------ src/lib-fts/fts-tokenizer-generic.c | 48 +++++++++--------- src/lib-fts/fts-tokenizer-private.h | 5 +- src/lib-fts/fts-tokenizer.c | 49 +++++++++--------- src/lib-fts/fts-tokenizer.h | 19 +++++-- src/lib-fts/test-fts-tokenizer.c | 94 ++++++++++++++++++++---------------- src/plugins/fts/fts-build-mail.c | 5 +- src/plugins/fts/fts-search-args.c | 12 ++-- 8 files changed, 144 insertions(+), 124 deletions(-) diffs (truncated from 682 to 300 lines): diff -r cebe8be92034 -r 7fe766887394 src/lib-fts/fts-tokenizer-address.c --- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 11:03:21 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 11:05:04 2015 +0300 @@ -122,25 +122,27 @@ i_free(tok); } -static const char * -fts_tokenizer_address_current_token(struct email_address_fts_tokenizer *tok) +static int +fts_tokenizer_address_current_token(struct email_address_fts_tokenizer *tok, + const char **token_r) { tok->tokenizer.skip_parents = TRUE; tok->state = EMAIL_ADDRESS_PARSER_STATE_NONE; - return t_strdup(str_c(tok->last_word)); + *token_r = t_strdup(str_c(tok->last_word)); + return 1; } -static const char * -fts_tokenizer_address_parent_data(struct email_address_fts_tokenizer *tok) +static int +fts_tokenizer_address_parent_data(struct email_address_fts_tokenizer *tok, + const char **token_r) { - const char *ret; /* TODO: search option removes address from data here. */ if (tok->search && tok->state >= EMAIL_ADDRESS_PARSER_STATE_DOMAIN) i_debug("Would remove current token"); - ret = t_strdup(str_c(tok->parent_data)); + *token_r = t_strdup(str_c(tok->parent_data)); str_truncate(tok->parent_data, 0); - return ret; + return 1; } /* Used to rewind past characters that can not be the start of a new localpart. @@ -256,10 +258,10 @@ if (!tok->no_parent) str_append_n(tok->parent_data, data, size); } -static const char * +static int fts_tokenizer_email_address_next(struct fts_tokenizer *_tok, - const unsigned char *data, size_t size, - size_t *skip_r) + const unsigned char *data, size_t size, + size_t *skip_r, const char **token_r) { struct email_address_fts_tokenizer *tok = (struct email_address_fts_tokenizer *)_tok; @@ -270,18 +272,18 @@ if (tok->state == EMAIL_ADDRESS_PARSER_STATE_COMPLETE) { *skip_r = pos; - return fts_tokenizer_address_current_token(tok); + return fts_tokenizer_address_current_token(tok, token_r); } /* end of data, output lingering tokens. first the parents data, then possibly our token, if complete enough */ if (size == 0) { if (!tok->no_parent && str_len(tok->parent_data) > 0) - return fts_tokenizer_address_parent_data(tok); + return fts_tokenizer_address_parent_data(tok, token_r); if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN && chars_after_at(tok) > 0) - return fts_tokenizer_address_current_token(tok); + return fts_tokenizer_address_current_token(tok, token_r); } /* 1) regular input data OR @@ -332,9 +334,9 @@ fts_tokenizer_address_update_parent(tok, data+pos, local_skip); if (!tok->no_parent) - return fts_tokenizer_address_parent_data(tok); + return fts_tokenizer_address_parent_data(tok, token_r); else { - return fts_tokenizer_address_current_token(tok); + return fts_tokenizer_address_current_token(tok, token_r); } default: i_unreached(); @@ -342,7 +344,7 @@ } *skip_r = pos; - return NULL; + return 0; } static const struct fts_tokenizer_vfuncs email_address_tokenizer_vfuncs = { diff -r cebe8be92034 -r 7fe766887394 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Sat May 09 11:03:21 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Sat May 09 11:05:04 2015 +0300 @@ -82,14 +82,13 @@ i_free(tok); } -static const char * -fts_tokenizer_generic_simple_current_token(struct generic_fts_tokenizer *tok) +static int +fts_tokenizer_generic_simple_current_token(struct generic_fts_tokenizer *tok, + const char **token_r) { - const char *ret; - - ret = t_strndup(tok->token->data, tok->token->used); + *token_r = t_strndup(tok->token->data, tok->token->used); buffer_set_used_size(tok->token, 0); - return ret; + return 1; } /* TODO: This is duplicated from unichar.c */ @@ -135,10 +134,10 @@ return is_word_break(c); } -static const char * +static int fts_tokenizer_generic_next_simple(struct fts_tokenizer *_tok, - const unsigned char *data, size_t size, - size_t *skip_r) + const unsigned char *data, size_t size, + size_t *skip_r, const char **token_r) { struct generic_fts_tokenizer *tok = (struct generic_fts_tokenizer *)_tok; @@ -157,7 +156,7 @@ } /* word boundary found - return a new token */ *skip_r = i + 1; - return fts_tokenizer_generic_simple_current_token(tok); + return fts_tokenizer_generic_simple_current_token(tok, token_r); } } /* word boundary not found yet */ @@ -168,9 +167,9 @@ if (size == 0 && tok->token->used > 0) { /* return the last token */ - return fts_tokenizer_generic_simple_current_token(tok); + return fts_tokenizer_generic_simple_current_token(tok, token_r); } - return NULL; + return 0; } /* TODO: Arrange array searches roughly in order of likelyhood of a match. @@ -464,20 +463,20 @@ return FALSE; } -static const char * -fts_tokenizer_generic_tr29_current_token(struct generic_fts_tokenizer *tok) +static int +fts_tokenizer_generic_tr29_current_token(struct generic_fts_tokenizer *tok, + const char **token_r) { - const char *ret; size_t end_skip = 0; if (is_one_past_end(tok)) end_skip = tok->last_size; - ret = t_strndup(tok->token->data, tok->token->used - end_skip); + *token_r = t_strndup(tok->token->data, tok->token->used - end_skip); buffer_set_used_size(tok->token, 0); tok->prev_prev_letter = LETTER_TYPE_NONE; tok->prev_letter = LETTER_TYPE_NONE; - return ret; + return 1; } /* Find word boundaries in input text. Based on Unicode standard annex @@ -516,10 +515,10 @@ return FALSE; } -static const char * +static int fts_tokenizer_generic_next_tr29(struct fts_tokenizer *_tok, const unsigned char *data, size_t size, - size_t *skip_r) + size_t *skip_r, const char **token_r) { struct generic_fts_tokenizer *tok = (struct generic_fts_tokenizer *)_tok; @@ -547,7 +546,7 @@ buffer_append(tok->token, data + start_skip, len - start_skip); *skip_r = i + 1; - return fts_tokenizer_generic_tr29_current_token(tok); + return fts_tokenizer_generic_tr29_current_token(tok, token_r); } } len = I_MIN(i, tok->max_length); @@ -558,16 +557,17 @@ if (size == 0 && tok->token->used > 0) { /* return the last token */ *skip_r = 0; - return fts_tokenizer_generic_tr29_current_token(tok); + return fts_tokenizer_generic_tr29_current_token(tok, token_r); } - return NULL; + return 0; } -static const char * +static int fts_tokenizer_generic_next(struct fts_tokenizer *_tok ATTR_UNUSED, const unsigned char *data ATTR_UNUSED, size_t size ATTR_UNUSED, - size_t *skip_r ATTR_UNUSED) + size_t *skip_r ATTR_UNUSED, + const char **token_r ATTR_UNUSED) { i_unreached(); } diff -r cebe8be92034 -r 7fe766887394 src/lib-fts/fts-tokenizer-private.h --- a/src/lib-fts/fts-tokenizer-private.h Sat May 09 11:03:21 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-private.h Sat May 09 11:05:04 2015 +0300 @@ -10,9 +10,8 @@ struct fts_tokenizer **tokenizer_r, const char **error_r); void (*destroy)(struct fts_tokenizer *tok); - const char *(*next)(struct fts_tokenizer *tok, - const unsigned char *data, size_t size, - size_t *skip_r); + int (*next)(struct fts_tokenizer *tok, const unsigned char *data, + size_t size, size_t *skip_r, const char **token_r); }; enum fts_tokenizer_parent_state { diff -r cebe8be92034 -r 7fe766887394 src/lib-fts/fts-tokenizer.c --- a/src/lib-fts/fts-tokenizer.c Sat May 09 11:03:21 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.c Sat May 09 11:05:04 2015 +0300 @@ -120,11 +120,12 @@ tok->v->destroy(tok); } -static const char * +static int fts_tokenizer_next_self(struct fts_tokenizer *tok, - const unsigned char *data, size_t size) + const unsigned char *data, size_t size, + const char **token_r) { - const char *token; + int ret = 0; size_t skip = 0; i_assert(tok->prev_reply_finished || @@ -132,60 +133,60 @@ if (tok->prev_reply_finished) { /* whole new data */ - token = tok->v->next(tok, data, size, &skip); + ret = tok->v->next(tok, data, size, &skip, token_r); } else { /* continuing previous data */ i_assert(tok->prev_skip <= size); - token = tok->v->next(tok, data + tok->prev_skip, - size - tok->prev_skip, &skip); + ret = tok->v->next(tok, data + tok->prev_skip, + size - tok->prev_skip, &skip, token_r); } - if (token != NULL) { + if (ret > 0) { i_assert(skip <= size - tok->prev_skip); tok->prev_data = data; tok->prev_size = size; tok->prev_skip = tok->prev_skip + skip; tok->prev_reply_finished = FALSE; - } else { + } else if (ret == 0) { /* we need a new data block */ tok->prev_data = NULL; tok->prev_size = 0; tok->prev_skip = 0; tok->prev_reply_finished = TRUE; } - return token; + return ret; } -const char * +int fts_tokenizer_next(struct fts_tokenizer *tok, - const unsigned char *data, size_t size) + const unsigned char *data, size_t size, const char **token_r) { - const char *token; + int ret; From dovecot at dovecot.org Sat May 9 08:32:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 08:32:19 +0000 Subject: dovecot-2.2: fts: Change filter API to be able to return errors Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/95a827d97e5b changeset: 18552:95a827d97e5b user: Teemu Huovila date: Sat May 09 11:06:45 2015 +0300 description: fts: Change filter API to be able to return errors Modify fts_filter_filter() to return integer status codes. It returns 1 if a token was returned, 0 if it was filtered away and -1 on error. diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 42 +++++++------ src/lib-fts/fts-filter-normalizer-simple.c | 15 ++- src/lib-fts/fts-filter-private.h | 2 +- src/lib-fts/fts-filter-stemmer-snowball.c | 16 ++-- src/lib-fts/fts-filter-stopwords.c | 14 ++- src/lib-fts/fts-filter.c | 14 ++-- src/lib-fts/fts-filter.h | 9 +- src/lib-fts/test-fts-filter.c | 95 +++++++++++++++++------------ src/plugins/fts/fts-build-mail.c | 6 +- src/plugins/fts/fts-search-args.c | 8 +- 10 files changed, 126 insertions(+), 95 deletions(-) diffs (truncated from 584 to 300 lines): diff -r 7fe766887394 -r 95a827d97e5b src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 11:05:04 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 11:06:45 2015 +0300 @@ -98,8 +98,9 @@ return 0; } -static int make_utf8(const UChar *src, char **dst, const char **error_r) +static int make_utf8(const UChar *src, const char **_dst, const char **error_r) { + char *dst; char *retp = NULL; int32_t dsize = 0; int32_t dsize_actual = 0; @@ -120,9 +121,9 @@ i_assert(NULL == retp); dsize++; /* room for '\0' byte */ - *dst = t_malloc(dsize); + dst = t_malloc(dsize); err = U_ZERO_ERROR; - retp = u_strToUTF8WithSub(*dst, dsize, &dsize_actual, src, usrc_len, + retp = u_strToUTF8WithSub(dst, dsize, &dsize_actual, src, usrc_len, UNICODE_REPLACEMENT_CHAR, &sub_num, &err); if (U_FAILURE(err)) i_panic("Lib ICU u_strToUTF8WithSub() failed: %s", @@ -137,8 +138,9 @@ " Substitutions (%d) were made.", sub_num); return -1; } - i_assert(retp == *dst); + i_assert(retp == dst); + *_dst = dst; return 0; } @@ -212,27 +214,24 @@ return 0; } -/* Returns 0 on success and -1 on error. */ -/* TODO: delay errors until _deinit() and return some other values? */ -static const char * -fts_filter_normalizer_icu_filter(struct fts_filter *filter, const char *token) +static int +fts_filter_normalizer_icu_filter(struct fts_filter *filter, const char **token) { UErrorCode err = U_ZERO_ERROR; UChar *utext = NULL; int32_t utext_cap = 0; int32_t utext_len = -1; int32_t utext_limit; - char *normalized = NULL; struct fts_filter_normalizer *np = (struct fts_filter_normalizer *)filter; /* TODO: fix error handling */ if (np->error != NULL) - return NULL; + goto err_exit; - if (make_uchar(token, &utext, &utext_cap) < 0) { + if (make_uchar(*token, &utext, &utext_cap) < 0) { fts_filter_normalizer_icu_error(&np->error, "Conversion to UChar failed"); - return NULL; + goto err_exit; } /* TODO: Some problems here. How much longer can the result @@ -249,8 +248,9 @@ size utrans_transUChars indicated */ utext_len++; /* room for '\0' bytes(2) */ utext_cap = utext_len; - if (make_uchar(token, &utext, &utext_cap) < 0) - return NULL; + if (make_uchar(*token, &utext, &utext_cap) < 0) { + goto err_exit; + } i_assert(utext_cap == utext_len); utext_limit = u_strlen(utext); utext_len = -1; @@ -262,13 +262,17 @@ if (U_FAILURE(err)) { icu_error(&np->error, err, "utrans_transUChars()"); - return NULL; + goto err_exit; } - if (make_utf8(utext, &normalized, &np->error) < 0) - return NULL; + if (make_utf8(utext, token, &np->error) < 0) { + goto err_exit; + } - return normalized; + return 1; + err_exit: + *token = NULL; + return -1; } #else @@ -289,7 +293,7 @@ return -1; } -static const char * +static int fts_filter_normalizer_icu_filter(struct fts_filter *filter ATTR_UNUSED, const char *token ATTR_UNUSED) { diff -r 7fe766887394 -r 95a827d97e5b src/lib-fts/fts-filter-normalizer-simple.c --- a/src/lib-fts/fts-filter-normalizer-simple.c Sat May 09 11:05:04 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-simple.c Sat May 09 11:06:45 2015 +0300 @@ -48,18 +48,21 @@ return 0; } -static const char * +static int fts_filter_normalizer_simple_filter(struct fts_filter *_filter, - const char *token) + const char **token) { struct fts_filter_normalizer_simple *filter = (struct fts_filter_normalizer_simple *)_filter; str_truncate(filter->str, 0); - if (uni_utf8_to_decomposed_titlecase(token, strlen(token), - filter->str) < 0) - return NULL; - return str_c(filter->str); + if (uni_utf8_to_decomposed_titlecase(*token, strlen(*token), + filter->str) < 0) { + *token = NULL; + return -1; + } + *token = str_c(filter->str); + return 1; } static const struct fts_filter_vfuncs normalizer_filter_vfuncs = { diff -r 7fe766887394 -r 95a827d97e5b src/lib-fts/fts-filter-private.h --- a/src/lib-fts/fts-filter-private.h Sat May 09 11:05:04 2015 +0300 +++ b/src/lib-fts/fts-filter-private.h Sat May 09 11:06:45 2015 +0300 @@ -17,7 +17,7 @@ const char *const *settings, struct fts_filter **filter_r, const char **error_r); - const char * (*filter)(struct fts_filter *filter, const char *token); + int (*filter)(struct fts_filter *filter, const char **token); void (*destroy)(struct fts_filter *filter); }; diff -r 7fe766887394 -r 95a827d97e5b src/lib-fts/fts-filter-stemmer-snowball.c --- a/src/lib-fts/fts-filter-stemmer-snowball.c Sat May 09 11:05:04 2015 +0300 +++ b/src/lib-fts/fts-filter-stemmer-snowball.c Sat May 09 11:06:45 2015 +0300 @@ -66,18 +66,20 @@ return 0; } -static const char * +static int fts_filter_stemmer_snowball_filter(struct fts_filter *filter, - const char *token) + const char **token) { const sb_symbol *base; int len; struct fts_filter_stemmer_snowball *sp = (struct fts_filter_stemmer_snowball *) filter; - base = sb_stemmer_stem(sp->stemmer, (const unsigned char *)token, strlen(token)); + base = sb_stemmer_stem(sp->stemmer, (const unsigned char *)*token, strlen(*token)); len = sb_stemmer_length(sp->stemmer); - return t_strdup_until(base, base + len); + *token = t_strdup_until(base, base + len); + + return *token != NULL? 1: -1; } #else @@ -101,11 +103,11 @@ { } -static const char * +static int fts_filter_stemmer_snowball_filter(struct fts_filter *filter ATTR_UNUSED, - const char *token ATTR_UNUSED) + const char **token ATTR_UNUSED) { - return NULL; + return -1; } #endif diff -r 7fe766887394 -r 95a827d97e5b src/lib-fts/fts-filter-stopwords.c --- a/src/lib-fts/fts-filter-stopwords.c Sat May 09 11:05:04 2015 +0300 +++ b/src/lib-fts/fts-filter-stopwords.c Sat May 09 11:06:45 2015 +0300 @@ -125,18 +125,20 @@ return ret; } -static const char * -fts_filter_stopwords_filter(struct fts_filter *filter, const char *token) +static int +fts_filter_stopwords_filter(struct fts_filter *filter, const char **token) { const char *stopword; struct fts_filter_stopwords *sp = (struct fts_filter_stopwords *) filter; - stopword = hash_table_lookup(sp->stopwords, token); - if (stopword != NULL) - return NULL; + stopword = hash_table_lookup(sp->stopwords, *token); + if (stopword != NULL) { + *token = NULL; + return 0; + } else - return token; + return 1; } const struct fts_filter_vfuncs stopwords_filter_vfuncs = { diff -r 7fe766887394 -r 95a827d97e5b src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Sat May 09 11:05:04 2015 +0300 +++ b/src/lib-fts/fts-filter.c Sat May 09 11:06:45 2015 +0300 @@ -92,19 +92,19 @@ /* TODO: Avoid multiple allocations by using a buffer in v->filter? Do this non-recursively? */ -const char * -fts_filter_filter(struct fts_filter *filter, const char *token) +int +fts_filter_filter(struct fts_filter *filter, const char **token) { - const char *filtered = NULL; + int ret; if (filter->parent == NULL) return filter->v->filter(filter, token); - filtered = fts_filter_filter(filter->parent, token); + ret = fts_filter_filter(filter->parent, token); - if(filtered != NULL) - return filter->v->filter(filter, filtered); + if(ret > 0) + return filter->v->filter(filter, token); - return NULL; + return ret; } diff -r 7fe766887394 -r 95a827d97e5b src/lib-fts/fts-filter.h --- a/src/lib-fts/fts-filter.h Sat May 09 11:05:04 2015 +0300 +++ b/src/lib-fts/fts-filter.h Sat May 09 11:06:45 2015 +0300 @@ -58,8 +58,11 @@ void fts_filter_ref(struct fts_filter *filter); void fts_filter_unref(struct fts_filter **filter); -/* Returns the filtered token or NULL, if it was completely removed */ -const char * -fts_filter_filter(struct fts_filter *filter, const char *token); +/* Returns 1 if token is returned in *token, 0 if token was filtered + out and -1 on error. + Input is also given via *token. +*/ +int +fts_filter_filter(struct fts_filter *filter, const char **token); #endif diff -r 7fe766887394 -r 95a827d97e5b src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Sat May 09 11:05:04 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Sat May 09 11:06:45 2015 +0300 @@ -24,7 +24,7 @@ "drive", NULL, NULL, NULL, "reason", NULL, NULL, NULL, "sing"}; const char **ip, **op; - const char *filtered; + const char *token; test_begin("fts filter stopwords, English"); filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); @@ -34,12 +34,14 @@ ip = input; op = output; while (*ip != NULL) { - filtered = fts_filter_filter(filter, *ip); - if (filtered == NULL) + token = *ip; + ret = fts_filter_filter(filter, &token); + test_assert(ret >= 0); + if (ret == 0) test_assert(*op == NULL); From dovecot at dovecot.org Sat May 9 08:32:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 08:32:19 +0000 Subject: dovecot-2.2: lib-fts: normalizer-icu now delays initialization o... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3ae8ae7f1022 changeset: 18553:3ae8ae7f1022 user: Teemu Huovila date: Sat May 09 11:09:37 2015 +0300 description: lib-fts: normalizer-icu now delays initialization of libicu. This way the normalizer can be quickly created even if it's never even used. diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 40 ++++++++++++++++++++------------ src/lib-fts/test-fts-filter.c | 7 +++-- 2 files changed, 29 insertions(+), 18 deletions(-) diffs (111 lines): diff -r 95a827d97e5b -r 3ae8ae7f1022 src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 11:06:45 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 11:09:37 2015 +0300 @@ -18,6 +18,7 @@ struct fts_filter filter; const char *error; pool_t pool; + const char *transliterator_id; UTransliterator *transliterator; }; @@ -170,15 +171,9 @@ { struct fts_filter_normalizer *np; pool_t pp; - UErrorCode err = U_ZERO_ERROR; - UParseError perr; - UChar *id_uchar = NULL; - int32_t id_len_uchar = 0; unsigned int i; const char *id = "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC"; - memset(&perr, 0, sizeof(perr)); - for (i = 0; settings[i] != NULL; i += 2) { const char *key = settings[i], *value = settings[i+1]; @@ -195,22 +190,36 @@ np = p_new(pp, struct fts_filter_normalizer, 1); np->pool = pp; np->filter = *fts_filter_normalizer_icu; - if (make_uchar(id, &id_uchar, &id_len_uchar) < 0) { + np->transliterator_id = p_strdup(pp, id); + *filter_r = &np->filter; + return 0; +} +static int fts_filter_normalizer_icu_create_trans(struct fts_filter_normalizer *np) +{ + UErrorCode err = U_ZERO_ERROR; + UParseError perr; + UChar *id_uchar = NULL; + int32_t id_len_uchar = 0; + + memset(&perr, 0, sizeof(perr)); + + if (make_uchar(np->transliterator_id, &id_uchar, &id_len_uchar) < 0) { + return -1; } np->transliterator = utrans_openU(id_uchar, u_strlen(id_uchar), UTRANS_FORWARD, NULL, 0, &perr, &err); if (U_FAILURE(err)) { if (perr.line >= 1) { - fts_filter_normalizer_icu_error(error_r, "Failed to open transliterator for id: %s. Lib ICU error: %s. Parse error on line %u offset %u.", id, u_errorName(err), perr.line, perr.offset); + fts_filter_normalizer_icu_error(&np->error, "Failed to open transliterator for id: %s. Lib ICU error: %s. Parse error on line %u offset %u.", + np->transliterator_id, u_errorName(err), perr.line, perr.offset); } else { - fts_filter_normalizer_icu_error(error_r, "Failed to open transliterator for id: %s. Lib ICU error: %s.", id, u_errorName(err)); + fts_filter_normalizer_icu_error(&np->error, "Failed to open transliterator for id: %s. Lib ICU error: %s.", + np->transliterator_id, u_errorName(err)); } - fts_filter_normalizer_icu_destroy(&np->filter); return -1; } - *filter_r = &np->filter; return 0; } @@ -229,14 +238,15 @@ if (np->error != NULL) goto err_exit; + if (np->transliterator == NULL) + if (fts_filter_normalizer_icu_create_trans(np) < 0) + goto err_exit; + if (make_uchar(*token, &utext, &utext_cap) < 0) { fts_filter_normalizer_icu_error(&np->error, "Conversion to UChar failed"); goto err_exit; } - /* - TODO: Some problems here. How much longer can the result - be, than the source? Can it be calculated? Preflighted? - */ + utext_limit = u_strlen(utext); utrans_transUChars(np->transliterator, utext, &utext_len, utext_cap, 0, &utext_limit, &err); diff -r 95a827d97e5b -r 3ae8ae7f1022 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Sat May 09 11:06:45 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Sat May 09 11:09:37 2015 +0300 @@ -459,14 +459,15 @@ const char *settings[] = {"id", "Any-One-Out-There; DKFN; [: Nonspacing Mark :] Remove", NULL}; - const char *error = NULL; + const char *error = NULL, *token = "foo"; int ret; test_begin("fts filter normalizer invalid id"); filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); ret = fts_filter_create(filter_class, NULL, NULL, settings, &norm, &error); - test_assert(ret < 0 && error != NULL); - test_assert(norm == NULL); + test_assert(ret == 0 && error == NULL); + ret = fts_filter_filter(norm, &token); + test_assert(ret < 0); test_end(); } From dovecot at dovecot.org Sat May 9 08:32:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 08:32:24 +0000 Subject: dovecot-2.2: lib-fts: Removed make_uchar() return value. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8c33d375c73e changeset: 18554:8c33d375c73e user: Teemu Huovila date: Sat May 09 11:10:31 2015 +0300 description: lib-fts: Removed make_uchar() return value. It was already de facto meaningless, so best we remove it. diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 21 ++++++--------------- 1 files changed, 6 insertions(+), 15 deletions(-) diffs (66 lines): diff -r 3ae8ae7f1022 -r 8c33d375c73e src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 11:09:37 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 11:10:31 2015 +0300 @@ -48,8 +48,7 @@ va_end(args); } -/* Helper to create UTF16, which libicu wants as input. Returns -1 on - error, 0 on success. +/* Helper to create UTF16, which libicu wants as input. On input, if *dst_uchars_r > 0, it indicates the number of UChar sized units that should be allocated for the text. However, the @@ -59,7 +58,7 @@ On return *dst_uchars_r will contain the number of UChar sized units allocated for the dst. NOT the number of bytes nor the length of the text. */ -static int make_uchar(const char *src, UChar **dst, int32_t *dst_uchars_r) +static void make_uchar(const char *src, UChar **dst, int32_t *dst_uchars_r) { UErrorCode err = U_ZERO_ERROR; int32_t len = strlen(src); @@ -96,7 +95,6 @@ i_panic("Lib ICU u_strFromUTF8 failed: %s", u_errorName(err)); i_assert(retp == *dst); i_assert(ustr_len == ustr_len_actual); - return 0; } static int make_utf8(const UChar *src, const char **_dst, const char **error_r) @@ -204,9 +202,8 @@ memset(&perr, 0, sizeof(perr)); - if (make_uchar(np->transliterator_id, &id_uchar, &id_len_uchar) < 0) { - return -1; - } + make_uchar(np->transliterator_id, &id_uchar, &id_len_uchar); + np->transliterator = utrans_openU(id_uchar, u_strlen(id_uchar), UTRANS_FORWARD, NULL, 0, &perr, &err); if (U_FAILURE(err)) { @@ -242,11 +239,7 @@ if (fts_filter_normalizer_icu_create_trans(np) < 0) goto err_exit; - if (make_uchar(*token, &utext, &utext_cap) < 0) { - fts_filter_normalizer_icu_error(&np->error, "Conversion to UChar failed"); - goto err_exit; - } - + make_uchar(*token, &utext, &utext_cap); utext_limit = u_strlen(utext); utrans_transUChars(np->transliterator, utext, &utext_len, utext_cap, 0, &utext_limit, &err); @@ -258,9 +251,7 @@ size utrans_transUChars indicated */ utext_len++; /* room for '\0' bytes(2) */ utext_cap = utext_len; - if (make_uchar(*token, &utext, &utext_cap) < 0) { - goto err_exit; - } + make_uchar(*token, &utext, &utext_cap); i_assert(utext_cap == utext_len); utext_limit = u_strlen(utext); utext_len = -1; From dovecot at dovecot.org Sat May 9 08:32:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 08:32:24 +0000 Subject: dovecot-2.2: fts: Improve filter error handling. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/230698a51fbb changeset: 18555:230698a51fbb user: Teemu Huovila date: Sat May 09 11:13:49 2015 +0300 description: fts: Improve filter error handling. In lib-fts: Move error storage to the generic filter struct level. Change make_utf8 helper to void also. In fts: Add the error argument to _filter() calls. diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 26 ++++++++------------------ src/lib-fts/fts-filter-private.h | 2 ++ src/lib-fts/fts-filter.c | 23 ++++++++++++++++------- src/lib-fts/fts-filter.h | 3 ++- src/lib-fts/test-fts-filter.c | 26 +++++++++++++------------- src/plugins/fts/fts-build-mail.c | 5 ++++- src/plugins/fts/fts-search-args.c | 6 ++++-- 7 files changed, 49 insertions(+), 42 deletions(-) diffs (truncated from 316 to 300 lines): diff -r 8c33d375c73e -r 230698a51fbb src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 11:10:31 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 11:13:49 2015 +0300 @@ -16,7 +16,6 @@ struct fts_filter_normalizer { struct fts_filter filter; - const char *error; pool_t pool; const char *transliterator_id; UTransliterator *transliterator; @@ -97,7 +96,7 @@ i_assert(ustr_len == ustr_len_actual); } -static int make_utf8(const UChar *src, const char **_dst, const char **error_r) +static void make_utf8(const UChar *src, const char **_dst) { char *dst; char *retp = NULL; @@ -133,14 +132,11 @@ dsize_actual, dsize); } if (0 != sub_num) { - fts_filter_normalizer_icu_error(error_r, "UTF8 string not well formed." - " Substitutions (%d) were made.", sub_num); - return -1; + i_panic("UTF8 string not well formed. " + "Substitutions (%d) were made.", sub_num); } i_assert(retp == dst); - *_dst = dst; - return 0; } static bool fts_filter_normalizer_icu_supports(const struct fts_language *lang) @@ -208,11 +204,11 @@ NULL, 0, &perr, &err); if (U_FAILURE(err)) { if (perr.line >= 1) { - fts_filter_normalizer_icu_error(&np->error, "Failed to open transliterator for id: %s. Lib ICU error: %s. Parse error on line %u offset %u.", + fts_filter_normalizer_icu_error(&np->filter.error, "Failed to open transliterator for id: %s. Lib ICU error: %s. Parse error on line %u offset %u.", np->transliterator_id, u_errorName(err), perr.line, perr.offset); } else { - fts_filter_normalizer_icu_error(&np->error, "Failed to open transliterator for id: %s. Lib ICU error: %s.", + fts_filter_normalizer_icu_error(&np->filter.error, "Failed to open transliterator for id: %s. Lib ICU error: %s.", np->transliterator_id, u_errorName(err)); } return -1; @@ -231,10 +227,6 @@ struct fts_filter_normalizer *np = (struct fts_filter_normalizer *)filter; - /* TODO: fix error handling */ - if (np->error != NULL) - goto err_exit; - if (np->transliterator == NULL) if (fts_filter_normalizer_icu_create_trans(np) < 0) goto err_exit; @@ -262,15 +254,13 @@ } if (U_FAILURE(err)) { - icu_error(&np->error, err, "utrans_transUChars()"); + icu_error(&np->filter.error, err, "utrans_transUChars()"); goto err_exit; } - if (make_utf8(utext, token, &np->error) < 0) { - goto err_exit; - } + make_utf8(utext, token); + return 1; - return 1; err_exit: *token = NULL; return -1; diff -r 8c33d375c73e -r 230698a51fbb src/lib-fts/fts-filter-private.h --- a/src/lib-fts/fts-filter-private.h Sat May 09 11:10:31 2015 +0300 +++ b/src/lib-fts/fts-filter-private.h Sat May 09 11:13:49 2015 +0300 @@ -18,12 +18,14 @@ struct fts_filter **filter_r, const char **error_r); int (*filter)(struct fts_filter *filter, const char **token); + void (*destroy)(struct fts_filter *filter); }; struct fts_filter { const char *class_name; /* name of the class this is based on */ const struct fts_filter_vfuncs *v; + const char *error; int refcount; struct fts_filter *parent; }; diff -r 8c33d375c73e -r 230698a51fbb src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Sat May 09 11:10:31 2015 +0300 +++ b/src/lib-fts/fts-filter.c Sat May 09 11:13:49 2015 +0300 @@ -93,18 +93,27 @@ /* TODO: Avoid multiple allocations by using a buffer in v->filter? Do this non-recursively? */ int -fts_filter_filter(struct fts_filter *filter, const char **token) +fts_filter_filter(struct fts_filter *filter, const char **token, + const char **error_r) { - int ret; + int ret = 0; - if (filter->parent == NULL) - return filter->v->filter(filter, token); + if (filter->error != NULL) { + *error_r = filter->error; + return -1; + } - ret = fts_filter_filter(filter->parent, token); + /* Recurse to parent. */ + if (filter->parent != NULL) + ret = fts_filter_filter(filter->parent, token, error_r); - if(ret > 0) - return filter->v->filter(filter, token); + /* Parent returned token or no parent. */ + if(ret > 0 || filter->parent == NULL) + ret = filter->v->filter(filter, token); + + if (filter->error != NULL) + *error_r = filter->error; return ret; } diff -r 8c33d375c73e -r 230698a51fbb src/lib-fts/fts-filter.h --- a/src/lib-fts/fts-filter.h Sat May 09 11:10:31 2015 +0300 +++ b/src/lib-fts/fts-filter.h Sat May 09 11:13:49 2015 +0300 @@ -63,6 +63,7 @@ Input is also given via *token. */ int -fts_filter_filter(struct fts_filter *filter, const char **token); +fts_filter_filter(struct fts_filter *filter, const char **token, + const char **error_r); #endif diff -r 8c33d375c73e -r 230698a51fbb src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Sat May 09 11:10:31 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Sat May 09 11:13:49 2015 +0300 @@ -35,7 +35,7 @@ op = output; while (*ip != NULL) { token = *ip; - ret = fts_filter_filter(filter, &token); + ret = fts_filter_filter(filter, &token, &error); test_assert(ret >= 0); if (ret == 0) test_assert(*op == NULL); @@ -79,7 +79,7 @@ op = output; while (*ip != NULL) { token = *ip; - ret = fts_filter_filter(filter, &token); + ret = fts_filter_filter(filter, &token, &error); test_assert(ret >= 0); if (ret == 0) test_assert(*op == NULL); @@ -100,7 +100,7 @@ op = output2; while (*ip != NULL) { token = *ip; - ret = fts_filter_filter(filter, &token); + ret = fts_filter_filter(filter, &token, &error); if (ret == 0) test_assert(*op == NULL); else { @@ -143,7 +143,7 @@ op = output; while (*ip != NULL) { token = *ip; - ret = fts_filter_filter(filter, &token); + ret = fts_filter_filter(filter, &token, &error); test_assert(ret >= 0); if (ret == 0) test_assert(*op == NULL); @@ -207,7 +207,7 @@ bpp = bases; for (tpp=tokens; *tpp != NULL; tpp++) { token = *tpp; - ret = fts_filter_filter(stemmer, &token); + ret = fts_filter_filter(stemmer, &token, &error); test_assert(token != NULL); test_assert(null_strcmp(token, *bpp) == 0); bpp++; @@ -242,7 +242,7 @@ bpp = bases; for (tpp=tokens; *tpp != NULL; tpp++) { token = *tpp; - ret = fts_filter_filter(stemmer, &token); + ret = fts_filter_filter(stemmer, &token, &error); test_assert(token != NULL); test_assert(null_strcmp(token, *bpp) == 0); bpp++; @@ -289,7 +289,7 @@ bpp = bases; for (tpp=tokens; *tpp != NULL; tpp++) { token = *tpp; - ret = fts_filter_filter(stemmer, &token); + ret = fts_filter_filter(stemmer, &token, &error); if (ret == 0) test_assert(*bpp == NULL); else { @@ -344,7 +344,7 @@ for (i = 0; i < N_ELEMENTS(input); i++) { if (input[i] != NULL) { token = input[i]; - test_assert_idx(fts_filter_filter(norm, &token) == 1, i); + test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i); test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); } } @@ -389,7 +389,7 @@ for (i = 0; i < N_ELEMENTS(input); i++) { if (input[i] != NULL) { token = input[i]; - test_assert_idx(fts_filter_filter(norm, &token) == 1, i); + test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i); test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); } } @@ -437,7 +437,7 @@ sha512_init(&ctx); while (NULL != fgets(buf, sizeof(buf), input)) { tokens = buf; - if (fts_filter_filter(norm, &tokens) != 1){ + if (fts_filter_filter(norm, &tokens, &error) != 1){ break; } sha512_loop(&ctx, tokens, strlen(tokens)); @@ -466,8 +466,8 @@ filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); ret = fts_filter_create(filter_class, NULL, NULL, settings, &norm, &error); test_assert(ret == 0 && error == NULL); - ret = fts_filter_filter(norm, &token); - test_assert(ret < 0); + ret = fts_filter_filter(norm, &token, &error); + test_assert(ret < 0 && error != NULL); test_end(); } @@ -517,7 +517,7 @@ bpp = bases; for (tpp = tokens; *tpp != NULL; tpp++) { token = *tpp; - ret = fts_filter_filter(stemmer, &token); + ret = fts_filter_filter(stemmer, &token, &error); if (ret == 0) test_assert(*bpp == NULL); else { diff -r 8c33d375c73e -r 230698a51fbb src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Sat May 09 11:10:31 2015 +0300 +++ b/src/plugins/fts/fts-build-mail.c Sat May 09 11:13:49 2015 +0300 @@ -244,12 +244,13 @@ struct fts_tokenizer *tokenizer; struct fts_filter *filter = ctx->cur_user_lang->filter; const char *token; + const char *error; int ret; tokenizer = fts_user_get_index_tokenizer(ctx->update_ctx->backend->ns->user); while ((ret = fts_tokenizer_next(tokenizer, data, size, &token)) > 0) { if (filter != NULL) { - ret = fts_filter_filter(filter, &token); + ret = fts_filter_filter(filter, &token, &error); if (ret == 0) continue; if (ret < 0) @@ -260,6 +261,8 @@ strlen(token)) < 0) return -1; } + if (ret < 0) + i_error("fts: Couldn't create indexable tokens: %s", error); return ret; } diff -r 8c33d375c73e -r 230698a51fbb src/plugins/fts/fts-search-args.c --- a/src/plugins/fts/fts-search-args.c Sat May 09 11:10:31 2015 +0300 +++ b/src/plugins/fts/fts-search-args.c Sat May 09 11:13:49 2015 +0300 @@ -63,7 +63,7 @@ struct mail_search_arg *arg; struct fts_user_language *const *langp; ARRAY_TYPE(const_string) tokens; - const char *token2; + const char *token2, *error; int ret; From dovecot at dovecot.org Sat May 9 08:32:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 08:32:24 +0000 Subject: dovecot-2.2: lib-fts: Delay snowball filter initialization until... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a45a328c5019 changeset: 18556:a45a328c5019 user: Teemu Huovila date: Sat May 09 11:14:51 2015 +0300 description: lib-fts: Delay snowball filter initialization until it's needed. diffstat: src/lib-fts/fts-filter-stemmer-snowball.c | 33 ++++++++++++++++++++++-------- 1 files changed, 24 insertions(+), 9 deletions(-) diffs (76 lines): diff -r 230698a51fbb -r a45a328c5019 src/lib-fts/fts-filter-stemmer-snowball.c --- a/src/lib-fts/fts-filter-stemmer-snowball.c Sat May 09 11:13:49 2015 +0300 +++ b/src/lib-fts/fts-filter-stemmer-snowball.c Sat May 09 11:14:51 2015 +0300 @@ -11,6 +11,8 @@ struct fts_filter_stemmer_snowball { struct fts_filter filter; + pool_t pool; + struct fts_language *lang; struct sb_stemmer *stemmer; }; @@ -33,7 +35,7 @@ if (sp->stemmer != NULL) sb_stemmer_delete(sp->stemmer); - i_free(sp); + pool_unref(&sp->pool); } static int @@ -43,6 +45,7 @@ const char **error_r) { struct fts_filter_stemmer_snowball *sp; + pool_t pp; *filter_r = NULL; @@ -50,19 +53,27 @@ *error_r = t_strdup_printf("Unknown setting: %s", settings[0]); return -1; } + pp = pool_alloconly_create(MEMPOOL_GROWING"fts_filter_stemmer_snowball", + sizeof(struct fts_filter)); + sp = p_new(pp, struct fts_filter_stemmer_snowball, 1); + sp->pool = pp; + sp->filter = *fts_filter_stemmer_snowball; + sp->lang = p_malloc(sp->pool, sizeof(struct fts_language)); + sp->lang->name = str_lcase(p_strdup(sp->pool, lang->name)); + *filter_r = &sp->filter; + return 0; +} - sp = i_new(struct fts_filter_stemmer_snowball, 1); - sp->filter = *fts_filter_stemmer_snowball; - sp->stemmer = sb_stemmer_new(t_str_lcase(lang->name), NULL); +static int +fts_filter_stemmer_snowball_create_stemmer(struct fts_filter_stemmer_snowball *sp) +{ + sp->stemmer = sb_stemmer_new(sp->lang->name, NULL); if (sp->stemmer == NULL) { - if (error_r != NULL) { - *error_r = t_strdup_printf("Creating a Snowball stemmer failed." \ - " lang: %s", lang->name); - } + sp->filter.error = t_strdup_printf("Creating a Snowball stemmer failed." \ + " lang: %s", sp->lang->name); fts_filter_stemmer_snowball_destroy(&sp->filter); return -1; } - *filter_r = &sp->filter; return 0; } @@ -75,6 +86,10 @@ struct fts_filter_stemmer_snowball *sp = (struct fts_filter_stemmer_snowball *) filter; + if (sp->stemmer == NULL) + if (fts_filter_stemmer_snowball_create_stemmer(sp) < 0) + return -1; + base = sb_stemmer_stem(sp->stemmer, (const unsigned char *)*token, strlen(*token)); len = sb_stemmer_length(sp->stemmer); *token = t_strdup_until(base, base + len); From dovecot at dovecot.org Sat May 9 08:32:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 08:32:35 +0000 Subject: dovecot-2.2: lib-fts: Delay stopwords filter full initialization... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6a6ce51597f7 changeset: 18557:6a6ce51597f7 user: Teemu Huovila date: Sat May 09 11:15:34 2015 +0300 description: lib-fts: Delay stopwords filter full initialization until it's needed. diffstat: src/lib-fts/fts-filter-stopwords.c | 27 +++++++++++++++++---------- src/lib-fts/test-fts-filter.c | 12 +++++++----- 2 files changed, 24 insertions(+), 15 deletions(-) diffs (108 lines): diff -r a45a328c5019 -r 6a6ce51597f7 src/lib-fts/fts-filter-stopwords.c --- a/src/lib-fts/fts-filter-stopwords.c Sat May 09 11:14:51 2015 +0300 +++ b/src/lib-fts/fts-filter-stopwords.c Sat May 09 11:15:34 2015 +0300 @@ -74,7 +74,8 @@ static void fts_filter_stopwords_destroy(struct fts_filter *filter) { struct fts_filter_stopwords *sp = (struct fts_filter_stopwords *)filter; - hash_table_destroy(&sp->stopwords); + if (hash_table_is_created(sp->stopwords)) + hash_table_destroy(&sp->stopwords); pool_unref(&sp->pool); return; } @@ -87,7 +88,6 @@ { struct fts_filter_stopwords *sp; pool_t pp; - int ret; const char *dir = NULL; unsigned int i; @@ -101,7 +101,6 @@ return -1; } } - pp = pool_alloconly_create(MEMPOOL_GROWING"fts_filter_stopwords", sizeof(struct fts_filter)); sp = p_new(pp, struct fts_filter_stopwords, 1); @@ -109,19 +108,24 @@ sp->pool = pp; sp->lang = p_malloc(sp->pool, sizeof(struct fts_language)); sp->lang->name = str_lcase(p_strdup(sp->pool, lang->name)); - hash_table_create(&sp->stopwords, sp->pool, 0, str_hash, strcmp); if (dir != NULL) sp->stopwords_dir = p_strdup(pp, dir); else sp->stopwords_dir = DATADIR"/stopwords"; *filter_r = &sp->filter; + return 0; +} + +static int +fts_filter_stopwords_create_stopwords(struct fts_filter_stopwords *sp) +{ + int ret; + + hash_table_create(&sp->stopwords, sp->pool, 0, str_hash, strcmp); ret = fts_filter_stopwords_read_list(sp); - if (ret < 0) { - *error_r = t_strdup_printf( - "Failed to read stopword list %s", sp->stopwords_dir); - fts_filter_stopwords_destroy(*filter_r); - *filter_r = NULL; - } + if (ret < 0) + sp->filter.error = t_strdup_printf("Failed to read stopword list %s", + sp->stopwords_dir); return ret; } @@ -132,6 +136,9 @@ struct fts_filter_stopwords *sp = (struct fts_filter_stopwords *) filter; + if (!hash_table_is_created(sp->stopwords)) + if (fts_filter_stopwords_create_stopwords(sp) < 0) + return -1; stopword = hash_table_lookup(sp->stopwords, *token); if (stopword != NULL) { *token = NULL; diff -r a45a328c5019 -r 6a6ce51597f7 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Sat May 09 11:14:51 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Sat May 09 11:15:34 2015 +0300 @@ -160,18 +160,20 @@ test_end(); } -static void test_fts_filter_stopwords_fail_create(void) +static void test_fts_filter_stopwords_fail_lazy_init(void) { const struct fts_filter *filter_class; const struct fts_language unknown = { .name = "bebobidoop" }; struct fts_filter *filter = NULL; - const char *error; + const char *error = NULL, *token = "foobar"; int ret; - test_begin("fts filter stopwords, fail create()"); + test_begin("fts filter stopwords, fail filter() (lazy init)"); filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); ret = fts_filter_create(filter_class, NULL, &unknown, stopword_settings, &filter, &error); - test_assert(ret == -1 && filter == NULL && error != NULL); + test_assert(ret == 0 && filter != NULL && error == NULL); + ret = fts_filter_filter(filter, &token, &error); + test_assert(ret == -1 && error != NULL); test_end(); } @@ -545,7 +547,7 @@ test_fts_filter_stopwords_eng, test_fts_filter_stopwords_fin, test_fts_filter_stopwords_fra, - test_fts_filter_stopwords_fail_create, + test_fts_filter_stopwords_fail_lazy_init, #ifdef HAVE_FTS_STEMMER test_fts_filter_stemmer_snowball_stem_english, test_fts_filter_stemmer_snowball_stem_french, From dovecot at dovecot.org Sat May 9 08:32:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 08:32:35 +0000 Subject: dovecot-2.2: lib-fts: Default to simple tokenizer algorithm Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8d445959df03 changeset: 18558:8d445959df03 user: Teemu Huovila date: Sat May 09 11:15:50 2015 +0300 description: lib-fts: Default to simple tokenizer algorithm diffstat: src/lib-fts/fts-tokenizer-generic.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diffs (26 lines): diff -r 6a6ce51597f7 -r 8d445959df03 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Sat May 09 11:15:34 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Sat May 09 11:15:50 2015 +0300 @@ -30,7 +30,7 @@ { struct generic_fts_tokenizer *tok; unsigned int max_length = FTS_DEFAULT_TOKEN_MAX_LENGTH; - enum boundary_algorithm algo = BOUNDARY_ALGORITHM_NONE; + enum boundary_algorithm algo = BOUNDARY_ALGORITHM_SIMPLE; unsigned int i; for (i = 0; settings[i] != NULL; i += 2) { @@ -44,10 +44,10 @@ return -1; } } else if (strcasecmp(key, "algorithm") == 0) { - if (strcasecmp(value, ALGORITHM_SIMPLE_NAME) == 0) - algo = BOUNDARY_ALGORITHM_SIMPLE; - else if (strcasecmp(value, ALGORITHM_TR29_NAME) == 0) + if (strcasecmp(value, ALGORITHM_TR29_NAME) == 0) algo = BOUNDARY_ALGORITHM_TR29; + else if (strcasecmp(value, ALGORITHM_SIMPLE_NAME) == 0) + ; else { *error_r = t_strdup_printf( "Invalid algorithm: %s", value); From dovecot at dovecot.org Sat May 9 08:32:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 08:32:35 +0000 Subject: dovecot-2.2: lib-fts: Fixed using max_length setting in simple t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2048dade16e7 changeset: 18559:2048dade16e7 user: Teemu Huovila date: Sat May 09 11:16:22 2015 +0300 description: lib-fts: Fixed using max_length setting in simple tokenizer diffstat: src/lib-fts/fts-tokenizer-generic.c | 17 +++++++++-------- src/lib-fts/test-fts-tokenizer.c | 9 +++++++-- 2 files changed, 16 insertions(+), 10 deletions(-) diffs (80 lines): diff -r 8d445959df03 -r 2048dade16e7 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Sat May 09 11:15:50 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Sat May 09 11:16:22 2015 +0300 @@ -86,7 +86,7 @@ fts_tokenizer_generic_simple_current_token(struct generic_fts_tokenizer *tok, const char **token_r) { - *token_r = t_strndup(tok->token->data, tok->token->used); + *token_r = t_strndup(tok->token->data, I_MIN(tok->token->used, tok->max_length)); buffer_set_used_size(tok->token, 0); return 1; } @@ -147,8 +147,7 @@ char_start_i = i; if (data_is_word_boundary(data, size, &i)) { len = char_start_i - start; - buffer_append(tok->token, data + start, - I_MIN(len, tok->max_length)); + buffer_append(tok->token, data + start, len); if (tok->token->used == 0) { /* no text read yet */ start = i + 1; @@ -161,14 +160,16 @@ } /* word boundary not found yet */ len = i - start; - buffer_append(tok->token, data + start, I_MIN(len, tok->max_length)); - + buffer_append(tok->token, data + start, len); *skip_r = i; - if (size == 0 && tok->token->used > 0) { - /* return the last token */ + /* return the last token */ + if (size == 0 && tok->token->used > 0) return fts_tokenizer_generic_simple_current_token(tok, token_r); - } + + /* token too long */ + if (tok->token->used > tok->max_length) + return fts_tokenizer_generic_simple_current_token(tok, token_r); return 0; } diff -r 8d445959df03 -r 2048dade16e7 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Sat May 09 11:15:50 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Sat May 09 11:16:22 2015 +0300 @@ -6,6 +6,8 @@ #include "test-common.h" #include "fts-tokenizer.h" #include "fts-tokenizer-private.h" +/* TODO: fix including and linking of this. */ +/* #include "fts-tokenizer-generic-private.h" */ #include @@ -13,11 +15,12 @@ { static const unsigned char input[] = "hello world\r\nAnd there\twas: text " - "galore, and more.\n\n (\"Hello world\")last "; + "galore, and longlonglongabcdefghijklmnopqrstuvwxyz more.\n\n (\"Hello world\")last "; static const char *const expected_output[] = { "hello", "world", "And", "there", "was", "text", "galore", - "and", "more", "Hello", "world", "last", NULL + "and", "longlonglongabcdefghijklmnopqr", + "more", "Hello", "world", "last", NULL }; const struct fts_tokenizer *tok_class; struct fts_tokenizer *tok; @@ -28,6 +31,8 @@ fts_tokenizers_init(); tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME); test_assert(fts_tokenizer_create(tok_class, NULL, NULL, &tok, &error) == 0); +/*TODO: Uncomment when fts-tokenizer-generic-private.h inclusion is fixed */ +/*test_assert(((struct generic_fts_tokenizer *) tok)->algorithm == BOUNDARY_ALGORITHM_SIMPLE);*/ while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) { test_assert(strcmp(token, *eopp) == 0); eopp++; From dovecot at dovecot.org Sat May 9 08:32:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 08:32:35 +0000 Subject: dovecot-2.2: lib-fts: Improve using max_length in tr29 tokenizer Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b0a934361563 changeset: 18560:b0a934361563 user: Teemu Huovila date: Sat May 09 11:17:03 2015 +0300 description: lib-fts: Improve using max_length in tr29 tokenizer diffstat: src/lib-fts/fts-tokenizer-generic.c | 17 +++++++++-------- src/lib-fts/test-fts-tokenizer.c | 5 +++-- 2 files changed, 12 insertions(+), 10 deletions(-) diffs (69 lines): diff -r 2048dade16e7 -r b0a934361563 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Sat May 09 11:16:22 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Sat May 09 11:17:03 2015 +0300 @@ -469,11 +469,14 @@ const char **token_r) { size_t end_skip = 0; + ssize_t len; if (is_one_past_end(tok)) end_skip = tok->last_size; - *token_r = t_strndup(tok->token->data, tok->token->used - end_skip); + len = I_MIN(tok->token->used, tok->max_length) - end_skip; + i_assert(len > 0); + *token_r = t_strndup(tok->token->data, len); buffer_set_used_size(tok->token, 0); tok->prev_prev_letter = LETTER_TYPE_NONE; tok->prev_letter = LETTER_TYPE_NONE; @@ -525,7 +528,7 @@ (struct generic_fts_tokenizer *)_tok; unichar_t c; - size_t i, char_start_i, start_skip = 0, len; + size_t i, char_start_i, start_skip = 0; enum letter_type lt; /* TODO: Process 8bit chars separately, to speed things up. */ @@ -542,17 +545,15 @@ continue; } if (uni_found_word_boundary(tok, lt)) { - len = I_MIN(char_start_i, tok->max_length); - i_assert(len >= start_skip && size >= start_skip); + i_assert(char_start_i >= start_skip && size >= start_skip); buffer_append(tok->token, data + start_skip, - len - start_skip); + char_start_i - start_skip); *skip_r = i + 1; return fts_tokenizer_generic_tr29_current_token(tok, token_r); } } - len = I_MIN(i, tok->max_length); - i_assert(len >= start_skip && size >= start_skip); - buffer_append(tok->token, data + start_skip, len - start_skip); + i_assert(i >= start_skip && size >= start_skip); + buffer_append(tok->token, data + start_skip, i - start_skip); *skip_r = i; if (size == 0 && tok->token->used > 0) { diff -r 2048dade16e7 -r b0a934361563 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Sat May 09 11:16:22 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Sat May 09 11:17:03 2015 +0300 @@ -126,12 +126,13 @@ { static const unsigned char input[] = "hello world\r\n\nAnd there\twas: text " - "galore, and more.\n\n (\"Hello world\")3.14 3,14 last 1."; + "galore, and more.\n\n (\"Hello world\")3.14 3,14 last" + " longlonglongabcdefghijklmnopqrstuvwxyz 1."; static const char *const expected_output[] = { "hello", "world", "And", "there", "was", "text", "galore", "and", "more", "Hello", "world", "3.14", - "3,14", "last", "1", NULL + "3,14", "last", "longlonglongabcdefghijklmnopqr", "1", NULL }; const struct fts_tokenizer *tok_class; struct fts_tokenizer *tok; From dovecot at dovecot.org Sat May 9 08:35:41 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 08:35:41 +0000 Subject: dovecot-2.2: lib-fts: Snowball's sb_stemmer_stem() can only fail... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7de648f42bc0 changeset: 18561:7de648f42bc0 user: Timo Sirainen date: Sat May 09 11:33:45 2015 +0300 description: lib-fts: Snowball's sb_stemmer_stem() can only fail because it runs out of memory. diffstat: src/lib-fts/fts-filter-stemmer-snowball.c | 19 ++++++++++++------- 1 files changed, 12 insertions(+), 7 deletions(-) diffs (36 lines): diff -r b0a934361563 -r 7de648f42bc0 src/lib-fts/fts-filter-stemmer-snowball.c --- a/src/lib-fts/fts-filter-stemmer-snowball.c Sat May 09 11:17:03 2015 +0300 +++ b/src/lib-fts/fts-filter-stemmer-snowball.c Sat May 09 11:33:45 2015 +0300 @@ -81,20 +81,25 @@ fts_filter_stemmer_snowball_filter(struct fts_filter *filter, const char **token) { - const sb_symbol *base; - int len; struct fts_filter_stemmer_snowball *sp = (struct fts_filter_stemmer_snowball *) filter; + const sb_symbol *base; - if (sp->stemmer == NULL) + if (sp->stemmer == NULL) { if (fts_filter_stemmer_snowball_create_stemmer(sp) < 0) return -1; + } base = sb_stemmer_stem(sp->stemmer, (const unsigned char *)*token, strlen(*token)); - len = sb_stemmer_length(sp->stemmer); - *token = t_strdup_until(base, base + len); - - return *token != NULL? 1: -1; + if (base == NULL) { + /* the only reason why this could fail is because of + out of memory. */ + i_fatal_status(FATAL_OUTOFMEM, + "sb_stemmer_stem(len=%"PRIuSIZE_T") failed: " + "Out of memory", strlen(*token)); + } + *token = t_strndup(base, sb_stemmer_length(sp->stemmer)); + return 1; } #else From dovecot at dovecot.org Sat May 9 09:30:41 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 09:30:41 +0000 Subject: dovecot-2.2: lib-fts: Changed filter's internal APIs to return e... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cfa35069876b changeset: 18562:cfa35069876b user: Timo Sirainen date: Sat May 09 11:46:09 2015 +0300 description: lib-fts: Changed filter's internal APIs to return error_r directly. It's not very safe to store strings allocated from data stack to any permanent structs. diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 73 +++++++++-------------------- src/lib-fts/fts-filter-normalizer-simple.c | 9 +-- src/lib-fts/fts-filter-private.h | 4 +- src/lib-fts/fts-filter-stemmer-snowball.c | 15 +++-- src/lib-fts/fts-filter-stopwords.c | 15 +++-- src/lib-fts/fts-filter.c | 14 +---- src/lib-fts/fts-filter.h | 2 +- 7 files changed, 53 insertions(+), 79 deletions(-) diffs (truncated from 331 to 300 lines): diff -r 7de648f42bc0 -r cfa35069876b src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 11:33:45 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 11:46:09 2015 +0300 @@ -1,6 +1,7 @@ /* Copyright (c) 2014-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "str.h" #include "unichar.h" /* unicode replacement char */ #include "fts-filter.h" #include "fts-filter-private.h" @@ -21,32 +22,6 @@ UTransliterator *transliterator; }; -static void -icu_error(const char **error_r, const UErrorCode err, const char *func) -{ - if (error_r == NULL) - return; - - if (U_FAILURE(err)) { - *error_r = t_strdup_printf("Lib ICU function %s failed: %s\n", - func, u_errorName(err)); - } -} - -/* Thin wrapper for vprintf */ -static void ATTR_FORMAT(2, 3) -fts_filter_normalizer_icu_error(const char **error_r, const char *format, ...) -{ - va_list args; - - if (error_r == NULL) - return; - - va_start(args, format); - *error_r = t_strdup_vprintf(format, args); - va_end(args); -} - /* Helper to create UTF16, which libicu wants as input. On input, if *dst_uchars_r > 0, it indicates the number of UChar @@ -154,7 +129,6 @@ if (np->transliterator != NULL) utrans_close(np->transliterator); pool_unref(&np->pool); - return; } static int @@ -183,13 +157,14 @@ sizeof(struct fts_filter_normalizer)); np = p_new(pp, struct fts_filter_normalizer, 1); np->pool = pp; - np->filter = *fts_filter_normalizer_icu; + np->filter = *fts_filter_normalizer_icu; np->transliterator_id = p_strdup(pp, id); *filter_r = &np->filter; return 0; } -static int fts_filter_normalizer_icu_create_trans(struct fts_filter_normalizer *np) +static int fts_filter_normalizer_icu_create_trans(struct fts_filter_normalizer *np, + const char **error_r) { UErrorCode err = U_ZERO_ERROR; UParseError perr; @@ -203,33 +178,36 @@ np->transliterator = utrans_openU(id_uchar, u_strlen(id_uchar), UTRANS_FORWARD, NULL, 0, &perr, &err); if (U_FAILURE(err)) { + string_t *str = t_str_new(128); + + str_printfa(str, "Failed to open transliterator for id '%s': %s", + np->transliterator_id, u_errorName(err)); if (perr.line >= 1) { - fts_filter_normalizer_icu_error(&np->filter.error, "Failed to open transliterator for id: %s. Lib ICU error: %s. Parse error on line %u offset %u.", - np->transliterator_id, u_errorName(err), perr.line, perr.offset); + /* we have only one line in our ID */ + str_printfa(str, " (parse error on offset %u)", perr.offset); } - else { - fts_filter_normalizer_icu_error(&np->filter.error, "Failed to open transliterator for id: %s. Lib ICU error: %s.", - np->transliterator_id, u_errorName(err)); - } + *error_r = str_c(str); return -1; } return 0; } static int -fts_filter_normalizer_icu_filter(struct fts_filter *filter, const char **token) +fts_filter_normalizer_icu_filter(struct fts_filter *filter, const char **token, + const char **error_r) { + struct fts_filter_normalizer *np = + (struct fts_filter_normalizer *)filter; UErrorCode err = U_ZERO_ERROR; UChar *utext = NULL; int32_t utext_cap = 0; int32_t utext_len = -1; int32_t utext_limit; - struct fts_filter_normalizer *np = - (struct fts_filter_normalizer *)filter; - if (np->transliterator == NULL) - if (fts_filter_normalizer_icu_create_trans(np) < 0) - goto err_exit; + if (np->transliterator == NULL) { + if (fts_filter_normalizer_icu_create_trans(np, error_r) < 0) + return -1; + } make_uchar(*token, &utext, &utext_cap); utext_limit = u_strlen(utext); @@ -238,7 +216,6 @@ /* Data did not fit into utext. */ if (utext_len > utext_cap || err == U_BUFFER_OVERFLOW_ERROR) { - /* This is a crude retry fix... Make a new utext of the size utrans_transUChars indicated */ utext_len++; /* room for '\0' bytes(2) */ @@ -254,16 +231,13 @@ } if (U_FAILURE(err)) { - icu_error(&np->filter.error, err, "utrans_transUChars()"); - goto err_exit; + *error_r = t_strdup_printf("utrans_transUChars() failed: %s\n", + u_errorName(err)); + return -1; } make_utf8(utext, token); return 1; - - err_exit: - *token = NULL; - return -1; } #else @@ -286,7 +260,8 @@ static int fts_filter_normalizer_icu_filter(struct fts_filter *filter ATTR_UNUSED, - const char *token ATTR_UNUSED) + const char **token ATTR_UNUSED, + const char **error_r ATTR_UNUSED) { return NULL; } diff -r 7de648f42bc0 -r cfa35069876b src/lib-fts/fts-filter-normalizer-simple.c --- a/src/lib-fts/fts-filter-normalizer-simple.c Sat May 09 11:33:45 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-simple.c Sat May 09 11:46:09 2015 +0300 @@ -50,17 +50,16 @@ static int fts_filter_normalizer_simple_filter(struct fts_filter *_filter, - const char **token) + const char **token, + const char **error_r ATTR_UNUSED) { struct fts_filter_normalizer_simple *filter = (struct fts_filter_normalizer_simple *)_filter; str_truncate(filter->str, 0); if (uni_utf8_to_decomposed_titlecase(*token, strlen(*token), - filter->str) < 0) { - *token = NULL; - return -1; - } + filter->str) < 0) + i_panic("fts-normalizer-simple: Token is not valid UTF-8: %s", *token); *token = str_c(filter->str); return 1; } diff -r 7de648f42bc0 -r cfa35069876b src/lib-fts/fts-filter-private.h --- a/src/lib-fts/fts-filter-private.h Sat May 09 11:33:45 2015 +0300 +++ b/src/lib-fts/fts-filter-private.h Sat May 09 11:46:09 2015 +0300 @@ -17,7 +17,8 @@ const char *const *settings, struct fts_filter **filter_r, const char **error_r); - int (*filter)(struct fts_filter *filter, const char **token); + int (*filter)(struct fts_filter *filter, const char **token, + const char **error_r); void (*destroy)(struct fts_filter *filter); }; @@ -25,7 +26,6 @@ struct fts_filter { const char *class_name; /* name of the class this is based on */ const struct fts_filter_vfuncs *v; - const char *error; int refcount; struct fts_filter *parent; }; diff -r 7de648f42bc0 -r cfa35069876b src/lib-fts/fts-filter-stemmer-snowball.c --- a/src/lib-fts/fts-filter-stemmer-snowball.c Sat May 09 11:33:45 2015 +0300 +++ b/src/lib-fts/fts-filter-stemmer-snowball.c Sat May 09 11:46:09 2015 +0300 @@ -65,12 +65,14 @@ } static int -fts_filter_stemmer_snowball_create_stemmer(struct fts_filter_stemmer_snowball *sp) +fts_filter_stemmer_snowball_create_stemmer(struct fts_filter_stemmer_snowball *sp, + const char **error_r) { sp->stemmer = sb_stemmer_new(sp->lang->name, NULL); if (sp->stemmer == NULL) { - sp->filter.error = t_strdup_printf("Creating a Snowball stemmer failed." \ - " lang: %s", sp->lang->name); + *error_r = t_strdup_printf( + "Creating a Snowball stemmer for language '%s' failed.", + sp->lang->name); fts_filter_stemmer_snowball_destroy(&sp->filter); return -1; } @@ -79,14 +81,14 @@ static int fts_filter_stemmer_snowball_filter(struct fts_filter *filter, - const char **token) + const char **token, const char **error_r) { struct fts_filter_stemmer_snowball *sp = (struct fts_filter_stemmer_snowball *) filter; const sb_symbol *base; if (sp->stemmer == NULL) { - if (fts_filter_stemmer_snowball_create_stemmer(sp) < 0) + if (fts_filter_stemmer_snowball_create_stemmer(sp, error_r) < 0) return -1; } @@ -125,7 +127,8 @@ static int fts_filter_stemmer_snowball_filter(struct fts_filter *filter ATTR_UNUSED, - const char **token ATTR_UNUSED) + const char **token ATTR_UNUSED, + const char **error_r ATTR_UNUSED) { return -1; } diff -r 7de648f42bc0 -r cfa35069876b src/lib-fts/fts-filter-stopwords.c --- a/src/lib-fts/fts-filter-stopwords.c Sat May 09 11:33:45 2015 +0300 +++ b/src/lib-fts/fts-filter-stopwords.c Sat May 09 11:46:09 2015 +0300 @@ -117,27 +117,30 @@ } static int -fts_filter_stopwords_create_stopwords(struct fts_filter_stopwords *sp) +fts_filter_stopwords_create_stopwords(struct fts_filter_stopwords *sp, + const char **error_r) { int ret; hash_table_create(&sp->stopwords, sp->pool, 0, str_hash, strcmp); ret = fts_filter_stopwords_read_list(sp); - if (ret < 0) - sp->filter.error = t_strdup_printf("Failed to read stopword list %s", - sp->stopwords_dir); + if (ret < 0) { + *error_r = t_strdup_printf("Failed to read stopword list %s", + sp->stopwords_dir); + } return ret; } static int -fts_filter_stopwords_filter(struct fts_filter *filter, const char **token) +fts_filter_stopwords_filter(struct fts_filter *filter, const char **token, + const char **error_r) { const char *stopword; struct fts_filter_stopwords *sp = (struct fts_filter_stopwords *) filter; if (!hash_table_is_created(sp->stopwords)) - if (fts_filter_stopwords_create_stopwords(sp) < 0) + if (fts_filter_stopwords_create_stopwords(sp, error_r) < 0) return -1; stopword = hash_table_lookup(sp->stopwords, *token); if (stopword != NULL) { diff -r 7de648f42bc0 -r cfa35069876b src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Sat May 09 11:33:45 2015 +0300 +++ b/src/lib-fts/fts-filter.c Sat May 09 11:46:09 2015 +0300 @@ -99,21 +99,15 @@ { int ret = 0; - if (filter->error != NULL) { - *error_r = filter->error; - return -1; From dovecot at dovecot.org Sat May 9 09:30:41 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 09:30:41 +0000 Subject: dovecot-2.2: lib-fts: Fixed test-fts-filter unit test to handle ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e1d2702eea84 changeset: 18563:e1d2702eea84 user: Timo Sirainen date: Sat May 09 11:57:40 2015 +0300 description: lib-fts: Fixed test-fts-filter unit test to handle errors a bit better. diffstat: src/lib-fts/fts-filter.c | 2 ++ src/lib-fts/test-fts-filter.c | 29 ++++++++++++++++------------- 2 files changed, 18 insertions(+), 13 deletions(-) diffs (90 lines): diff -r cfa35069876b -r e1d2702eea84 src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Sat May 09 11:46:09 2015 +0300 +++ b/src/lib-fts/fts-filter.c Sat May 09 11:57:40 2015 +0300 @@ -109,5 +109,7 @@ if (ret <= 0) *token = NULL; + else + i_assert(*token != NULL); return ret; } diff -r cfa35069876b -r e1d2702eea84 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Sat May 09 11:46:09 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Sat May 09 11:57:40 2015 +0300 @@ -36,10 +36,10 @@ while (*ip != NULL) { token = *ip; ret = fts_filter_filter(filter, &token, &error); - test_assert(ret >= 0); - if (ret == 0) + if (ret <= 0) { + test_assert(ret == 0); test_assert(*op == NULL); - else { + } else { test_assert(*op != NULL); test_assert(strcmp(*ip, token) == 0); } @@ -80,10 +80,10 @@ while (*ip != NULL) { token = *ip; ret = fts_filter_filter(filter, &token, &error); - test_assert(ret >= 0); - if (ret == 0) + if (ret <= 0) { + test_assert(ret == 0); test_assert(*op == NULL); - else { + } else { test_assert(*op != NULL); test_assert(strcmp(*ip, token) == 0); } @@ -101,9 +101,10 @@ while (*ip != NULL) { token = *ip; ret = fts_filter_filter(filter, &token, &error); - if (ret == 0) + if (ret <= 0) { + test_assert(ret == 0); test_assert(*op == NULL); - else { + } else { test_assert(*op != NULL); test_assert(strcmp(*ip, token) == 0); } @@ -144,10 +145,10 @@ while (*ip != NULL) { token = *ip; ret = fts_filter_filter(filter, &token, &error); - test_assert(ret >= 0); - if (ret == 0) + if (ret <= 0) { + test_assert(ret == 0); test_assert(*op == NULL); - else { + } else { test_assert(*op != NULL); test_assert(strcmp(*ip, token) == 0); } @@ -292,6 +293,7 @@ for (tpp=tokens; *tpp != NULL; tpp++) { token = *tpp; ret = fts_filter_filter(stemmer, &token, &error); + test_assert(ret >= 0); if (ret == 0) test_assert(*bpp == NULL); else { @@ -520,9 +522,10 @@ for (tpp = tokens; *tpp != NULL; tpp++) { token = *tpp; ret = fts_filter_filter(stemmer, &token, &error); - if (ret == 0) + if (ret <= 0) { + test_assert(ret == 0); test_assert(*bpp == NULL); - else { + } else { test_assert(*bpp != NULL); test_assert(strcasecmp(*bpp, token) == 0); } From dovecot at dovecot.org Sat May 9 09:30:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 09:30:46 +0000 Subject: dovecot-2.2: lib-fts: Assume that fts_language.name is always lo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/92ee245b1406 changeset: 18564:92ee245b1406 user: Timo Sirainen date: Sat May 09 12:04:56 2015 +0300 description: lib-fts: Assume that fts_language.name is always lowercased and has two letters. Anything else just unnecessarily complicates the code. diffstat: src/lib-fts/fts-filter-stemmer-snowball.c | 5 ++--- src/lib-fts/fts-filter-stopwords.c | 2 +- src/lib-fts/fts-language.h | 1 + src/lib-fts/test-fts-filter.c | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diffs (84 lines): diff -r e1d2702eea84 -r 92ee245b1406 src/lib-fts/fts-filter-stemmer-snowball.c --- a/src/lib-fts/fts-filter-stemmer-snowball.c Sat May 09 11:57:40 2015 +0300 +++ b/src/lib-fts/fts-filter-stemmer-snowball.c Sat May 09 12:04:56 2015 +0300 @@ -19,8 +19,7 @@ static bool fts_filter_stemmer_snowball_supports(const struct fts_language *lang) { - struct sb_stemmer *stemmer = sb_stemmer_new(t_str_lcase(lang->name), - NULL); + struct sb_stemmer *stemmer = sb_stemmer_new(lang->name, NULL); if (stemmer != NULL) { sb_stemmer_delete(stemmer); return TRUE; @@ -59,7 +58,7 @@ sp->pool = pp; sp->filter = *fts_filter_stemmer_snowball; sp->lang = p_malloc(sp->pool, sizeof(struct fts_language)); - sp->lang->name = str_lcase(p_strdup(sp->pool, lang->name)); + sp->lang->name = p_strdup(sp->pool, lang->name); *filter_r = &sp->filter; return 0; } diff -r e1d2702eea84 -r 92ee245b1406 src/lib-fts/fts-filter-stopwords.c --- a/src/lib-fts/fts-filter-stopwords.c Sat May 09 11:57:40 2015 +0300 +++ b/src/lib-fts/fts-filter-stopwords.c Sat May 09 12:04:56 2015 +0300 @@ -107,7 +107,7 @@ sp->filter = *fts_filter_stopwords; sp->pool = pp; sp->lang = p_malloc(sp->pool, sizeof(struct fts_language)); - sp->lang->name = str_lcase(p_strdup(sp->pool, lang->name)); + sp->lang->name = p_strdup(sp->pool, lang->name); if (dir != NULL) sp->stopwords_dir = p_strdup(pp, dir); else diff -r e1d2702eea84 -r 92ee245b1406 src/lib-fts/fts-language.h --- a/src/lib-fts/fts-language.h Sat May 09 11:57:40 2015 +0300 +++ b/src/lib-fts/fts-language.h Sat May 09 12:04:56 2015 +0300 @@ -13,6 +13,7 @@ }; struct fts_language { + /* Two-letter language name lowercased, e.g. "en" */ const char *name; }; ARRAY_DEFINE_TYPE(fts_language, const struct fts_language *); diff -r e1d2702eea84 -r 92ee245b1406 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Sat May 09 11:57:40 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Sat May 09 12:04:56 2015 +0300 @@ -186,7 +186,7 @@ const struct fts_filter *filter_class; struct fts_filter *stemmer; const char *error; - struct fts_language language = { .name = "EN" }; + struct fts_language language = { .name = "en" }; const char *token = NULL; const char * const tokens[] = { "dries" ,"friendlies", "All", "human", "beings", "are", @@ -226,7 +226,7 @@ const struct fts_filter *filter_class; struct fts_filter *stemmer; const char *error; - struct fts_language language = { .name = "fRench" }; + struct fts_language language = { .name = "fr" }; const char *token = NULL; const char * const tokens[] = { "Tous", "les", "\xC3\xAAtres", "humains", "naissent", @@ -262,7 +262,7 @@ struct fts_filter *stemmer; struct fts_filter *filter; const char *error; - struct fts_language language = { .name = "eN" }; + struct fts_language language = { .name = "en" }; const char *token = NULL; const char * const tokens[] = { "dries" ,"friendlies", "All", "human", "beings", "are", @@ -486,7 +486,7 @@ const char * const id_settings[] = //{"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC", NULL}; {"id", "Lower", NULL}; - struct fts_language language = { .name = "En" }; + struct fts_language language = { .name = "en" }; const char *token = NULL; const char * const tokens[] = { "dries" ,"friendlies", "All", "human", "beings", "are", From dovecot at dovecot.org Sat May 9 09:30:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 09:30:52 +0000 Subject: dovecot-2.2: lib-fts: Various fixes and cleanups to stopwords fi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/adf9eb277499 changeset: 18565:adf9eb277499 user: Timo Sirainen date: Sat May 09 12:01:42 2015 +0300 description: lib-fts: Various fixes and cleanups to stopwords filter. Most importantly words added to hash table needs to be allocated from the pool and not data stack. diffstat: src/lib-fts/fts-filter-stopwords.c | 82 ++++++++++++------------------------- 1 files changed, 27 insertions(+), 55 deletions(-) diffs (132 lines): diff -r 92ee245b1406 -r adf9eb277499 src/lib-fts/fts-filter-stopwords.c --- a/src/lib-fts/fts-filter-stopwords.c Sat May 09 12:04:56 2015 +0300 +++ b/src/lib-fts/fts-filter-stopwords.c Sat May 09 12:01:42 2015 +0300 @@ -11,9 +11,9 @@ #include "fts-filter-private.h" #define STOPWORDS_FILE_FORMAT "%s/stopwords_%s.txt" -/* TODO: Configure special characters */ -static const char stopwords_eol_comment = '|'; -static const char stopwords_comment = '#'; + +#define STOPWORDS_COMMENT_CHAR1 '|' +#define STOPWORDS_COMMENT_CHAR2 '#' struct fts_filter_stopwords { struct fts_filter filter; @@ -32,41 +32,33 @@ return TRUE; } -static int fts_filter_stopwords_read_list(struct fts_filter_stopwords *filter) +static int fts_filter_stopwords_read_list(struct fts_filter_stopwords *filter, + const char **error_r) { struct istream *input; - const char *line; - const char **words; - const char *list_path = NULL; + const char *line, **words, *path; int ret = 0; - list_path = t_strdup_printf(STOPWORDS_FILE_FORMAT, - filter->stopwords_dir, filter->lang->name); + path = t_strdup_printf(STOPWORDS_FILE_FORMAT, + filter->stopwords_dir, filter->lang->name); - input = i_stream_create_file(list_path, IO_BLOCK_SIZE); - while ((line = i_stream_read_next_line(input)) != NULL) { + input = i_stream_create_file(path, IO_BLOCK_SIZE); + while ((line = i_stream_read_next_line(input)) != NULL) T_BEGIN { + line = t_strcut(line, STOPWORDS_COMMENT_CHAR1); + line = t_strcut(line, STOPWORDS_COMMENT_CHAR2); - if (uni_utf8_strlen(line) < 1) - continue; - if (strchr(line, stopwords_comment) != NULL) - continue; /* TODO: support eol hashed comments */ - if (strchr(line, stopwords_eol_comment)!= NULL) { - line = t_strcut(line, stopwords_eol_comment); - if (line == NULL || strcmp(line, "") == 0) - continue; + words = t_strsplit_spaces(line, " \t"); + for (; *words != NULL; words++) { + const char *word = p_strdup(filter->pool, *words); + hash_table_insert(filter->stopwords, word, word); } - words = t_strsplit_spaces(line, " \t"); - while (*words != NULL) { - hash_table_insert(filter->stopwords, *words, *words); - words++; - } + } T_END; + + if (input->stream_errno != 0) { + *error_r = t_strdup_printf("Failed to read stopword list %s: %s", + path, i_stream_get_error(input)); + ret = -1; } - /* - TODO: How to detect non-existing file? - TODO: istream error handling and reporting (i_error()?). - */ - if (input->stream_errno != 0) - ret = -1; i_stream_destroy(&input); return ret; } @@ -74,10 +66,10 @@ static void fts_filter_stopwords_destroy(struct fts_filter *filter) { struct fts_filter_stopwords *sp = (struct fts_filter_stopwords *)filter; + if (hash_table_is_created(sp->stopwords)) hash_table_destroy(&sp->stopwords); pool_unref(&sp->pool); - return; } static int @@ -117,38 +109,18 @@ } static int -fts_filter_stopwords_create_stopwords(struct fts_filter_stopwords *sp, - const char **error_r) -{ - int ret; - - hash_table_create(&sp->stopwords, sp->pool, 0, str_hash, strcmp); - ret = fts_filter_stopwords_read_list(sp); - if (ret < 0) { - *error_r = t_strdup_printf("Failed to read stopword list %s", - sp->stopwords_dir); - } - return ret; -} - -static int fts_filter_stopwords_filter(struct fts_filter *filter, const char **token, const char **error_r) { - const char *stopword; struct fts_filter_stopwords *sp = (struct fts_filter_stopwords *) filter; - if (!hash_table_is_created(sp->stopwords)) - if (fts_filter_stopwords_create_stopwords(sp, error_r) < 0) + if (!hash_table_is_created(sp->stopwords)) { + hash_table_create(&sp->stopwords, sp->pool, 0, str_hash, strcmp); + if (fts_filter_stopwords_read_list(sp, error_r) < 0) return -1; - stopword = hash_table_lookup(sp->stopwords, *token); - if (stopword != NULL) { - *token = NULL; - return 0; } - else - return 1; + return hash_table_lookup(sp->stopwords, *token) == NULL ? 1 : 0; } const struct fts_filter_vfuncs stopwords_filter_vfuncs = { From dovecot at dovecot.org Sat May 9 09:34:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 09:34:40 +0000 Subject: dovecot-2.2: lib-fts: Removed unnecessary code from fts-address-... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b9f85e125639 changeset: 18566:b9f85e125639 user: Timo Sirainen date: Sat May 09 12:32:46 2015 +0300 description: lib-fts: Removed unnecessary code from fts-address-tokenizer. chars_after_at() was only used to check if local-part was empty, but it was checked at a state where it never could have been empty. diffstat: src/lib-fts/fts-tokenizer-address.c | 21 ++------------------- src/lib-fts/test-fts-tokenizer.c | 16 ++++++++-------- 2 files changed, 10 insertions(+), 27 deletions(-) diffs (110 lines): diff -r adf9eb277499 -r b9f85e125639 src/lib-fts/fts-tokenizer-address.c --- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 12:01:42 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 12:32:46 2015 +0300 @@ -199,22 +199,6 @@ return EMAIL_ADDRESS_PARSER_STATE_NONE; } -/* TODO: might be nice if error was -1, but that requires a _r - param */ -static size_t chars_after_at(struct email_address_fts_tokenizer *tok) -{ - - size_t len = 0; - const unsigned char *at; - const unsigned char *str; - - str = buffer_get_data(tok->last_word, &len); - at = memchr(str, '@', len); - if (at == NULL) - return 0; - return at - str; -} - /* TODO: - allow address literals - reject "@..." @@ -234,7 +218,7 @@ } /* A complete domain name */ if ((pos > 1 && pos < size) || /* non-atext after atext in this data*/ - (pos < size && chars_after_at(tok) > 0)) { /* non-atext after previous atext */ + pos < size) { /* non-atext after previous atext */ str_append_n(tok->last_word, data, pos); *skip_r = pos; return EMAIL_ADDRESS_PARSER_STATE_COMPLETE; @@ -281,8 +265,7 @@ if (!tok->no_parent && str_len(tok->parent_data) > 0) return fts_tokenizer_address_parent_data(tok, token_r); - if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN - && chars_after_at(tok) > 0) + if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN) return fts_tokenizer_address_current_token(tok, token_r); } diff -r adf9eb277499 -r b9f85e125639 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Sat May 09 12:01:42 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Sat May 09 12:32:46 2015 +0300 @@ -308,7 +308,7 @@ static void test_fts_tokenizer_char_address_only(void) { static const unsigned char input[] = - "abc at example.com, " + "@invalid abc at example.com, " "Bar Baz , " "foo at domain"; static const char *const expected_output[] = { @@ -346,7 +346,7 @@ static void test_fts_tokenizer_rand_address_only(void) { static const unsigned char input[] = - "Abc Dfg , " + "@invalid Abc Dfg , " "Foo Bar (comment)foo.bar at host.example.org foo "; static const char *const expected_output[] = { @@ -388,11 +388,11 @@ static void test_fts_tokenizer_address_char(void) { static const unsigned char input[] = - "abc at example.com, " + "@invalid abc at example.com, " "Bar Baz , " "foo at domain"; static const char *const expected_output[] = { - "abc", "example", "com", "abc at example.com", "Bar", "Baz", + "invalid", "abc", "example", "com", "abc at example.com", "Bar", "Baz", "bar", "example", "org", "bar at example.org", "foo", "domain", "foo at domain", NULL }; @@ -430,13 +430,13 @@ static void test_fts_tokenizer_address_line(void) { static const char *const input[] = { - "abc at example.com, ", + "@invalid abc at example.com, ", "Bar Baz , ", "foo at domain, ", "foo at domain Bar Baz , " }; static const char *const expected_output[] = { - "abc", "example", "com", "abc at example.com", "Bar", "Baz", + "invalid", "abc", "example", "com", "abc at example.com", "Bar", "Baz", "bar", "example", "org", "bar at example.org", "foo", "domain", "foo at domain", "foo", "domain", "foo at domain", "Bar", "Baz", @@ -477,11 +477,11 @@ static void test_fts_tokenizer_address_rand(void) { static const unsigned char input[] = - "abc at example.com, " + "@invalid abc at example.com, " "Bar Baz , " "foo at domain"; static const char *const expected_output[] = { - "abc", "example", "com", "abc at example.com", "Bar", "Baz", + "invalid", "abc", "example", "com", "abc at example.com", "Bar", "Baz", "bar", "example", "org", "bar at example.org", "foo", "domain", "foo at domain", NULL }; From dovecot at dovecot.org Sat May 9 09:41:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 09:41:16 +0000 Subject: dovecot-2.2: lib-fts: Don't treat empty domains as valid addresses. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bcfe4c592427 changeset: 18567:bcfe4c592427 user: Timo Sirainen date: Sat May 09 12:39:21 2015 +0300 description: lib-fts: Don't treat empty domains as valid addresses. This actually pretty much reverts the previous patch and just fixes the original code's chars_after_at() to work correctly. Also renamed the function to make it a bit clearer what was intended. diffstat: src/lib-fts/fts-tokenizer-address.c | 14 ++++++++++++-- src/lib-fts/test-fts-tokenizer.c | 16 ++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diffs (103 lines): diff -r b9f85e125639 -r bcfe4c592427 src/lib-fts/fts-tokenizer-address.c --- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 12:32:46 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 12:39:21 2015 +0300 @@ -199,6 +199,15 @@ return EMAIL_ADDRESS_PARSER_STATE_NONE; } +static bool domain_is_empty(struct email_address_fts_tokenizer *tok) +{ + const char *p, *str = str_c(tok->last_word); + + if ((p = strchr(str, '@')) == NULL) + return TRUE; + return p[1] == '\0'; +} + /* TODO: - allow address literals - reject "@..." @@ -218,7 +227,7 @@ } /* A complete domain name */ if ((pos > 1 && pos < size) || /* non-atext after atext in this data*/ - pos < size) { /* non-atext after previous atext */ + (pos < size && !domain_is_empty(tok))) { /* non-atext after previous atext */ str_append_n(tok->last_word, data, pos); *skip_r = pos; return EMAIL_ADDRESS_PARSER_STATE_COMPLETE; @@ -265,7 +274,8 @@ if (!tok->no_parent && str_len(tok->parent_data) > 0) return fts_tokenizer_address_parent_data(tok, token_r); - if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN) + if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN && + !domain_is_empty(tok)) return fts_tokenizer_address_current_token(tok, token_r); } diff -r b9f85e125639 -r bcfe4c592427 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Sat May 09 12:32:46 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Sat May 09 12:39:21 2015 +0300 @@ -308,7 +308,7 @@ static void test_fts_tokenizer_char_address_only(void) { static const unsigned char input[] = - "@invalid abc at example.com, " + "@invalid invalid@ abc at example.com, " "Bar Baz , " "foo at domain"; static const char *const expected_output[] = { @@ -346,7 +346,7 @@ static void test_fts_tokenizer_rand_address_only(void) { static const unsigned char input[] = - "@invalid Abc Dfg , " + "@invalid invalid@ Abc Dfg , " "Foo Bar (comment)foo.bar at host.example.org foo "; static const char *const expected_output[] = { @@ -388,11 +388,11 @@ static void test_fts_tokenizer_address_char(void) { static const unsigned char input[] = - "@invalid abc at example.com, " + "@invalid invalid@ abc at example.com, " "Bar Baz , " "foo at domain"; static const char *const expected_output[] = { - "invalid", "abc", "example", "com", "abc at example.com", "Bar", "Baz", + "invalid", "invalid", "abc", "example", "com", "abc at example.com", "Bar", "Baz", "bar", "example", "org", "bar at example.org", "foo", "domain", "foo at domain", NULL }; @@ -430,13 +430,13 @@ static void test_fts_tokenizer_address_line(void) { static const char *const input[] = { - "@invalid abc at example.com, ", + "@invalid invalid@ abc at example.com, ", "Bar Baz , ", "foo at domain, ", "foo at domain Bar Baz , " }; static const char *const expected_output[] = { - "invalid", "abc", "example", "com", "abc at example.com", "Bar", "Baz", + "invalid", "invalid", "abc", "example", "com", "abc at example.com", "Bar", "Baz", "bar", "example", "org", "bar at example.org", "foo", "domain", "foo at domain", "foo", "domain", "foo at domain", "Bar", "Baz", @@ -477,11 +477,11 @@ static void test_fts_tokenizer_address_rand(void) { static const unsigned char input[] = - "@invalid abc at example.com, " + "@invalid invalid@ abc at example.com, " "Bar Baz , " "foo at domain"; static const char *const expected_output[] = { - "invalid", "abc", "example", "com", "abc at example.com", "Bar", "Baz", + "invalid", "invalid", "abc", "example", "com", "abc at example.com", "Bar", "Baz", "bar", "example", "org", "bar at example.org", "foo", "domain", "foo at domain", NULL }; From dovecot at dovecot.org Sat May 9 09:43:53 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 09:43:53 +0000 Subject: dovecot-2.2: lib-fts: Use rfc822-parser in fts-tokenizer-address... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f4b2aa500cde changeset: 18568:f4b2aa500cde user: Timo Sirainen date: Sat May 09 12:41:59 2015 +0300 description: lib-fts: Use rfc822-parser in fts-tokenizer-address instead of duplicating its code. diffstat: src/lib-fts/Makefile.am | 5 +- src/lib-fts/fts-tokenizer-address.c | 58 ++---------------------------------- 2 files changed, 7 insertions(+), 56 deletions(-) diffs (99 lines): diff -r bcfe4c592427 -r f4b2aa500cde src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Sat May 09 12:39:21 2015 +0300 +++ b/src/lib-fts/Makefile.am Sat May 09 12:41:59 2015 +0300 @@ -3,6 +3,7 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-mail \ $(LIBEXTTEXTCAT_CFLAGS) \ $(LIBICU_CFLAGS) \ -DUDHRDIR=\""$(top_srcdir)/src/lib-fts"\" \ @@ -101,8 +102,8 @@ endif test_fts_tokenizer_SOURCES = test-fts-tokenizer.c -test_fts_tokenizer_LDADD = fts-tokenizer.lo fts-tokenizer-generic.lo fts-tokenizer-address.lo $(test_libs) -test_fts_tokenizer_DEPENDENCIES = $(test_deps) +test_fts_tokenizer_LDADD = fts-tokenizer.lo fts-tokenizer-generic.lo fts-tokenizer-address.lo ../lib-mail/libmail.la $(test_libs) +test_fts_tokenizer_DEPENDENCIES = ../lib-mail/libmail.la $(test_deps) check: check-am check-test check-test: all-am diff -r bcfe4c592427 -r f4b2aa500cde src/lib-fts/fts-tokenizer-address.c --- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 12:39:21 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 12:41:59 2015 +0300 @@ -3,8 +3,12 @@ #include "lib.h" #include "str.h" #include "buffer.h" +#include "rfc822-parser.h" #include "fts-tokenizer-private.h" +#define IS_DTEXT(c) \ + (rfc822_atext_chars[(int)(unsigned char)(c)] == 2) + #define FTS_DEFAULT_NO_PARENT FALSE #define FTS_DEFAULT_SEARCH FALSE @@ -25,60 +29,6 @@ bool search; }; -/* - Extracted from core rfc822-parser.c - - atext = ALPHA / DIGIT / ; Any character except controls, - "!" / "#" / ; SP, and specials. - "$" / "%" / ; Used for atoms - "&" / "'" / - "*" / "+" / - "-" / "/" / - "=" / "?" / - "^" / "_" / - "`" / "{" / - "|" / "}" / - "~" - - MIME: - - token := 1* - tspecials := "(" / ")" / "<" / ">" / "@" / - "," / ";" / ":" / "\" / <"> - "/" / "[" / "]" / "?" / "=" - - So token is same as dot-atom, except stops also at '/', '?' and '='. -*/ - -/* atext chars are marked with 1, alpha and digits with 2, - atext-but-mime-tspecials with 4 */ -unsigned char rfc822_atext_chars[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0-15 */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 16-31 */ - 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 4, /* 32-47 */ - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 4, 0, 4, /* 48-63 */ - 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 64-79 */ - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 1, 1, /* 80-95 */ - 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 96-111 */ - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 0, /* 112-127 */ - - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 -}; - -#define IS_ATEXT(c) \ - (rfc822_atext_chars[(int)(unsigned char)(c)] != 0) -#define IS_DTEXT(c) \ - (rfc822_atext_chars[(int)(unsigned char)(c)] == 2) - - static int fts_tokenizer_email_address_create(const char *const *settings, struct fts_tokenizer **tokenizer_r, From dovecot at dovecot.org Sat May 9 09:50:06 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 09:50:06 +0000 Subject: dovecot-2.2: lib-fts: Removed explicit no_parent setting from ft... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7c1fe66e8855 changeset: 18569:7c1fe66e8855 user: Timo Sirainen date: Sat May 09 12:48:13 2015 +0300 description: lib-fts: Removed explicit no_parent setting from fts-address-tokenizer It was used only by unit tests and we can check it without the setting as well. diffstat: src/lib-fts/fts-tokenizer-address.c | 13 ++++--------- src/lib-fts/test-fts-tokenizer.c | 9 +++------ 2 files changed, 7 insertions(+), 15 deletions(-) diffs (117 lines): diff -r f4b2aa500cde -r 7c1fe66e8855 src/lib-fts/fts-tokenizer-address.c --- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 12:41:59 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 12:48:13 2015 +0300 @@ -25,7 +25,6 @@ string_t *last_word; string_t *parent_data; /* Copy of input data between tokens. TODO: could be buffer_t maybe */ - bool no_parent; bool search; }; @@ -35,16 +34,13 @@ const char **error_r) { struct email_address_fts_tokenizer *tok; - bool no_parent = FTS_DEFAULT_NO_PARENT; bool search = FTS_DEFAULT_SEARCH; unsigned int i; for (i = 0; settings[i] != NULL; i += 2) { const char *key = settings[i]; - if (strcmp(key, "no_parent") == 0) { - no_parent = TRUE; - }else if (strcmp(key, "search") == 0) { + if (strcmp(key, "search") == 0) { search = TRUE; } else { *error_r = t_strdup_printf("Unknown setting: %s", key); @@ -56,7 +52,6 @@ tok->tokenizer = *fts_tokenizer_email_address; tok->last_word = str_new(default_pool, 128); tok->parent_data = str_new(default_pool, 128); - tok->no_parent = no_parent; tok->search = search; *tokenizer_r = &tok->tokenizer; return 0; @@ -198,7 +193,7 @@ fts_tokenizer_address_update_parent(struct email_address_fts_tokenizer *tok, const unsigned char *data, size_t size) { - if (!tok->no_parent) + if (tok->tokenizer.parent != NULL) str_append_n(tok->parent_data, data, size); } static int @@ -221,7 +216,7 @@ /* end of data, output lingering tokens. first the parents data, then possibly our token, if complete enough */ if (size == 0) { - if (!tok->no_parent && str_len(tok->parent_data) > 0) + if (tok->tokenizer.parent != NULL && str_len(tok->parent_data) > 0) return fts_tokenizer_address_parent_data(tok, token_r); if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN && @@ -276,7 +271,7 @@ *skip_r = pos + local_skip; fts_tokenizer_address_update_parent(tok, data+pos, local_skip); - if (!tok->no_parent) + if (tok->tokenizer.parent != NULL) return fts_tokenizer_address_parent_data(tok, token_r); else { return fts_tokenizer_address_current_token(tok, token_r); diff -r f4b2aa500cde -r 7c1fe66e8855 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Sat May 09 12:41:59 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Sat May 09 12:48:13 2015 +0300 @@ -275,7 +275,6 @@ "abc at example.com", "bar at example.org", "foo at domain", "foo at domain", "bar at example.org", NULL }; - const char *const settings[] = {"no_parent", "foo", NULL}; struct fts_tokenizer *tok; const char * const *eopp = expected_output; const char *token, *error; @@ -285,7 +284,7 @@ test_begin("fts tokenizer email address only, input one line at a time"); fts_tokenizer_register(fts_tokenizer_email_address); - test_assert(fts_tokenizer_create(fts_tokenizer_email_address, NULL, settings, &tok, &error) == 0); + test_assert(fts_tokenizer_create(fts_tokenizer_email_address, NULL, NULL, &tok, &error) == 0); for (i = 0; i <= N_ELEMENTS(input);) { ret = i < N_ELEMENTS(input) ? @@ -315,7 +314,6 @@ "abc at example.com", "bar at example.org", "foo at domain", NULL }; - const char *const settings[] = {"no_parent", "0", NULL}; struct fts_tokenizer *tok; const char * const *eopp = expected_output; const char *token, *error; @@ -324,7 +322,7 @@ test_begin("fts tokenizer email address only, input one character at a time"); fts_tokenizer_register(fts_tokenizer_email_address); - test_assert(fts_tokenizer_create(fts_tokenizer_email_address, NULL, settings, &tok, &error) == 0); + test_assert(fts_tokenizer_create(fts_tokenizer_email_address, NULL, NULL, &tok, &error) == 0); for (i = 0; i <= sizeof(input)-1; ) { ret = i < sizeof(input)-1 ? @@ -357,14 +355,13 @@ struct fts_tokenizer *tok; const char * const *eopp = expected_output; const char *token, *error; - const char *const settings[] = {"no_parent", "abc", NULL}; unsigned int i, step, step_max = 10; int ret; test_begin("fts tokenizer email address, input random length"); fts_tokenizer_register(fts_tokenizer_email_address); test_assert(fts_tokenizer_create(fts_tokenizer_email_address, NULL, - settings, &tok, &error) == 0); + NULL, &tok, &error) == 0); step = rand() % step_max + 1; for (i = 0; i <= sizeof(input)-1; ) { ret = i < sizeof(input)-1 ? From dovecot at dovecot.org Sat May 9 09:52:49 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 09:52:49 +0000 Subject: dovecot-2.2: lib-fts: Minor fts-tokenizer-address cleanups Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ac6ae1b9f5c4 changeset: 18570:ac6ae1b9f5c4 user: Timo Sirainen date: Sat May 09 12:50:11 2015 +0300 description: lib-fts: Minor fts-tokenizer-address cleanups diffstat: src/lib-fts/fts-tokenizer-address.c | 28 ++++++++-------------------- 1 files changed, 8 insertions(+), 20 deletions(-) diffs (88 lines): diff -r 7c1fe66e8855 -r ac6ae1b9f5c4 src/lib-fts/fts-tokenizer-address.c --- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 12:48:13 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 12:50:11 2015 +0300 @@ -23,8 +23,7 @@ struct fts_tokenizer tokenizer; enum email_address_parser_state state; string_t *last_word; - string_t *parent_data; /* Copy of input data between tokens. - TODO: could be buffer_t maybe */ + string_t *parent_data; /* Copy of input data between tokens. */ bool search; }; @@ -94,34 +93,27 @@ Returns size that can be skipped. */ static size_t skip_nonlocal_part(const unsigned char *data, size_t size) { - const unsigned char *p = data; size_t skip = 0; /* Yes, a dot can start an address. De facto before de jure. */ - while ( skip < size && (!IS_ATEXT(*p) && *p != '.')) { + while (skip < size && (!IS_ATEXT(data[skip]) && data[skip] != '.')) skip++; - p++; - } return skip; } -/* TODO: - - DONT dereference *p past size! -*/ static enum email_address_parser_state fts_tokenizer_email_address_parse_local(struct email_address_fts_tokenizer *tok, const unsigned char *data, size_t size, size_t *skip_r) { size_t pos = 0; - const unsigned char *p = data; bool at = FALSE; - while (pos < size && (IS_ATEXT(*p) || (*p == '@' || *p == '.'))) { - if (*p == '@') + while (pos < size && (IS_ATEXT(data[pos]) || + data[pos] == '@' || data[pos] == '.')) { + if (data[pos] == '@') at = TRUE; pos++; - p++; if (at) break; } @@ -133,13 +125,13 @@ } /* localpart, @ not included yet */ - if (pos > 0 && (IS_ATEXT(*(p-1)) || *(p-1) == '.')) { + if (pos > 0 && (IS_ATEXT(data[pos-1]) || data[pos-1] == '.')) { str_append_n(tok->last_word, data, pos); *skip_r = pos; return EMAIL_ADDRESS_PARSER_STATE_LOCALPART; } /* not a localpart. skip past rest of no-good chars. */ - pos += skip_nonlocal_part(p, size - pos); + pos += skip_nonlocal_part(data+pos, size - pos); *skip_r = pos; return EMAIL_ADDRESS_PARSER_STATE_NONE; } @@ -153,11 +145,6 @@ return p[1] == '\0'; } -/* TODO: - - allow address literals - - reject "@..." - - reject "@.host.tld" -*/ static enum email_address_parser_state fts_tokenizer_email_address_parse_domain(struct email_address_fts_tokenizer *tok, const unsigned char *data, size_t size, @@ -196,6 +183,7 @@ if (tok->tokenizer.parent != NULL) str_append_n(tok->parent_data, data, size); } + static int fts_tokenizer_email_address_next(struct fts_tokenizer *_tok, const unsigned char *data, size_t size, From dovecot at dovecot.org Sat May 9 09:52:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 09:52:55 +0000 Subject: dovecot-2.2: lib-fts: Compiler warning fix when building without... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3363fadc2e13 changeset: 18571:3363fadc2e13 user: Timo Sirainen date: Sat May 09 12:50:54 2015 +0300 description: lib-fts: Compiler warning fix when building without libicu diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r ac6ae1b9f5c4 -r 3363fadc2e13 src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 12:50:11 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 12:50:54 2015 +0300 @@ -263,7 +263,7 @@ const char **token ATTR_UNUSED, const char **error_r ATTR_UNUSED) { - return NULL; + return -1; } static void From dovecot at dovecot.org Sat May 9 10:03:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 10:03:50 +0000 Subject: dovecot-2.2: fts: If filtering fails during search arg expansion... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/07597666aa29 changeset: 18572:07597666aa29 user: Timo Sirainen date: Sat May 09 13:01:45 2015 +0300 description: fts: If filtering fails during search arg expansion, fail the fts search. diffstat: src/plugins/fts/fts-search-args.c | 56 +++++++++++++++++++++++++------------- 1 files changed, 37 insertions(+), 19 deletions(-) diffs (131 lines): diff -r 3363fadc2e13 -r 07597666aa29 src/plugins/fts/fts-search-args.c --- a/src/plugins/fts/fts-search-args.c Sat May 09 12:50:54 2015 +0300 +++ b/src/plugins/fts/fts-search-args.c Sat May 09 13:01:45 2015 +0300 @@ -53,7 +53,7 @@ return or_arg; } -static void +static int fts_backend_dovecot_expand_lang_tokens(const ARRAY_TYPE(fts_user_language) *languages, pool_t pool, struct mail_search_arg *parent_arg, @@ -75,14 +75,15 @@ /* add the word filtered */ array_foreach(languages, langp) { token2 = t_strdup(token); - if ((*langp)->filter != NULL) - ret = fts_filter_filter((*langp)->filter, &token2, &error); + ret = (*langp)->filter == NULL ? 1 : + fts_filter_filter((*langp)->filter, &token2, &error); if (ret > 0) { token2 = t_strdup(token2); array_append(&tokens, &token2, 1); + } else if (ret < 0) { + i_error("fts: Couldn't create search tokens: %s", error); + return -1; } - else if (ret < 0) - i_error("fts: Couldn't create search tokens: %s", error); } array_sort(&tokens, i_strcmp_p); strings_deduplicate(&tokens); @@ -90,10 +91,11 @@ arg = fts_search_arg_create_or(orig_arg, pool, &tokens); arg->next = parent_arg->value.subargs; parent_arg->value.subargs = arg; + return 0; } -static void fts_search_arg_expand(struct fts_backend *backend, pool_t pool, - struct mail_search_arg **argp) +static int fts_search_arg_expand(struct fts_backend *backend, pool_t pool, + struct mail_search_arg **argp) { const ARRAY_TYPE(fts_user_language) *languages; struct mail_search_arg *and_arg, *orig_arg = *argp; @@ -111,33 +113,39 @@ and_arg->type = SEARCH_SUB; and_arg->match_not = orig_arg->match_not; and_arg->next = orig_arg->next; - *argp = and_arg; while (fts_tokenizer_next(tokenizer, (const void *)orig_token, orig_token_len, &token) > 0) { - fts_backend_dovecot_expand_lang_tokens(languages, pool, and_arg, - orig_arg, orig_token, - token); + if (fts_backend_dovecot_expand_lang_tokens(languages, pool, and_arg, + orig_arg, orig_token, + token) < 0) + return -1; } while (fts_tokenizer_next(tokenizer, NULL, 0, &token) > 0) { - fts_backend_dovecot_expand_lang_tokens(languages, pool, and_arg, - orig_arg, orig_token, - token); + if (fts_backend_dovecot_expand_lang_tokens(languages, pool, and_arg, + orig_arg, orig_token, + token) < 0) + return -1; } + *argp = and_arg; + return 0; } -static void +static int fts_search_args_expand_tree(struct fts_backend *backend, pool_t pool, struct mail_search_arg **argp) { + int ret; + for (; *argp != NULL; argp = &(*argp)->next) { switch ((*argp)->type) { case SEARCH_OR: case SEARCH_SUB: case SEARCH_INTHREAD: - fts_search_args_expand_tree(backend, pool, - &(*argp)->value.subargs); + if (fts_search_args_expand_tree(backend, pool, + &(*argp)->value.subargs) < 0) + return -1; break; case SEARCH_HEADER: case SEARCH_HEADER_ADDRESS: @@ -145,22 +153,32 @@ case SEARCH_BODY: case SEARCH_TEXT: T_BEGIN { - fts_search_arg_expand(backend, pool, argp); + ret = fts_search_arg_expand(backend, pool, argp); } T_END; + if (ret < 0) + return -1; break; default: break; } } + return 0; } int fts_search_args_expand(struct fts_backend *backend, struct mail_search_args *args) { - fts_search_args_expand_tree(backend, args->pool, &args->args); + struct mail_search_arg *args_dup; + + /* duplicate the args, so if expansion fails we haven't changed + anything */ + args_dup = mail_search_arg_dup(args->pool, args->args); + if (fts_search_args_expand_tree(backend, args->pool, &args_dup) < 0) + return -1; /* we'll need to re-simplify the args if we changed anything */ args->simplified = FALSE; + args->args = args_dup; mail_search_args_simplify(args); return 0; } From dovecot at dovecot.org Sat May 9 10:17:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 10:17:37 +0000 Subject: dovecot-2.2: lib-fts: Implemented "search" parameter to fts-toke... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/60f07e741c57 changeset: 18573:60f07e741c57 user: Timo Sirainen date: Sat May 09 13:15:09 2015 +0300 description: lib-fts: Implemented "search" parameter to fts-tokenizer-address. diffstat: src/lib-fts/fts-tokenizer-address.c | 41 ++++++++++++++++++++-------------- src/lib-fts/test-fts-tokenizer.c | 43 +++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 17 deletions(-) diffs (131 lines): diff -r 07597666aa29 -r 60f07e741c57 src/lib-fts/fts-tokenizer-address.c --- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 13:01:45 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 13:15:09 2015 +0300 @@ -76,17 +76,30 @@ return 1; } -static int +static bool fts_tokenizer_address_parent_data(struct email_address_fts_tokenizer *tok, const char **token_r) { - /* TODO: search option removes address from data here. */ - if (tok->search && tok->state >= EMAIL_ADDRESS_PARSER_STATE_DOMAIN) - i_debug("Would remove current token"); + if (tok->tokenizer.parent == NULL || str_len(tok->parent_data) == 0) + return FALSE; + + if (tok->search && tok->state >= EMAIL_ADDRESS_PARSER_STATE_DOMAIN) { + /* we're searching and we want to find only the full + user at domain (not "user" and "domain"). we'll do this by + not feeding the last user at domain to parent tokenizer. */ + unsigned int parent_prefix_len = + str_len(tok->parent_data) - str_len(tok->last_word); + i_assert(str_len(tok->parent_data) >= str_len(tok->last_word) && + strcmp(str_c(tok->parent_data) + parent_prefix_len, + str_c(tok->last_word)) == 0); + str_truncate(tok->parent_data, parent_prefix_len); + if (str_len(tok->parent_data) == 0) + return FALSE; + } *token_r = t_strdup(str_c(tok->parent_data)); str_truncate(tok->parent_data, 0); - return 1; + return TRUE; } /* Used to rewind past characters that can not be the start of a new localpart. @@ -204,8 +217,8 @@ /* end of data, output lingering tokens. first the parents data, then possibly our token, if complete enough */ if (size == 0) { - if (tok->tokenizer.parent != NULL && str_len(tok->parent_data) > 0) - return fts_tokenizer_address_parent_data(tok, token_r); + if (fts_tokenizer_address_parent_data(tok, token_r)) + return 1; if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN && !domain_is_empty(tok)) @@ -254,16 +267,10 @@ break; case EMAIL_ADDRESS_PARSER_STATE_COMPLETE: - /* skip tailing non-atext */ - local_skip = skip_nonlocal_part(data+pos, size - pos); - *skip_r = pos + local_skip; - fts_tokenizer_address_update_parent(tok, data+pos, - local_skip); - if (tok->tokenizer.parent != NULL) - return fts_tokenizer_address_parent_data(tok, token_r); - else { - return fts_tokenizer_address_current_token(tok, token_r); - } + *skip_r = pos; + if (fts_tokenizer_address_parent_data(tok, token_r)) + return 1; + return fts_tokenizer_address_current_token(tok, token_r); default: i_unreached(); } diff -r 07597666aa29 -r 60f07e741c57 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Sat May 09 13:01:45 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Sat May 09 13:15:09 2015 +0300 @@ -518,6 +518,48 @@ test_end(); } +static void test_fts_tokenizer_address_search(void) +{ + static const unsigned char input[] = + "@invalid invalid@ abc at example.com, " + "Bar Baz , " + "foo at domain"; + static const char *const expected_output[] = { + "invalid", "invalid", "abc at example.com", "Bar", "Baz", + "bar at example.org", "foo at domain", NULL + }; + static const char *const settings[] = { "search", "" }; + struct fts_tokenizer *tok, *gen_tok; + const char * const *eopp = expected_output; + const char *token, *error; + unsigned int i; + int ret; + + test_begin("fts tokenizer search email address + parent, input one character at a time"); + fts_tokenizers_init(); + + test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &gen_tok, &error) == 0); + test_assert(fts_tokenizer_create(fts_tokenizer_email_address, gen_tok, settings, &tok, &error) == 0); + + for (i = 0; i <= sizeof(input)-1; ) { + ret = i < sizeof(input)-1 ? + fts_tokenizer_next(tok, &input[i], 1, &token) : + fts_tokenizer_next(tok, NULL, 0, &token); + if (ret == 0) { + i++; + continue; + } + test_assert(*eopp != NULL); + test_assert(null_strcmp(token, *eopp) == 0); + eopp++; + } + test_assert(*eopp == NULL); + fts_tokenizer_unref(&tok); + fts_tokenizer_unref(&gen_tok); + fts_tokenizers_deinit(); + test_end(); +} + int main(void) { static void (*test_functions[])(void) = { @@ -534,6 +576,7 @@ test_fts_tokenizer_address_char, test_fts_tokenizer_address_line, test_fts_tokenizer_address_rand, + test_fts_tokenizer_address_search, NULL }; From dovecot at dovecot.org Sat May 9 10:17:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 10:17:37 +0000 Subject: dovecot-2.2: lib-storage: mail_search_arg_dup() is now a public ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/eb6fbab30cf6 changeset: 18574:eb6fbab30cf6 user: Timo Sirainen date: Sat May 09 13:15:43 2015 +0300 description: lib-storage: mail_search_arg_dup() is now a public function. diffstat: src/lib-storage/mail-search.c | 4 +--- src/lib-storage/mail-search.h | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diffs (33 lines): diff -r 60f07e741c57 -r eb6fbab30cf6 src/lib-storage/mail-search.c --- a/src/lib-storage/mail-search.c Sat May 09 13:15:09 2015 +0300 +++ b/src/lib-storage/mail-search.c Sat May 09 13:15:43 2015 +0300 @@ -9,8 +9,6 @@ #include "mail-search-build.h" #include "mail-search.h" -static struct mail_search_arg * -mail_search_arg_dup(pool_t pool, const struct mail_search_arg *arg); static bool mail_search_arg_equals(const struct mail_search_arg *arg1, const struct mail_search_arg *arg2); @@ -324,7 +322,7 @@ return new_arg; } -static struct mail_search_arg * +struct mail_search_arg * mail_search_arg_dup(pool_t pool, const struct mail_search_arg *arg) { struct mail_search_arg *new_arg = NULL, **dest = &new_arg; diff -r 60f07e741c57 -r eb6fbab30cf6 src/lib-storage/mail-search.h --- a/src/lib-storage/mail-search.h Sat May 09 13:15:09 2015 +0300 +++ b/src/lib-storage/mail-search.h Sat May 09 13:15:43 2015 +0300 @@ -145,6 +145,8 @@ struct mail_search_args * mail_search_args_dup(const struct mail_search_args *args); +struct mail_search_arg * +mail_search_arg_dup(pool_t pool, const struct mail_search_arg *arg); /* Reset the results in search arguments. match_always is reset only if full_reset is TRUE. */ From dovecot at dovecot.org Sat May 9 10:33:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 10:33:13 +0000 Subject: dovecot-2.2: lib-fts: Minor unit test fix - use case sensitive c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6c284e61013f changeset: 18575:6c284e61013f user: Timo Sirainen date: Sat May 09 13:19:21 2015 +0300 description: lib-fts: Minor unit test fix - use case sensitive checks for word comparison. diffstat: src/lib-fts/test-fts-filter.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r eb6fbab30cf6 -r 6c284e61013f src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Sat May 09 13:15:43 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Sat May 09 13:19:21 2015 +0300 @@ -527,7 +527,7 @@ test_assert(*bpp == NULL); } else { test_assert(*bpp != NULL); - test_assert(strcasecmp(*bpp, token) == 0); + test_assert(strcmp(*bpp, token) == 0); } bpp++; } From dovecot at dovecot.org Sat May 9 10:33:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 10:33:13 +0000 Subject: dovecot-2.2: lib-fts: Use case-sensitive settings comparisons in... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3bcd04bf5635 changeset: 18576:3bcd04bf5635 user: Timo Sirainen date: Sat May 09 13:20:29 2015 +0300 description: lib-fts: Use case-sensitive settings comparisons in fts-tokenizer Dovecot in general doesn't allow case-insensitive settings. diffstat: src/lib-fts/fts-tokenizer-generic.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (17 lines): diff -r 6c284e61013f -r 3bcd04bf5635 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Sat May 09 13:19:21 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Sat May 09 13:20:29 2015 +0300 @@ -43,10 +43,10 @@ "Invalid maxlen setting: %s", value); return -1; } - } else if (strcasecmp(key, "algorithm") == 0) { - if (strcasecmp(value, ALGORITHM_TR29_NAME) == 0) + } else if (strcmp(key, "algorithm") == 0) { + if (strcmp(value, ALGORITHM_TR29_NAME) == 0) algo = BOUNDARY_ALGORITHM_TR29; - else if (strcasecmp(value, ALGORITHM_SIMPLE_NAME) == 0) + else if (strcmp(value, ALGORITHM_SIMPLE_NAME) == 0) ; else { *error_r = t_strdup_printf( From dovecot at dovecot.org Sat May 9 10:33:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 10:33:13 +0000 Subject: dovecot-2.2: fts: Don't crash if search arg string expands to em... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/162668a63045 changeset: 18577:162668a63045 user: Timo Sirainen date: Sat May 09 13:30:41 2015 +0300 description: fts: Don't crash if search arg string expands to empty token list. diffstat: src/plugins/fts/fts-search-args.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (16 lines): diff -r 3bcd04bf5635 -r 162668a63045 src/plugins/fts/fts-search-args.c --- a/src/plugins/fts/fts-search-args.c Sat May 09 13:20:29 2015 +0300 +++ b/src/plugins/fts/fts-search-args.c Sat May 09 13:30:41 2015 +0300 @@ -128,6 +128,12 @@ token) < 0) return -1; } + + if (and_arg->value.subargs == NULL) { + /* we couldn't parse any tokens from the input */ + and_arg->type = SEARCH_ALL; + and_arg->match_not = !and_arg->match_not; + } *argp = and_arg; return 0; } From dovecot at dovecot.org Sat May 9 10:33:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 10:33:13 +0000 Subject: dovecot-2.2: fts: When tokenizing a search word, give "search" p... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/eff53aa9cb58 changeset: 18578:eff53aa9cb58 user: Timo Sirainen date: Sat May 09 13:31:14 2015 +0300 description: fts: When tokenizing a search word, give "search" parameter to all the tokenizers. diffstat: src/lib-fts/fts-tokenizer-generic.c | 3 +++ src/plugins/fts/fts-user.c | 7 +++---- 2 files changed, 6 insertions(+), 4 deletions(-) diffs (30 lines): diff -r 162668a63045 -r eff53aa9cb58 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Sat May 09 13:30:41 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Sat May 09 13:31:14 2015 +0300 @@ -53,6 +53,9 @@ "Invalid algorithm: %s", value); return -1; } + } else if (strcmp(key, "search") == 0) { + /* tokenizing a search string - + makes no difference to us */ } else { *error_r = t_strdup_printf("Unknown setting: %s", key); return -1; diff -r 162668a63045 -r eff53aa9cb58 src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Sat May 09 13:30:41 2015 +0300 +++ b/src/plugins/fts/fts-user.c Sat May 09 13:31:14 2015 +0300 @@ -148,10 +148,9 @@ set_key = t_strdup_printf("fts_tokenizers_%s", tokenizer_set_name); str = mail_user_plugin_getenv(user, set_key); - /* If the email-address tokenizer is included in the search - tokenizer, add a setting. */ - if (search && strcmp(fts_tokenizer_name(tokenizer_class), - FTS_TOKENIZER_EMAIL_ADDRESS_NAME) == 0) { + /* tell the tokenizers that we're tokenizing a search string + (instead of tokenizing indexed data) */ + if (search) { if (str == NULL) str = "search yes"; else From dovecot at dovecot.org Sat May 9 10:54:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 10:54:31 +0000 Subject: dovecot-2.2: lib-fts: fts-tokenizer-address didn't reset state p... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/363397c3701e changeset: 18579:363397c3701e user: Timo Sirainen date: Sat May 09 13:46:37 2015 +0300 description: lib-fts: fts-tokenizer-address didn't reset state properly when input ended. diffstat: src/lib-fts/fts-tokenizer-address.c | 11 +++++++++-- src/lib-fts/test-fts-tokenizer.c | 13 +++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diffs (48 lines): diff -r eff53aa9cb58 -r 363397c3701e src/lib-fts/fts-tokenizer-address.c --- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 13:31:14 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 13:46:37 2015 +0300 @@ -217,12 +217,19 @@ /* end of data, output lingering tokens. first the parents data, then possibly our token, if complete enough */ if (size == 0) { + if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN && + domain_is_empty(tok)) { + /* user@ without domain - reset state */ + str_truncate(tok->last_word, 0); + tok->state = EMAIL_ADDRESS_PARSER_STATE_NONE; + } + if (fts_tokenizer_address_parent_data(tok, token_r)) return 1; - if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN && - !domain_is_empty(tok)) + if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN) return fts_tokenizer_address_current_token(tok, token_r); + tok->state = EMAIL_ADDRESS_PARSER_STATE_NONE; } /* 1) regular input data OR diff -r eff53aa9cb58 -r 363397c3701e src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Sat May 09 13:31:14 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Sat May 09 13:46:37 2015 +0300 @@ -554,6 +554,19 @@ eopp++; } test_assert(*eopp == NULL); + + test_assert(fts_tokenizer_next(tok, (const void *)"foo", 3, &token) == 0); + test_assert(fts_tokenizer_next(tok, NULL, 0, &token) > 0); + test_assert(fts_tokenizer_next(tok, NULL, 0, &token) == 0); + + test_assert(fts_tokenizer_next(tok, (const void *)"bar at baz", 7, &token) == 0); + test_assert(fts_tokenizer_next(tok, NULL, 0, &token) > 0); + test_assert(fts_tokenizer_next(tok, NULL, 0, &token) == 0); + + test_assert(fts_tokenizer_next(tok, (const void *)"foo@", 4, &token) == 0); + test_assert(fts_tokenizer_next(tok, NULL, 0, &token) > 0); + test_assert(fts_tokenizer_next(tok, NULL, 0, &token) == 0); + fts_tokenizer_unref(&tok); fts_tokenizer_unref(&gen_tok); fts_tokenizers_deinit(); From dovecot at dovecot.org Sat May 9 10:54:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 10:54:31 +0000 Subject: dovecot-2.2: lib-fts: Added fts_tokenizer_reset() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2dca6925bd88 changeset: 18580:2dca6925bd88 user: Timo Sirainen date: Sat May 09 13:52:37 2015 +0300 description: lib-fts: Added fts_tokenizer_reset() diffstat: src/lib-fts/fts-tokenizer-address.c | 11 +++++++++++ src/lib-fts/fts-tokenizer-generic.c | 14 ++++++++++++++ src/lib-fts/fts-tokenizer-private.h | 1 + src/lib-fts/fts-tokenizer.c | 5 +++++ src/lib-fts/fts-tokenizer.h | 4 ++++ src/lib-fts/test-fts-tokenizer.c | 19 ++++++++++++++++--- 6 files changed, 51 insertions(+), 3 deletions(-) diffs (153 lines): diff -r 363397c3701e -r 2dca6925bd88 src/lib-fts/fts-tokenizer-address.c --- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 13:46:37 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 13:52:37 2015 +0300 @@ -197,6 +197,16 @@ str_append_n(tok->parent_data, data, size); } +static void fts_tokenizer_email_address_reset(struct fts_tokenizer *_tok) +{ + struct email_address_fts_tokenizer *tok = + (struct email_address_fts_tokenizer *)_tok; + + tok->state = EMAIL_ADDRESS_PARSER_STATE_NONE; + str_truncate(tok->last_word, 0); + str_truncate(tok->parent_data, 0); +} + static int fts_tokenizer_email_address_next(struct fts_tokenizer *_tok, const unsigned char *data, size_t size, @@ -290,6 +300,7 @@ static const struct fts_tokenizer_vfuncs email_address_tokenizer_vfuncs = { fts_tokenizer_email_address_create, fts_tokenizer_email_address_destroy, + fts_tokenizer_email_address_reset, fts_tokenizer_email_address_next }; diff -r 363397c3701e -r 2dca6925bd88 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Sat May 09 13:46:37 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Sat May 09 13:52:37 2015 +0300 @@ -137,6 +137,17 @@ return is_word_break(c); } +static void fts_tokenizer_generic_reset(struct fts_tokenizer *_tok) +{ + struct generic_fts_tokenizer *tok = + (struct generic_fts_tokenizer *)_tok; + + tok->prev_letter = LETTER_TYPE_NONE; + tok->prev_prev_letter = LETTER_TYPE_NONE; + tok->last_size = 0; + buffer_set_used_size(tok->token, 0); +} + static int fts_tokenizer_generic_next_simple(struct fts_tokenizer *_tok, const unsigned char *data, size_t size, @@ -580,6 +591,7 @@ static const struct fts_tokenizer_vfuncs generic_tokenizer_vfuncs = { fts_tokenizer_generic_create, fts_tokenizer_generic_destroy, + fts_tokenizer_generic_reset, fts_tokenizer_generic_next }; @@ -592,10 +604,12 @@ const struct fts_tokenizer_vfuncs generic_tokenizer_vfuncs_simple = { fts_tokenizer_generic_create, fts_tokenizer_generic_destroy, + fts_tokenizer_generic_reset, fts_tokenizer_generic_next_simple }; const struct fts_tokenizer_vfuncs generic_tokenizer_vfuncs_tr29 = { fts_tokenizer_generic_create, fts_tokenizer_generic_destroy, + fts_tokenizer_generic_reset, fts_tokenizer_generic_next_tr29 }; diff -r 363397c3701e -r 2dca6925bd88 src/lib-fts/fts-tokenizer-private.h --- a/src/lib-fts/fts-tokenizer-private.h Sat May 09 13:46:37 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-private.h Sat May 09 13:52:37 2015 +0300 @@ -10,6 +10,7 @@ struct fts_tokenizer **tokenizer_r, const char **error_r); void (*destroy)(struct fts_tokenizer *tok); + void (*reset)(struct fts_tokenizer *tok); int (*next)(struct fts_tokenizer *tok, const unsigned char *data, size_t size, size_t *skip_r, const char **token_r); }; diff -r 363397c3701e -r 2dca6925bd88 src/lib-fts/fts-tokenizer.c --- a/src/lib-fts/fts-tokenizer.c Sat May 09 13:46:37 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.c Sat May 09 13:52:37 2015 +0300 @@ -157,6 +157,11 @@ return ret; } +void fts_tokenizer_reset(struct fts_tokenizer *tok) +{ + tok->v->reset(tok); +} + int fts_tokenizer_next(struct fts_tokenizer *tok, const unsigned char *data, size_t size, const char **token_r) diff -r 363397c3701e -r 2dca6925bd88 src/lib-fts/fts-tokenizer.h --- a/src/lib-fts/fts-tokenizer.h Sat May 09 13:46:37 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.h Sat May 09 13:52:37 2015 +0300 @@ -63,6 +63,9 @@ void fts_tokenizer_ref(struct fts_tokenizer *tok); void fts_tokenizer_unref(struct fts_tokenizer **tok); +/* Reset FTS tokenizer state */ +void fts_tokenizer_reset(struct fts_tokenizer *tok); + /* Returns 1 if token was returned, 0 if input was non-blocking and more data is needed, -1 if EOF/error. @@ -82,4 +85,5 @@ const unsigned char *data, size_t size, const char **token_r); const char *fts_tokenizer_name(const struct fts_tokenizer *tok); + #endif diff -r 363397c3701e -r 2dca6925bd88 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Sat May 09 13:46:37 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Sat May 09 13:52:37 2015 +0300 @@ -555,18 +555,31 @@ } test_assert(*eopp == NULL); + /* make sure state is forgotten at EOF */ test_assert(fts_tokenizer_next(tok, (const void *)"foo", 3, &token) == 0); - test_assert(fts_tokenizer_next(tok, NULL, 0, &token) > 0); + test_assert(fts_tokenizer_next(tok, NULL, 0, &token) > 0 && + strcmp(token, "foo") == 0); test_assert(fts_tokenizer_next(tok, NULL, 0, &token) == 0); test_assert(fts_tokenizer_next(tok, (const void *)"bar at baz", 7, &token) == 0); - test_assert(fts_tokenizer_next(tok, NULL, 0, &token) > 0); + test_assert(fts_tokenizer_next(tok, NULL, 0, &token) > 0 && + strcmp(token, "bar at baz") == 0); test_assert(fts_tokenizer_next(tok, NULL, 0, &token) == 0); test_assert(fts_tokenizer_next(tok, (const void *)"foo@", 4, &token) == 0); - test_assert(fts_tokenizer_next(tok, NULL, 0, &token) > 0); + test_assert(fts_tokenizer_next(tok, NULL, 0, &token) > 0 && + strcmp(token, "foo") == 0); test_assert(fts_tokenizer_next(tok, NULL, 0, &token) == 0); + /* test reset explicitly */ + test_assert(fts_tokenizer_next(tok, (const void *)"a", 1, &token) == 0); + fts_tokenizer_reset(tok); + test_assert(fts_tokenizer_next(tok, (const void *)"b at c", 3, &token) == 0); + test_assert(fts_tokenizer_next(tok, NULL, 0, &token) > 0 && + strcmp(token, "b at c") == 0); + test_assert(fts_tokenizer_next(tok, NULL, 0, &token) == 0); + + fts_tokenizer_unref(&tok); fts_tokenizer_unref(&gen_tok); fts_tokenizers_deinit(); From dovecot at dovecot.org Sat May 9 10:59:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 10:59:36 +0000 Subject: dovecot-2.2: fts: Reset tokenizers before using them Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d2e0e9aed81c changeset: 18581:d2e0e9aed81c user: Timo Sirainen date: Sat May 09 13:57:41 2015 +0300 description: fts: Reset tokenizers before using them This is mainly needed if the previous tokenization had failed. diffstat: src/plugins/fts/fts-build-mail.c | 10 +++++++++- src/plugins/fts/fts-search-args.c | 3 +++ 2 files changed, 12 insertions(+), 1 deletions(-) diffs (34 lines): diff -r 2dca6925bd88 -r d2e0e9aed81c src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Sat May 09 13:52:37 2015 +0300 +++ b/src/plugins/fts/fts-build-mail.c Sat May 09 13:57:41 2015 +0300 @@ -456,8 +456,16 @@ memset(&ctx, 0, sizeof(ctx)); ctx.update_ctx = update_ctx; ctx.mail = mail; - if ((update_ctx->backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) + if ((update_ctx->backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) { ctx.pending_input = buffer_create_dynamic(default_pool, 128); + /* reset tokenizer between mails - just to be sure no state + leaks between mails (especially if previous indexing had + failed) */ + struct fts_tokenizer *tokenizer; + + tokenizer = fts_user_get_index_tokenizer(update_ctx->backend->ns->user); + fts_tokenizer_reset(tokenizer); + } prev_part = NULL; parser = message_parser_init(pool_datastack_create(), input, diff -r 2dca6925bd88 -r d2e0e9aed81c src/plugins/fts/fts-search-args.c --- a/src/plugins/fts/fts-search-args.c Sat May 09 13:52:37 2015 +0300 +++ b/src/plugins/fts/fts-search-args.c Sat May 09 13:57:41 2015 +0300 @@ -114,6 +114,9 @@ and_arg->match_not = orig_arg->match_not; and_arg->next = orig_arg->next; + /* reset tokenizer between search args in case there's any state left + from some previous failure */ + fts_tokenizer_reset(tokenizer); while (fts_tokenizer_next(tokenizer, (const void *)orig_token, orig_token_len, &token) > 0) { From dovecot at dovecot.org Sat May 9 11:21:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 11:21:50 +0000 Subject: dovecot-2.2: lib-fts: Minor unit test cleanups Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b2f9e4cf17db changeset: 18582:b2f9e4cf17db user: Timo Sirainen date: Sat May 09 14:19:48 2015 +0300 description: lib-fts: Minor unit test cleanups diffstat: src/lib-fts/test-fts-filter.c | 74 +++++++++++++----------------------------- 1 files changed, 24 insertions(+), 50 deletions(-) diffs (249 lines): diff -r d2e0e9aed81c -r b2f9e4cf17db src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Sat May 09 13:57:41 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Sat May 09 14:19:48 2015 +0300 @@ -8,12 +8,12 @@ #include -const char *const stopword_settings[] = {"stopwords_dir", TEST_STOPWORDS_DIR, NULL}; +static const char *const stopword_settings[] = {"stopwords_dir", TEST_STOPWORDS_DIR, NULL}; +static struct fts_language english_language = { .name = "en" }; static void test_fts_filter_stopwords_eng(void) { const struct fts_filter *filter_class; - const struct fts_language english = { .name = "en" }; struct fts_filter *filter; const char *error; int ret; @@ -28,8 +28,7 @@ test_begin("fts filter stopwords, English"); filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); - ret = fts_filter_create(filter_class, NULL, &english, stopword_settings, &filter, &error); - test_assert(ret == 0); + test_assert(fts_filter_create(filter_class, NULL, &english_language, stopword_settings, &filter, &error) == 0); ip = input; op = output; @@ -72,8 +71,7 @@ test_begin("fts filter stopwords, Finnish"); filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); - ret = fts_filter_create(filter_class, NULL, &finnish, stopword_settings, &filter, &error); - test_assert(ret == 0); + test_assert(fts_filter_create(filter_class, NULL, &finnish, stopword_settings, &filter, &error) == 0); ip = input; op = output; @@ -94,8 +92,7 @@ fts_filter_unref(&filter); test_assert(filter == NULL); - ret = fts_filter_create(filter_class, NULL, &finnish, stopword_settings, &filter, &error); - test_assert(ret == 0); + test_assert(fts_filter_create(filter_class, NULL, &finnish, stopword_settings, &filter, &error) == 0); ip = input2; op = output2; while (*ip != NULL) { @@ -137,8 +134,7 @@ test_begin("fts filter stopwords, French"); filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); - ret = fts_filter_create(filter_class, NULL, &french, stopword_settings, &filter, &error); - test_assert(ret == 0); + test_assert(fts_filter_create(filter_class, NULL, &french, stopword_settings, &filter, &error) == 0); ip = input; op = output; @@ -167,14 +163,12 @@ const struct fts_language unknown = { .name = "bebobidoop" }; struct fts_filter *filter = NULL; const char *error = NULL, *token = "foobar"; - int ret; test_begin("fts filter stopwords, fail filter() (lazy init)"); filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); - ret = fts_filter_create(filter_class, NULL, &unknown, stopword_settings, &filter, &error); - test_assert(ret == 0 && filter != NULL && error == NULL); - ret = fts_filter_filter(filter, &token, &error); - test_assert(ret == -1 && error != NULL); + test_assert(fts_filter_create(filter_class, NULL, &unknown, stopword_settings, &filter, &error) == 0); + test_assert(filter != NULL && error == NULL); + test_assert(fts_filter_filter(filter, &token, &error) < 0 && error != NULL); test_end(); } @@ -182,11 +176,9 @@ #ifdef HAVE_FTS_STEMMER static void test_fts_filter_stemmer_snowball_stem_english(void) { - int ret; const struct fts_filter *filter_class; struct fts_filter *stemmer; const char *error; - struct fts_language language = { .name = "en" }; const char *token = NULL; const char * const tokens[] = { "dries" ,"friendlies", "All", "human", "beings", "are", @@ -205,12 +197,11 @@ test_begin("fts filter stem English"); filter_class = fts_filter_find(SNOWBALL_STEMMER_FILTER_NAME); - ret = fts_filter_create(filter_class, NULL, &language, NULL, &stemmer, &error); - test_assert(ret == 0); + test_assert(fts_filter_create(filter_class, NULL, &english_language, NULL, &stemmer, &error) == 0); bpp = bases; for (tpp=tokens; *tpp != NULL; tpp++) { token = *tpp; - ret = fts_filter_filter(stemmer, &token, &error); + test_assert(fts_filter_filter(stemmer, &token, &error) > 0); test_assert(token != NULL); test_assert(null_strcmp(token, *bpp) == 0); bpp++; @@ -222,7 +213,6 @@ static void test_fts_filter_stemmer_snowball_stem_french(void) { - int ret; const struct fts_filter *filter_class; struct fts_filter *stemmer; const char *error; @@ -240,12 +230,11 @@ test_begin("fts filter stem French"); filter_class = fts_filter_find(SNOWBALL_STEMMER_FILTER_NAME); - ret = fts_filter_create(filter_class, NULL, &language, NULL, &stemmer, &error); - test_assert(ret == 0); + test_assert(fts_filter_create(filter_class, NULL, &language, NULL, &stemmer, &error) == 0); bpp = bases; for (tpp=tokens; *tpp != NULL; tpp++) { token = *tpp; - ret = fts_filter_filter(stemmer, &token, &error); + test_assert(fts_filter_filter(stemmer, &token, &error) > 0); test_assert(token != NULL); test_assert(null_strcmp(token, *bpp) == 0); bpp++; @@ -262,7 +251,6 @@ struct fts_filter *stemmer; struct fts_filter *filter; const char *error; - struct fts_language language = { .name = "en" }; const char *token = NULL; const char * const tokens[] = { "dries" ,"friendlies", "All", "human", "beings", "are", @@ -282,12 +270,10 @@ test_begin("fts filters stopwords and stemming chained, English"); filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); - ret = fts_filter_create(filter_class, NULL, &language, stopword_settings, &filter, &error); - test_assert(ret == 0); + test_assert(fts_filter_create(filter_class, NULL, &english_language, stopword_settings, &filter, &error) == 0); filter_class = fts_filter_find(SNOWBALL_STEMMER_FILTER_NAME); - ret = fts_filter_create(filter_class, filter, &language, NULL, &stemmer, &error); - test_assert(ret == 0); + test_assert(fts_filter_create(filter_class, filter, &english_language, NULL, &stemmer, &error) == 0); bpp = bases; for (tpp=tokens; *tpp != NULL; tpp++) { @@ -315,7 +301,6 @@ { const struct fts_filter *filter_class; struct fts_filter *norm = NULL; - int ret; const char *input[] = { NULL, "", @@ -343,8 +328,7 @@ T_BEGIN { filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); - ret = fts_filter_create(filter_class, NULL, NULL, settings, &norm, &error); - test_assert(ret == 0); + test_assert(fts_filter_create(filter_class, NULL, NULL, settings, &norm, &error) == 0); for (i = 0; i < N_ELEMENTS(input); i++) { if (input[i] != NULL) { token = input[i]; @@ -362,7 +346,6 @@ { const struct fts_filter *filter_class; struct fts_filter *norm = NULL; - int ret; const char *input[] = { NULL, "", @@ -388,8 +371,7 @@ T_BEGIN { filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); - ret = fts_filter_create(filter_class, NULL, NULL, NULL, &norm, &error); - test_assert(ret == 0); + test_assert(fts_filter_create(filter_class, NULL, NULL, NULL, &norm, &error) == 0); for (i = 0; i < N_ELEMENTS(input); i++) { if (input[i] != NULL) { token = input[i]; @@ -415,7 +397,6 @@ char buf[4096] = {0}; const char *error = NULL; const char *tokens; - int ret; unsigned char sha512_digest[SHA512_RESULTLEN]; struct sha512_ctx ctx; const unsigned char correct_digest[] = { @@ -434,8 +415,7 @@ T_BEGIN { udhr_path = t_strconcat(UDHRDIR, UDHR_FRA_NAME, NULL); filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); - ret = fts_filter_create(filter_class, NULL, NULL, settings, &norm, &error); - test_assert(ret == 0); + test_assert(fts_filter_create(filter_class, NULL, NULL, settings, &norm, &error) == 0); input = fopen(udhr_path, "r"); test_assert(input != NULL); sha512_init(&ctx); @@ -464,14 +444,12 @@ {"id", "Any-One-Out-There; DKFN; [: Nonspacing Mark :] Remove", NULL}; const char *error = NULL, *token = "foo"; - int ret; test_begin("fts filter normalizer invalid id"); filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); - ret = fts_filter_create(filter_class, NULL, NULL, settings, &norm, &error); - test_assert(ret == 0 && error == NULL); - ret = fts_filter_filter(norm, &token, &error); - test_assert(ret < 0 && error != NULL); + test_assert(fts_filter_create(filter_class, NULL, NULL, settings, &norm, &error) == 0); + test_assert(error == NULL); + test_assert(fts_filter_filter(norm, &token, &error) < 0 && error != NULL); test_end(); } @@ -486,7 +464,6 @@ const char * const id_settings[] = //{"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC", NULL}; {"id", "Lower", NULL}; - struct fts_language language = { .name = "en" }; const char *token = NULL; const char * const tokens[] = { "dries" ,"friendlies", "All", "human", "beings", "are", @@ -507,16 +484,13 @@ test_begin("fts filters normalizer, stopwords and stemming chained, English"); filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); - ret = fts_filter_create(filter_class, NULL, NULL, id_settings, &normalizer, &error); - test_assert(ret == 0); + test_assert(fts_filter_create(filter_class, NULL, NULL, id_settings, &normalizer, &error) == 0); filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); - ret = fts_filter_create(filter_class, normalizer, &language, stopword_settings, &filter, &error); - test_assert(ret == 0); + test_assert(fts_filter_create(filter_class, normalizer, &english_language, stopword_settings, &filter, &error) == 0); filter_class = fts_filter_find(SNOWBALL_STEMMER_FILTER_NAME); - ret = fts_filter_create(filter_class, filter, &language, NULL, &stemmer, &error); - test_assert(ret == 0); + test_assert(fts_filter_create(filter_class, filter, &english_language, NULL, &stemmer, &error) == 0); bpp = bases; for (tpp = tokens; *tpp != NULL; tpp++) { From dovecot at dovecot.org Sat May 9 11:28:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 11:28:36 +0000 Subject: dovecot-2.2: lib-fts: Removed "simple" normalizer. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1963690280b7 changeset: 18583:1963690280b7 user: Timo Sirainen date: Sat May 09 14:09:37 2015 +0300 description: lib-fts: Removed "simple" normalizer. It translated input to titlecase, which wasn't suitable for snowball stemming that wanted lowercase input. Since that doesn't work, there's probably no good for the existence of this (perhaps in future it's replaced by unicode-aware lowercaser). diffstat: src/lib-fts/Makefile.am | 1 - src/lib-fts/fts-filter-normalizer-simple.c | 79 ------------------------------ src/lib-fts/fts-filter.c | 1 - src/lib-fts/fts-filter.h | 4 - 4 files changed, 0 insertions(+), 85 deletions(-) diffs (119 lines): diff -r b2f9e4cf17db -r 1963690280b7 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Sat May 09 14:19:48 2015 +0300 +++ b/src/lib-fts/Makefile.am Sat May 09 14:09:37 2015 +0300 @@ -62,7 +62,6 @@ libfts_la_SOURCES = \ fts-filter.c \ fts-filter-normalizer-icu.c \ - fts-filter-normalizer-simple.c \ fts-filter-stopwords.c \ fts-filter-stemmer-snowball.c \ fts-language.c \ diff -r b2f9e4cf17db -r 1963690280b7 src/lib-fts/fts-filter-normalizer-simple.c --- a/src/lib-fts/fts-filter-normalizer-simple.c Sat May 09 14:19:48 2015 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,79 +0,0 @@ -/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "unichar.h" -#include "str.h" -#include "fts-filter.h" -#include "fts-filter-private.h" -#include "fts-language.h" - -struct fts_filter_normalizer_simple { - struct fts_filter filter; - string_t *str; -}; - -static bool -fts_filter_normalizer_simple_supports(const struct fts_language *lang ATTR_UNUSED) -{ - return TRUE; -} - -static void -fts_filter_normalizer_simple_destroy(struct fts_filter *_filter) -{ - struct fts_filter_normalizer_simple *filter = - (struct fts_filter_normalizer_simple *)_filter; - - str_free(&filter->str); - i_free(filter); -} - -static int -fts_filter_normalizer_simple_create(const struct fts_language *lang ATTR_UNUSED, - const char *const *settings, - struct fts_filter **filter_r, - const char **error_r) -{ - struct fts_filter_normalizer_simple *filter; - - if (settings[0] != NULL) { - *error_r = t_strdup_printf("Unknown setting: %s", settings[0]); - return -1; - } - filter = i_new(struct fts_filter_normalizer_simple, 1); - filter->filter = *fts_filter_normalizer_simple; - filter->str = str_new(default_pool, 128); - - *filter_r = &filter->filter; - return 0; -} - -static int -fts_filter_normalizer_simple_filter(struct fts_filter *_filter, - const char **token, - const char **error_r ATTR_UNUSED) -{ - struct fts_filter_normalizer_simple *filter = - (struct fts_filter_normalizer_simple *)_filter; - - str_truncate(filter->str, 0); - if (uni_utf8_to_decomposed_titlecase(*token, strlen(*token), - filter->str) < 0) - i_panic("fts-normalizer-simple: Token is not valid UTF-8: %s", *token); - *token = str_c(filter->str); - return 1; -} - -static const struct fts_filter_vfuncs normalizer_filter_vfuncs = { - fts_filter_normalizer_simple_supports, - fts_filter_normalizer_simple_create, - fts_filter_normalizer_simple_filter, - fts_filter_normalizer_simple_destroy -}; - -static const struct fts_filter fts_filter_normalizer_simple_real = { - .class_name = SIMPLE_NORMALIZER_FILTER_NAME, - .v = &normalizer_filter_vfuncs -}; - -const struct fts_filter *fts_filter_normalizer_simple = &fts_filter_normalizer_simple_real; diff -r b2f9e4cf17db -r 1963690280b7 src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Sat May 09 14:19:48 2015 +0300 +++ b/src/lib-fts/fts-filter.c Sat May 09 14:09:37 2015 +0300 @@ -15,7 +15,6 @@ fts_filter_register(fts_filter_stopwords); fts_filter_register(fts_filter_stemmer_snowball); fts_filter_register(fts_filter_normalizer_icu); - fts_filter_register(fts_filter_normalizer_simple); } void fts_filters_deinit(void) diff -r b2f9e4cf17db -r 1963690280b7 src/lib-fts/fts-filter.h --- a/src/lib-fts/fts-filter.h Sat May 09 14:19:48 2015 +0300 +++ b/src/lib-fts/fts-filter.h Sat May 09 14:09:37 2015 +0300 @@ -33,10 +33,6 @@ extern const struct fts_filter *fts_filter_normalizer_icu; #define ICU_NORMALIZER_FILTER_NAME "normalizer-icu" -/* Normalization using i;unicode-casemap (RFC 5051) */ -extern const struct fts_filter *fts_filter_normalizer_simple; -#define SIMPLE_NORMALIZER_FILTER_NAME "normalizer-simple" - /* Register all built-in filters. */ void fts_filters_init(void); void fts_filters_deinit(void); From dovecot at dovecot.org Sat May 9 11:28:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 11:28:36 +0000 Subject: dovecot-2.2: lib-fts: Added "lowercase" filter. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/75b4b312ea09 changeset: 18584:75b4b312ea09 user: Timo Sirainen date: Sat May 09 14:26:42 2015 +0300 description: lib-fts: Added "lowercase" filter. For now it handles only ASCII characters, but that's enough for our use. diffstat: src/lib-fts/Makefile.am | 1 + src/lib-fts/fts-filter-lowercase.c | 61 ++++++++++++++++++++++++++++++++++++++ src/lib-fts/fts-filter.c | 1 + src/lib-fts/fts-filter.h | 4 ++ src/lib-fts/test-fts-filter.c | 30 ++++++++++++++++++ 5 files changed, 97 insertions(+), 0 deletions(-) diffs (148 lines): diff -r 1963690280b7 -r 75b4b312ea09 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Sat May 09 14:09:37 2015 +0300 +++ b/src/lib-fts/Makefile.am Sat May 09 14:26:42 2015 +0300 @@ -61,6 +61,7 @@ libfts_la_SOURCES = \ fts-filter.c \ + fts-filter-lowercase.c \ fts-filter-normalizer-icu.c \ fts-filter-stopwords.c \ fts-filter-stemmer-snowball.c \ diff -r 1963690280b7 -r 75b4b312ea09 src/lib-fts/fts-filter-lowercase.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/fts-filter-lowercase.c Sat May 09 14:26:42 2015 +0300 @@ -0,0 +1,61 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "fts-filter.h" +#include "fts-filter-private.h" +#include "fts-language.h" + +static bool +fts_filter_lowercase_supports(const struct fts_language *lang ATTR_UNUSED) +{ + return TRUE; +} + +static void +fts_filter_lowercase_destroy(struct fts_filter *filter) +{ + i_free(filter); +} + +static int +fts_filter_lowercase_create(const struct fts_language *lang ATTR_UNUSED, + const char *const *settings, + struct fts_filter **filter_r, + const char **error_r) +{ + struct fts_filter *filter; + + if (settings[0] != NULL) { + *error_r = t_strdup_printf("Unknown setting: %s", settings[0]); + return -1; + } + filter = i_new(struct fts_filter, 1); + *filter = *fts_filter_lowercase; + + *filter_r = filter; + return 0; +} + +static int +fts_filter_lowercase_filter(struct fts_filter *_filter ATTR_UNUSED, + const char **token, + const char **error_r ATTR_UNUSED) +{ + *token = t_str_lcase(*token); + return 1; +} + +static const struct fts_filter_vfuncs normalizer_filter_vfuncs = { + fts_filter_lowercase_supports, + fts_filter_lowercase_create, + fts_filter_lowercase_filter, + fts_filter_lowercase_destroy +}; + +static const struct fts_filter fts_filter_lowercase_real = { + .class_name = LOWERCASE_FILTER_NAME, + .v = &normalizer_filter_vfuncs +}; + +const struct fts_filter *fts_filter_lowercase = &fts_filter_lowercase_real; diff -r 1963690280b7 -r 75b4b312ea09 src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Sat May 09 14:09:37 2015 +0300 +++ b/src/lib-fts/fts-filter.c Sat May 09 14:26:42 2015 +0300 @@ -15,6 +15,7 @@ fts_filter_register(fts_filter_stopwords); fts_filter_register(fts_filter_stemmer_snowball); fts_filter_register(fts_filter_normalizer_icu); + fts_filter_register(fts_filter_lowercase); } void fts_filters_deinit(void) diff -r 1963690280b7 -r 75b4b312ea09 src/lib-fts/fts-filter.h --- a/src/lib-fts/fts-filter.h Sat May 09 14:09:37 2015 +0300 +++ b/src/lib-fts/fts-filter.h Sat May 09 14:26:42 2015 +0300 @@ -33,6 +33,10 @@ extern const struct fts_filter *fts_filter_normalizer_icu; #define ICU_NORMALIZER_FILTER_NAME "normalizer-icu" +/* Lowecases the input. Currently only ASCII data is lowercased. */ +extern const struct fts_filter *fts_filter_lowercase; +#define LOWERCASE_FILTER_NAME "lowercase" + /* Register all built-in filters. */ void fts_filters_init(void); void fts_filters_deinit(void); diff -r 1963690280b7 -r 75b4b312ea09 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Sat May 09 14:09:37 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Sat May 09 14:26:42 2015 +0300 @@ -11,6 +11,35 @@ static const char *const stopword_settings[] = {"stopwords_dir", TEST_STOPWORDS_DIR, NULL}; static struct fts_language english_language = { .name = "en" }; +static void test_fts_filter_lowercase(void) +{ + struct { + const char *input; + const char *output; + } tests[] = { + { "foo", "foo" }, + { "FOO", "foo" }, + { "fOo", "foo" } + }; + const struct fts_filter *filter_class; + struct fts_filter *filter; + const char *error; + const char *token; + unsigned int i; + + test_begin("fts filter lowercase"); + filter_class = fts_filter_find(LOWERCASE_FILTER_NAME); + test_assert(fts_filter_create(filter_class, NULL, &english_language, NULL, &filter, &error) == 0); + + for (i = 0; i < N_ELEMENTS(tests); i++) { + token = tests[i].input; + test_assert_idx(fts_filter_filter(filter, &token, &error) > 0 && + strcmp(token, tests[i].output) == 0, 0); + } + fts_filter_unref(&filter); + test_end(); +} + static void test_fts_filter_stopwords_eng(void) { const struct fts_filter *filter_class; @@ -521,6 +550,7 @@ int main(void) { static void (*test_functions[])(void) = { + test_fts_filter_lowercase, test_fts_filter_stopwords_eng, test_fts_filter_stopwords_fin, test_fts_filter_stopwords_fra, From dovecot at dovecot.org Sat May 9 11:43:00 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 11:43:00 +0000 Subject: dovecot-2.2: fts: Lowecase non-human language input while indexing. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fcc20dce3c83 changeset: 18585:fcc20dce3c83 user: Timo Sirainen date: Sat May 09 14:41:05 2015 +0300 description: fts: Lowecase non-human language input while indexing. diffstat: src/plugins/fts/fts-build-mail.c | 29 +++++++++++++++++++++-------- src/plugins/fts/fts-user.c | 23 +++++++++++++++++++++++ src/plugins/fts/fts-user.h | 1 + 3 files changed, 45 insertions(+), 8 deletions(-) diffs (112 lines): diff -r 75b4b312ea09 -r fcc20dce3c83 src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Sat May 09 14:26:42 2015 +0300 +++ b/src/plugins/fts/fts-build-mail.c Sat May 09 14:41:05 2015 +0300 @@ -35,11 +35,6 @@ struct fts_user_language *cur_user_lang; }; -static struct fts_user_language fts_user_language_data = { - .lang = &fts_language_data, - .filter = NULL -}; - static int fts_build_data(struct fts_mail_build_context *ctx, const unsigned char *data, size_t size, bool last); @@ -127,6 +122,17 @@ i_free(buf); } +static bool data_has_8bit(const unsigned char *data, size_t size) +{ + size_t i; + + for (i = 0; i < size; i++) { + if ((data[i] & 0x80) != 0) + return TRUE; + } + return FALSE; +} + static void fts_build_mail_header(struct fts_mail_build_context *ctx, const struct message_block *block) { @@ -145,10 +151,17 @@ key.part = block->part; key.hdr_name = hdr->name; - if (!header_has_language(key.hdr_name)) - ctx->cur_user_lang = &fts_user_language_data; + /* Headers that don't contain any human language will only be + translated to lowercase - no stemming or other filtering. There's + unfortunately no pefect way of detecting which headers contain + human languages, so we have a list of some hardcoded header names + and we'll also assume that if there's any 8bit content it's a human + language. */ + if (header_has_language(key.hdr_name) || + data_has_8bit(hdr->full_value, hdr->full_value_len)) + ctx->cur_user_lang = NULL; else - ctx->cur_user_lang = NULL; + ctx->cur_user_lang = fts_user_get_data_lang(ctx->update_ctx->backend->ns->user); if (!fts_backend_update_set_build_key(ctx->update_ctx, &key)) return; diff -r 75b4b312ea09 -r fcc20dce3c83 src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Sat May 09 14:26:42 2015 +0300 +++ b/src/plugins/fts/fts-user.c Sat May 09 14:41:05 2015 +0300 @@ -16,6 +16,7 @@ struct fts_language_list *lang_list; struct fts_tokenizer *index_tokenizer, *search_tokenizer; + struct fts_user_language *data_lang; ARRAY_TYPE(fts_user_language) languages; }; @@ -269,6 +270,26 @@ return &fuser->languages; } +struct fts_user_language *fts_user_get_data_lang(struct mail_user *user) +{ + struct fts_user *fuser = FTS_USER_CONTEXT(user); + struct fts_user_language *lang; + const char *error; + + if (fuser->data_lang != NULL) + return fuser->data_lang; + + lang = p_new(user->pool, struct fts_user_language, 1); + lang->lang = &fts_language_data; + + if (fts_filter_create(fts_filter_lowercase, NULL, lang->lang, NULL, + &lang->filter, &error) < 0) + i_unreached(); + i_assert(lang->filter != NULL); + fuser->data_lang = lang; + return fuser->data_lang; +} + static void fts_user_free(struct fts_user *fuser) { struct fts_user_language *const *user_langp; @@ -280,6 +301,8 @@ if ((*user_langp)->filter != NULL) fts_filter_unref(&(*user_langp)->filter); } + if (fuser->data_lang != NULL && fuser->data_lang->filter != NULL) + fts_filter_unref(&fuser->data_lang->filter); if (fuser->index_tokenizer != NULL) fts_tokenizer_unref(&fuser->index_tokenizer); diff -r 75b4b312ea09 -r fcc20dce3c83 src/plugins/fts/fts-user.h --- a/src/plugins/fts/fts-user.h Sat May 09 14:26:42 2015 +0300 +++ b/src/plugins/fts/fts-user.h Sat May 09 14:41:05 2015 +0300 @@ -15,6 +15,7 @@ struct fts_language_list *fts_user_get_language_list(struct mail_user *user); const ARRAY_TYPE(fts_user_language) * fts_user_get_all_languages(struct mail_user *user); +struct fts_user_language *fts_user_get_data_lang(struct mail_user *user); int fts_mail_user_init(struct mail_user *user, const char **error_r); void fts_mail_user_deinit(struct mail_user *user); From dovecot at dovecot.org Sat May 9 11:54:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 11:54:07 +0000 Subject: dovecot-2.2: lib-fts: Store pointers to fts_filter classes inste... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/307fe289f1b7 changeset: 18586:307fe289f1b7 user: Timo Sirainen date: Sat May 09 14:49:20 2015 +0300 description: lib-fts: Store pointers to fts_filter classes instead of copying the data. Doesn't really matter, but it's a bit cleaner when fts_filter_find() returns the same pointer as the fts_filter class originally was. diffstat: src/lib-fts/fts-filter.c | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diffs (33 lines): diff -r fcc20dce3c83 -r 307fe289f1b7 src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Sat May 09 14:41:05 2015 +0300 +++ b/src/lib-fts/fts-filter.c Sat May 09 14:49:20 2015 +0300 @@ -6,7 +6,7 @@ #include "fts-filter.h" #include "fts-filter-private.h" -ARRAY(struct fts_filter) fts_filter_classes; +static ARRAY(const struct fts_filter *) fts_filter_classes; void fts_filters_init(void) { @@ -27,16 +27,16 @@ { i_assert(fts_filter_find(filter_class->class_name) == NULL); - array_append(&fts_filter_classes, filter_class, 1); + array_append(&fts_filter_classes, &filter_class, 1); } const struct fts_filter *fts_filter_find(const char *name) { - const struct fts_filter *fp = NULL; + const struct fts_filter *const *fp = NULL; array_foreach(&fts_filter_classes, fp) { - if (strcmp(fp->class_name, name) == 0) - return fp; + if (strcmp((*fp)->class_name, name) == 0) + return *fp; } return NULL; } From dovecot at dovecot.org Sat May 9 11:54:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 11:54:07 +0000 Subject: dovecot-2.2: lib-fts: Removed filter name macros from fts-filter.h Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/055666aaba5a changeset: 18587:055666aaba5a user: Timo Sirainen date: Sat May 09 14:50:10 2015 +0300 description: lib-fts: Removed filter name macros from fts-filter.h The filters can be directly accessed via their class structs. diffstat: src/lib-fts/fts-filter-lowercase.c | 2 +- src/lib-fts/fts-filter-normalizer-icu.c | 2 +- src/lib-fts/fts-filter-stemmer-snowball.c | 2 +- src/lib-fts/fts-filter-stopwords.c | 2 +- src/lib-fts/fts-filter.h | 4 - src/lib-fts/test-fts-filter.c | 77 +++++++++++------------------- 6 files changed, 32 insertions(+), 57 deletions(-) diffs (truncated from 356 to 300 lines): diff -r 307fe289f1b7 -r 055666aaba5a src/lib-fts/fts-filter-lowercase.c --- a/src/lib-fts/fts-filter-lowercase.c Sat May 09 14:49:20 2015 +0300 +++ b/src/lib-fts/fts-filter-lowercase.c Sat May 09 14:50:10 2015 +0300 @@ -54,7 +54,7 @@ }; static const struct fts_filter fts_filter_lowercase_real = { - .class_name = LOWERCASE_FILTER_NAME, + .class_name = "lowercase", .v = &normalizer_filter_vfuncs }; diff -r 307fe289f1b7 -r 055666aaba5a src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 14:49:20 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 14:50:10 2015 +0300 @@ -281,7 +281,7 @@ }; static const struct fts_filter fts_filter_normalizer_icu_real = { - .class_name = ICU_NORMALIZER_FILTER_NAME, + .class_name = "normalizer-icu", .v = &normalizer_filter_vfuncs }; diff -r 307fe289f1b7 -r 055666aaba5a src/lib-fts/fts-filter-stemmer-snowball.c --- a/src/lib-fts/fts-filter-stemmer-snowball.c Sat May 09 14:49:20 2015 +0300 +++ b/src/lib-fts/fts-filter-stemmer-snowball.c Sat May 09 14:50:10 2015 +0300 @@ -141,7 +141,7 @@ }; static const struct fts_filter fts_filter_stemmer_snowball_real = { - .class_name = SNOWBALL_STEMMER_FILTER_NAME, + .class_name = "snowball", .v = &snowball_stemmer_filter_vfuncs }; diff -r 307fe289f1b7 -r 055666aaba5a src/lib-fts/fts-filter-stopwords.c --- a/src/lib-fts/fts-filter-stopwords.c Sat May 09 14:49:20 2015 +0300 +++ b/src/lib-fts/fts-filter-stopwords.c Sat May 09 14:50:10 2015 +0300 @@ -131,7 +131,7 @@ }; const struct fts_filter fts_filter_stopwords_real = { - .class_name = STOPWORDS_FILTER_NAME, + .class_name = "stopwords", .v = &stopwords_filter_vfuncs }; const struct fts_filter *fts_filter_stopwords = &fts_filter_stopwords_real; diff -r 307fe289f1b7 -r 055666aaba5a src/lib-fts/fts-filter.h --- a/src/lib-fts/fts-filter.h Sat May 09 14:49:20 2015 +0300 +++ b/src/lib-fts/fts-filter.h Sat May 09 14:50:10 2015 +0300 @@ -15,13 +15,11 @@ */ extern const struct fts_filter *fts_filter_stopwords; -#define STOPWORDS_FILTER_NAME "stopwords" /* Settings: "lang", language of the stemmed language. */ extern const struct fts_filter *fts_filter_stemmer_snowball; -#define SNOWBALL_STEMMER_FILTER_NAME "snowball" /* Settings: "id", description of the normalizing/translitterating rules @@ -31,11 +29,9 @@ Remove; NFC" */ extern const struct fts_filter *fts_filter_normalizer_icu; -#define ICU_NORMALIZER_FILTER_NAME "normalizer-icu" /* Lowecases the input. Currently only ASCII data is lowercased. */ extern const struct fts_filter *fts_filter_lowercase; -#define LOWERCASE_FILTER_NAME "lowercase" /* Register all built-in filters. */ void fts_filters_init(void); diff -r 307fe289f1b7 -r 055666aaba5a src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Sat May 09 14:49:20 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Sat May 09 14:50:10 2015 +0300 @@ -11,6 +11,16 @@ static const char *const stopword_settings[] = {"stopwords_dir", TEST_STOPWORDS_DIR, NULL}; static struct fts_language english_language = { .name = "en" }; +static void test_fts_filter_find(void) +{ + test_begin("fts filter find"); + test_assert(fts_filter_find("stopwords") == fts_filter_stopwords); + test_assert(fts_filter_find("snowball") == fts_filter_stemmer_snowball); + test_assert(fts_filter_find("normalizer-icu") == fts_filter_normalizer_icu); + test_assert(fts_filter_find("lowercase") == fts_filter_lowercase); + test_end(); +} + static void test_fts_filter_lowercase(void) { struct { @@ -21,15 +31,13 @@ { "FOO", "foo" }, { "fOo", "foo" } }; - const struct fts_filter *filter_class; struct fts_filter *filter; const char *error; const char *token; unsigned int i; test_begin("fts filter lowercase"); - filter_class = fts_filter_find(LOWERCASE_FILTER_NAME); - test_assert(fts_filter_create(filter_class, NULL, &english_language, NULL, &filter, &error) == 0); + test_assert(fts_filter_create(fts_filter_lowercase, NULL, &english_language, NULL, &filter, &error) == 0); for (i = 0; i < N_ELEMENTS(tests); i++) { token = tests[i].input; @@ -42,7 +50,6 @@ static void test_fts_filter_stopwords_eng(void) { - const struct fts_filter *filter_class; struct fts_filter *filter; const char *error; int ret; @@ -56,8 +63,7 @@ const char *token; test_begin("fts filter stopwords, English"); - filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); - test_assert(fts_filter_create(filter_class, NULL, &english_language, stopword_settings, &filter, &error) == 0); + test_assert(fts_filter_create(fts_filter_stopwords, NULL, &english_language, stopword_settings, &filter, &error) == 0); ip = input; op = output; @@ -82,7 +88,6 @@ static void test_fts_filter_stopwords_fin(void) { - const struct fts_filter *filter_class; const struct fts_language finnish = { .name = "fi" }; struct fts_filter *filter; const char *error; @@ -99,8 +104,7 @@ const char *token; test_begin("fts filter stopwords, Finnish"); - filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); - test_assert(fts_filter_create(filter_class, NULL, &finnish, stopword_settings, &filter, &error) == 0); + test_assert(fts_filter_create(fts_filter_stopwords, NULL, &finnish, stopword_settings, &filter, &error) == 0); ip = input; op = output; @@ -121,7 +125,7 @@ fts_filter_unref(&filter); test_assert(filter == NULL); - test_assert(fts_filter_create(filter_class, NULL, &finnish, stopword_settings, &filter, &error) == 0); + test_assert(fts_filter_create(fts_filter_stopwords, NULL, &finnish, stopword_settings, &filter, &error) == 0); ip = input2; op = output2; while (*ip != NULL) { @@ -145,7 +149,6 @@ static void test_fts_filter_stopwords_fra(void) { - const struct fts_filter *filter_class; const struct fts_language french = { .name = "fr" }; struct fts_filter *filter; const char *error; @@ -162,8 +165,7 @@ const char *token; test_begin("fts filter stopwords, French"); - filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); - test_assert(fts_filter_create(filter_class, NULL, &french, stopword_settings, &filter, &error) == 0); + test_assert(fts_filter_create(fts_filter_stopwords, NULL, &french, stopword_settings, &filter, &error) == 0); ip = input; op = output; @@ -188,14 +190,12 @@ static void test_fts_filter_stopwords_fail_lazy_init(void) { - const struct fts_filter *filter_class; const struct fts_language unknown = { .name = "bebobidoop" }; struct fts_filter *filter = NULL; const char *error = NULL, *token = "foobar"; test_begin("fts filter stopwords, fail filter() (lazy init)"); - filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); - test_assert(fts_filter_create(filter_class, NULL, &unknown, stopword_settings, &filter, &error) == 0); + test_assert(fts_filter_create(fts_filter_stopwords, NULL, &unknown, stopword_settings, &filter, &error) == 0); test_assert(filter != NULL && error == NULL); test_assert(fts_filter_filter(filter, &token, &error) < 0 && error != NULL); test_end(); @@ -205,7 +205,6 @@ #ifdef HAVE_FTS_STEMMER static void test_fts_filter_stemmer_snowball_stem_english(void) { - const struct fts_filter *filter_class; struct fts_filter *stemmer; const char *error; const char *token = NULL; @@ -225,8 +224,7 @@ const char * const *bpp; test_begin("fts filter stem English"); - filter_class = fts_filter_find(SNOWBALL_STEMMER_FILTER_NAME); - test_assert(fts_filter_create(filter_class, NULL, &english_language, NULL, &stemmer, &error) == 0); + test_assert(fts_filter_create(fts_filter_stemmer_snowball, NULL, &english_language, NULL, &stemmer, &error) == 0); bpp = bases; for (tpp=tokens; *tpp != NULL; tpp++) { token = *tpp; @@ -242,7 +240,6 @@ static void test_fts_filter_stemmer_snowball_stem_french(void) { - const struct fts_filter *filter_class; struct fts_filter *stemmer; const char *error; struct fts_language language = { .name = "fr" }; @@ -258,8 +255,7 @@ const char * const *bpp; test_begin("fts filter stem French"); - filter_class = fts_filter_find(SNOWBALL_STEMMER_FILTER_NAME); - test_assert(fts_filter_create(filter_class, NULL, &language, NULL, &stemmer, &error) == 0); + test_assert(fts_filter_create(fts_filter_stemmer_snowball, NULL, &language, NULL, &stemmer, &error) == 0); bpp = bases; for (tpp=tokens; *tpp != NULL; tpp++) { token = *tpp; @@ -276,7 +272,6 @@ static void test_fts_filter_stopwords_stemmer_eng(void) { int ret; - const struct fts_filter *filter_class; struct fts_filter *stemmer; struct fts_filter *filter; const char *error; @@ -298,11 +293,8 @@ test_begin("fts filters stopwords and stemming chained, English"); - filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); - test_assert(fts_filter_create(filter_class, NULL, &english_language, stopword_settings, &filter, &error) == 0); - - filter_class = fts_filter_find(SNOWBALL_STEMMER_FILTER_NAME); - test_assert(fts_filter_create(filter_class, filter, &english_language, NULL, &stemmer, &error) == 0); + test_assert(fts_filter_create(fts_filter_stopwords, NULL, &english_language, stopword_settings, &filter, &error) == 0); + test_assert(fts_filter_create(fts_filter_stemmer_snowball, filter, &english_language, NULL, &stemmer, &error) == 0); bpp = bases; for (tpp=tokens; *tpp != NULL; tpp++) { @@ -328,7 +320,6 @@ #ifdef HAVE_LIBICU static void test_fts_filter_normalizer_swedish_short(void) { - const struct fts_filter *filter_class; struct fts_filter *norm = NULL; const char *input[] = { NULL, @@ -356,8 +347,7 @@ test_begin("fts filter normalizer Swedish short text"); T_BEGIN { - filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); - test_assert(fts_filter_create(filter_class, NULL, NULL, settings, &norm, &error) == 0); + test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0); for (i = 0; i < N_ELEMENTS(input); i++) { if (input[i] != NULL) { token = input[i]; @@ -373,7 +363,6 @@ static void test_fts_filter_normalizer_swedish_short_default_id(void) { - const struct fts_filter *filter_class; struct fts_filter *norm = NULL; const char *input[] = { NULL, @@ -399,8 +388,7 @@ test_begin("fts filter normalizer Swedish short text using default ID"); T_BEGIN { - filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); - test_assert(fts_filter_create(filter_class, NULL, NULL, NULL, &norm, &error) == 0); + test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, NULL, &norm, &error) == 0); for (i = 0; i < N_ELEMENTS(input); i++) { if (input[i] != NULL) { token = input[i]; @@ -419,7 +407,6 @@ static void test_fts_filter_normalizer_french(void) { struct fts_filter *norm = NULL; - const struct fts_filter *filter_class; FILE *input; const char * const settings[] = {"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove", NULL}; @@ -443,8 +430,7 @@ T_BEGIN { udhr_path = t_strconcat(UDHRDIR, UDHR_FRA_NAME, NULL); - filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); From dovecot at dovecot.org Sat May 9 11:55:41 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 11:55:41 +0000 Subject: dovecot-2.2: lib-fts: test-fts-tokenizer cleanup - moved tokeniz... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cd376c0ac132 changeset: 18588:cd376c0ac132 user: Timo Sirainen date: Sat May 09 14:53:46 2015 +0300 description: lib-fts: test-fts-tokenizer cleanup - moved tokenizers_init/deinit() to be done globally diffstat: src/lib-fts/test-fts-tokenizer.c | 14 +++++--------- 1 files changed, 5 insertions(+), 9 deletions(-) diffs (79 lines): diff -r 055666aaba5a -r cd376c0ac132 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Sat May 09 14:50:10 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Sat May 09 14:53:46 2015 +0300 @@ -28,7 +28,6 @@ const char *token, *error; test_begin("fts tokenizer generic simple"); - fts_tokenizers_init(); tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME); test_assert(fts_tokenizer_create(tok_class, NULL, NULL, &tok, &error) == 0); /*TODO: Uncomment when fts-tokenizer-generic-private.h inclusion is fixed */ @@ -43,7 +42,6 @@ } test_assert(*eopp == NULL); fts_tokenizer_unref(&tok); - fts_tokenizers_deinit(); test_end(); } @@ -400,7 +398,6 @@ int ret; test_begin("fts tokenizer email address + parent, input one character at a time"); - fts_tokenizers_init(); test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &gen_tok, &error) == 0); test_assert(fts_tokenizer_create(fts_tokenizer_email_address, gen_tok, NULL, &tok, &error) == 0); @@ -420,7 +417,6 @@ test_assert(*eopp == NULL); fts_tokenizer_unref(&tok); fts_tokenizer_unref(&gen_tok); - fts_tokenizers_deinit(); test_end(); } @@ -446,7 +442,6 @@ int ret; test_begin("fts tokenizer email address + parent, input one line at a time"); - fts_tokenizers_init(); test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &gen_tok, &error) == 0); test_assert(fts_tokenizer_create(fts_tokenizer_email_address, gen_tok, NULL, &tok, &error) == 0); @@ -466,7 +461,6 @@ test_assert(*eopp == NULL); fts_tokenizer_unref(&tok); fts_tokenizer_unref(&gen_tok); - fts_tokenizers_deinit(); test_end(); } @@ -536,7 +530,6 @@ int ret; test_begin("fts tokenizer search email address + parent, input one character at a time"); - fts_tokenizers_init(); test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &gen_tok, &error) == 0); test_assert(fts_tokenizer_create(fts_tokenizer_email_address, gen_tok, settings, &tok, &error) == 0); @@ -582,7 +575,6 @@ fts_tokenizer_unref(&tok); fts_tokenizer_unref(&gen_tok); - fts_tokenizers_deinit(); test_end(); } @@ -605,6 +597,10 @@ test_fts_tokenizer_address_search, NULL }; + int ret; - return test_run(test_functions); + fts_tokenizers_init(); + ret = test_run(test_functions); + fts_tokenizers_deinit(); + return ret; } From dovecot at dovecot.org Sat May 9 12:00:26 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 12:00:26 +0000 Subject: dovecot-2.2: lib-fts: Store pointers to fts_tokenizer classes in... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5937f4d505c5 changeset: 18589:5937f4d505c5 user: Timo Sirainen date: Sat May 09 14:55:47 2015 +0300 description: lib-fts: Store pointers to fts_tokenizer classes instead of copying the data. The same reason as for doing this for fts-filter. diffstat: src/lib-fts/fts-tokenizer.c | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) diffs (48 lines): diff -r cd376c0ac132 -r 5937f4d505c5 src/lib-fts/fts-tokenizer.c --- a/src/lib-fts/fts-tokenizer.c Sat May 09 14:53:46 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.c Sat May 09 14:55:47 2015 +0300 @@ -8,7 +8,7 @@ #include "fts-tokenizer.h" #include "fts-tokenizer-private.h" -ARRAY(struct fts_tokenizer) fts_tokenizer_classes; +static ARRAY(const struct fts_tokenizer *) fts_tokenizer_classes; void fts_tokenizers_init(void) { @@ -29,17 +29,17 @@ { if (!array_is_created(&fts_tokenizer_classes)) i_array_init(&fts_tokenizer_classes, FTS_TOKENIZER_CLASSES_NR); - array_append(&fts_tokenizer_classes, tok_class, 1); + array_append(&fts_tokenizer_classes, &tok_class, 1); } /* private */ void fts_tokenizer_unregister(const struct fts_tokenizer *tok_class) { - const struct fts_tokenizer *tp; + const struct fts_tokenizer *const *tp; unsigned int idx; array_foreach(&fts_tokenizer_classes, tp) { - if (strcmp(tp->name, tok_class->name) == 0) { + if (strcmp((*tp)->name, tok_class->name) == 0) { idx = array_foreach_idx(&fts_tokenizer_classes, tp); array_delete(&fts_tokenizer_classes, idx, 1); if (array_count(&fts_tokenizer_classes) == 0) @@ -52,11 +52,11 @@ const struct fts_tokenizer *fts_tokenizer_find(const char *name) { - const struct fts_tokenizer *tp; + const struct fts_tokenizer *const *tp; array_foreach(&fts_tokenizer_classes, tp) { - if (strcmp(tp->name, name) == 0) - return tp; + if (strcmp((*tp)->name, name) == 0) + return *tp; } return NULL; } From dovecot at dovecot.org Sat May 9 12:00:27 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 12:00:27 +0000 Subject: dovecot-2.2: lib-fts: Removed tokenizer name macros from fts-tok... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/12aeb1ac8f0d changeset: 18590:12aeb1ac8f0d user: Timo Sirainen date: Sat May 09 14:56:33 2015 +0300 description: lib-fts: Removed tokenizer name macros from fts-tokenizer.h The tokenizers can be directly accessed via their class structs. diffstat: src/lib-fts/fts-tokenizer-address.c | 2 +- src/lib-fts/fts-tokenizer-generic.c | 2 +- src/lib-fts/fts-tokenizer.h | 2 -- src/lib-fts/test-fts-tokenizer.c | 29 ++++++++++++++--------------- 4 files changed, 16 insertions(+), 19 deletions(-) diffs (153 lines): diff -r 5937f4d505c5 -r 12aeb1ac8f0d src/lib-fts/fts-tokenizer-address.c --- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 14:55:47 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 14:56:33 2015 +0300 @@ -305,7 +305,7 @@ }; static const struct fts_tokenizer fts_tokenizer_email_address_real = { - .name = FTS_TOKENIZER_EMAIL_ADDRESS_NAME, + .name = "email-address", .v = &email_address_tokenizer_vfuncs }; const struct fts_tokenizer *fts_tokenizer_email_address = diff -r 5937f4d505c5 -r 12aeb1ac8f0d src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Sat May 09 14:55:47 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Sat May 09 14:56:33 2015 +0300 @@ -596,7 +596,7 @@ }; static const struct fts_tokenizer fts_tokenizer_generic_real = { - .name = FTS_TOKENIZER_GENERIC_NAME, + .name = "generic", .v = &generic_tokenizer_vfuncs }; const struct fts_tokenizer *fts_tokenizer_generic = &fts_tokenizer_generic_real; diff -r 5937f4d505c5 -r 12aeb1ac8f0d src/lib-fts/fts-tokenizer.h --- a/src/lib-fts/fts-tokenizer.h Sat May 09 14:55:47 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.h Sat May 09 14:56:33 2015 +0300 @@ -23,7 +23,6 @@ further. Defaults to disabled. Enable by defining the keyword (and any value). */ extern const struct fts_tokenizer *fts_tokenizer_email_address; -#define FTS_TOKENIZER_EMAIL_ADDRESS_NAME "email-address" /* Generic email content tokenizer. Cuts text into tokens. */ /* Settings: @@ -41,7 +40,6 @@ differ in some details, e.g. simple will cut "a.b" and tr29 will not. The default is "simple" */ extern const struct fts_tokenizer *fts_tokenizer_generic; -#define FTS_TOKENIZER_GENERIC_NAME "generic" /* Tokenizing workflow, find --> create --> filter --> destroy. diff -r 5937f4d505c5 -r 12aeb1ac8f0d src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Sat May 09 14:55:47 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Sat May 09 14:56:33 2015 +0300 @@ -11,6 +11,14 @@ #include +static void test_fts_tokenizer_find(void) +{ + test_begin("fts tokenizer find"); + test_assert(fts_tokenizer_find("email-address") == fts_tokenizer_email_address); + test_assert(fts_tokenizer_find("generic") == fts_tokenizer_generic); + test_end(); +} + static void test_fts_tokenizer_generic_only(void) { static const unsigned char input[] = @@ -22,14 +30,12 @@ "and", "longlonglongabcdefghijklmnopqr", "more", "Hello", "world", "last", NULL }; - const struct fts_tokenizer *tok_class; struct fts_tokenizer *tok; const char * const *eopp = expected_output; const char *token, *error; test_begin("fts tokenizer generic simple"); - tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME); - test_assert(fts_tokenizer_create(tok_class, NULL, NULL, &tok, &error) == 0); + test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &tok, &error) == 0); /*TODO: Uncomment when fts-tokenizer-generic-private.h inclusion is fixed */ /*test_assert(((struct generic_fts_tokenizer *) tok)->algorithm == BOUNDARY_ALGORITHM_SIMPLE);*/ while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) { @@ -57,15 +63,13 @@ "there", "was", "text", "galore", "and", "more", NULL }; - const struct fts_tokenizer *tok_class; struct fts_tokenizer *tok; const char * const *eopp = expected_output; const char *token, *error; test_begin("fts tokenizer generic simple with Unicode whitespace"); fts_tokenizer_register(fts_tokenizer_generic); - tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME); - test_assert(fts_tokenizer_create(tok_class, NULL, NULL, &tok, &error) == 0); + test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &tok, &error) == 0); while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) { test_assert(strcmp(token, *eopp) == 0); eopp++; @@ -132,15 +136,13 @@ "and", "more", "Hello", "world", "3.14", "3,14", "last", "longlonglongabcdefghijklmnopqr", "1", NULL }; - const struct fts_tokenizer *tok_class; struct fts_tokenizer *tok; const char * const *eopp = expected_output; const char *token, *error; test_begin("fts tokenizer generic TR29"); fts_tokenizer_register(fts_tokenizer_generic); - tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME); - test_assert(fts_tokenizer_create(tok_class, NULL, tr29_settings, &tok, &error) == 0); + test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0); while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) { test_assert(strcmp(token, *eopp) == 0); eopp++; @@ -169,15 +171,13 @@ "there", "was", "text", "galore", "and", "more", NULL }; - const struct fts_tokenizer *tok_class; struct fts_tokenizer *tok; const char * const *eopp = expected_output; const char *token, *error; test_begin("fts tokenizer generic TR29 with Unicode whitespace"); fts_tokenizer_register(fts_tokenizer_generic); - tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME); - test_assert(fts_tokenizer_create(tok_class, NULL, tr29_settings, &tok, &error) == 0); + test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0); while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) { test_assert(strcmp(token, *eopp) == 0); eopp++; @@ -200,15 +200,13 @@ static const char *const expected_output[] = { "hello", "world", NULL }; - const struct fts_tokenizer *tok_class; struct fts_tokenizer *tok; const char * const *eopp = expected_output; const char *token, *error; test_begin("fts tokenizer generic TR29 with MinNumLet U+FF0E at end"); fts_tokenizer_register(fts_tokenizer_generic); - tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME); - test_assert(fts_tokenizer_create(tok_class, NULL, tr29_settings, &tok, &error) == 0); + test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0); while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) { test_assert(null_strcmp(token, *eopp) == 0); eopp++; @@ -581,6 +579,7 @@ int main(void) { static void (*test_functions[])(void) = { + test_fts_tokenizer_find, test_fts_tokenizer_generic_only, test_fts_tokenizer_generic_unicode_whitespace, test_fts_tokenizer_char_generic_only, From dovecot at dovecot.org Sat May 9 12:02:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 12:02:22 +0000 Subject: dovecot-2.2: lib-fts: Removed supports() function from filter API Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e2a0b43e73ef changeset: 18591:e2a0b43e73ef user: Timo Sirainen date: Sat May 09 15:00:28 2015 +0300 description: lib-fts: Removed supports() function from filter API I think the original idea was that this could be used to automatically create filters for many languages, but this probably won't be needed or wanted. diffstat: src/lib-fts/fts-filter-lowercase.c | 7 ------- src/lib-fts/fts-filter-normalizer-icu.c | 16 +--------------- src/lib-fts/fts-filter-private.h | 1 - src/lib-fts/fts-filter-stemmer-snowball.c | 17 ----------------- src/lib-fts/fts-filter-stopwords.c | 10 ---------- 5 files changed, 1 insertions(+), 50 deletions(-) diffs (149 lines): diff -r 12aeb1ac8f0d -r e2a0b43e73ef src/lib-fts/fts-filter-lowercase.c --- a/src/lib-fts/fts-filter-lowercase.c Sat May 09 14:56:33 2015 +0300 +++ b/src/lib-fts/fts-filter-lowercase.c Sat May 09 15:00:28 2015 +0300 @@ -6,12 +6,6 @@ #include "fts-filter-private.h" #include "fts-language.h" -static bool -fts_filter_lowercase_supports(const struct fts_language *lang ATTR_UNUSED) -{ - return TRUE; -} - static void fts_filter_lowercase_destroy(struct fts_filter *filter) { @@ -47,7 +41,6 @@ } static const struct fts_filter_vfuncs normalizer_filter_vfuncs = { - fts_filter_lowercase_supports, fts_filter_lowercase_create, fts_filter_lowercase_filter, fts_filter_lowercase_destroy diff -r 12aeb1ac8f0d -r e2a0b43e73ef src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 14:56:33 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 15:00:28 2015 +0300 @@ -114,13 +114,6 @@ *_dst = dst; } -static bool fts_filter_normalizer_icu_supports(const struct fts_language *lang) -{ - if (lang == NULL || lang->name == NULL) - return FALSE; - return TRUE; -} - static void fts_filter_normalizer_icu_destroy(struct fts_filter *filter) { struct fts_filter_normalizer *np = @@ -242,19 +235,13 @@ #else -static bool -fts_filter_normalizer_icu_supports(const struct fts_language *lang ATTR_UNUSED) -{ - return FALSE; -} - static int fts_filter_normalizer_icu_create(const struct fts_language *lang ATTR_UNUSED, const char *const *settings ATTR_UNUSED, struct fts_filter **filter_r ATTR_UNUSED, const char **error_r) { - *error_r = "libicu support not built in - can't use "ICU_NORMALIZER_FILTER_NAME; + *error_r = "libicu support not built in - can't use "ICU_NORMALIZER_FILTER_NAME; return -1; } @@ -274,7 +261,6 @@ #endif static const struct fts_filter_vfuncs normalizer_filter_vfuncs = { - fts_filter_normalizer_icu_supports, fts_filter_normalizer_icu_create, fts_filter_normalizer_icu_filter, fts_filter_normalizer_icu_destroy diff -r 12aeb1ac8f0d -r e2a0b43e73ef src/lib-fts/fts-filter-private.h --- a/src/lib-fts/fts-filter-private.h Sat May 09 14:56:33 2015 +0300 +++ b/src/lib-fts/fts-filter-private.h Sat May 09 15:00:28 2015 +0300 @@ -12,7 +12,6 @@ */ struct fts_filter_vfuncs { - bool (*supports)(const struct fts_language *lang); int (*create)(const struct fts_language *lang, const char *const *settings, struct fts_filter **filter_r, diff -r 12aeb1ac8f0d -r e2a0b43e73ef src/lib-fts/fts-filter-stemmer-snowball.c --- a/src/lib-fts/fts-filter-stemmer-snowball.c Sat May 09 14:56:33 2015 +0300 +++ b/src/lib-fts/fts-filter-stemmer-snowball.c Sat May 09 15:00:28 2015 +0300 @@ -16,17 +16,6 @@ struct sb_stemmer *stemmer; }; -static bool -fts_filter_stemmer_snowball_supports(const struct fts_language *lang) -{ - struct sb_stemmer *stemmer = sb_stemmer_new(lang->name, NULL); - if (stemmer != NULL) { - sb_stemmer_delete(stemmer); - return TRUE; - } - return FALSE; -} - static void fts_filter_stemmer_snowball_destroy(struct fts_filter *filter) { struct fts_filter_stemmer_snowball *sp = @@ -105,11 +94,6 @@ #else -static bool -fts_filter_stemmer_snowball_supports(const struct fts_language *lang ATTR_UNUSED) -{ - return FALSE; -} static int fts_filter_stemmer_snowball_create(const struct fts_language *lang ATTR_UNUSED, const char *const *settings ATTR_UNUSED, @@ -134,7 +118,6 @@ #endif static const struct fts_filter_vfuncs snowball_stemmer_filter_vfuncs = { - fts_filter_stemmer_snowball_supports, fts_filter_stemmer_snowball_create, fts_filter_stemmer_snowball_filter, fts_filter_stemmer_snowball_destroy diff -r 12aeb1ac8f0d -r e2a0b43e73ef src/lib-fts/fts-filter-stopwords.c --- a/src/lib-fts/fts-filter-stopwords.c Sat May 09 14:56:33 2015 +0300 +++ b/src/lib-fts/fts-filter-stopwords.c Sat May 09 15:00:28 2015 +0300 @@ -23,15 +23,6 @@ const char *stopwords_dir; }; -/* TODO: Write this function or remove it from api. */ -static bool fts_filter_stopwords_supports(const struct fts_language *lang) -{ - /* TODO: former NULL check is for unit test _fail_create() */ - if (lang == NULL || lang->name == NULL) - return FALSE; - return TRUE; -} - static int fts_filter_stopwords_read_list(struct fts_filter_stopwords *filter, const char **error_r) { @@ -124,7 +115,6 @@ } const struct fts_filter_vfuncs stopwords_filter_vfuncs = { - fts_filter_stopwords_supports, fts_filter_stopwords_create, fts_filter_stopwords_filter, fts_filter_stopwords_destroy From dovecot at dovecot.org Sat May 9 12:09:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 12:09:07 +0000 Subject: dovecot-2.2: lib-fts: fts-tokenizer-generic-private.h had conten... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/00240870d0bd changeset: 18592:00240870d0bd user: Timo Sirainen date: Sat May 09 15:06:45 2015 +0300 description: lib-fts: fts-tokenizer-generic-private.h had content that didn't really belog there. diffstat: src/lib-fts/fts-tokenizer-generic-private.h | 29 ----------------------------- src/lib-fts/fts-tokenizer-generic.c | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 29 deletions(-) diffs (62 lines): diff -r e2a0b43e73ef -r 00240870d0bd src/lib-fts/fts-tokenizer-generic-private.h --- a/src/lib-fts/fts-tokenizer-generic-private.h Sat May 09 15:00:28 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic-private.h Sat May 09 15:06:45 2015 +0300 @@ -48,33 +48,4 @@ buffer_t *token; }; -static bool letter_panic(struct generic_fts_tokenizer *tok); -static bool letter_cr_lf_newline(struct generic_fts_tokenizer *tok); -static bool letter_extend_format(struct generic_fts_tokenizer *tok); -static bool letter_regional_indicator(struct generic_fts_tokenizer *tok); -static bool letter_katakana(struct generic_fts_tokenizer *tok); -static bool letter_hebrew(struct generic_fts_tokenizer *tok); -static bool letter_aletter(struct generic_fts_tokenizer *tok); -static bool letter_single_quote(struct generic_fts_tokenizer *tok); -static bool letter_double_quote(struct generic_fts_tokenizer *tok); -static bool letter_midnumlet(struct generic_fts_tokenizer *tok); -static bool letter_midletter(struct generic_fts_tokenizer *tok); -static bool letter_midnum(struct generic_fts_tokenizer *tok); -static bool letter_numeric(struct generic_fts_tokenizer *tok); -static bool letter_extendnumlet(struct generic_fts_tokenizer *tok); -static bool letter_other(struct generic_fts_tokenizer *tok); - -struct letter_fn { - bool (*fn)(struct generic_fts_tokenizer *tok); -}; -struct letter_fn letter_fns[] = { - {letter_panic}, {letter_cr_lf_newline}, {letter_cr_lf_newline}, - {letter_cr_lf_newline}, {letter_extend_format}, - {letter_regional_indicator}, {letter_extend_format}, - {letter_katakana}, {letter_hebrew}, {letter_aletter}, - {letter_single_quote}, {letter_double_quote}, - {letter_midnumlet}, {letter_midletter}, {letter_midnum}, - {letter_numeric}, {letter_extendnumlet}, {letter_panic}, - {letter_panic}, {letter_other} -}; #endif diff -r e2a0b43e73ef -r 00240870d0bd src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Sat May 09 15:00:28 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Sat May 09 15:06:45 2015 +0300 @@ -496,6 +496,21 @@ tok->prev_letter = LETTER_TYPE_NONE; return 1; } + +struct letter_fn { + bool (*fn)(struct generic_fts_tokenizer *tok); +}; +static struct letter_fn letter_fns[] = { + {letter_panic}, {letter_cr_lf_newline}, {letter_cr_lf_newline}, + {letter_cr_lf_newline}, {letter_extend_format}, + {letter_regional_indicator}, {letter_extend_format}, + {letter_katakana}, {letter_hebrew}, {letter_aletter}, + {letter_single_quote}, {letter_double_quote}, + {letter_midnumlet}, {letter_midletter}, {letter_midnum}, + {letter_numeric}, {letter_extendnumlet}, {letter_panic}, + {letter_panic}, {letter_other} +}; + /* Find word boundaries in input text. Based on Unicode standard annex #29, but tailored for FTS purposes. From dovecot at dovecot.org Sat May 9 12:09:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 12:09:07 +0000 Subject: dovecot-2.2: lib-fts: Minor code cleanups Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7bd196b84518 changeset: 18593:7bd196b84518 user: Timo Sirainen date: Sat May 09 15:07:13 2015 +0300 description: lib-fts: Minor code cleanups diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 2 +- src/lib-fts/fts-filter.c | 8 ++------ src/lib-fts/fts-filter.h | 5 ++--- src/lib-fts/fts-tokenizer-address.c | 20 +++++++------------- src/lib-fts/fts-tokenizer-generic-private.h | 3 +-- src/lib-fts/fts-tokenizer.c | 6 +++--- src/lib-fts/fts-tokenizer.h | 6 +++--- 7 files changed, 19 insertions(+), 31 deletions(-) diffs (157 lines): diff -r 00240870d0bd -r 7bd196b84518 src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 15:06:45 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 15:07:13 2015 +0300 @@ -241,7 +241,7 @@ struct fts_filter **filter_r ATTR_UNUSED, const char **error_r) { - *error_r = "libicu support not built in - can't use "ICU_NORMALIZER_FILTER_NAME; + *error_r = "libicu support not built in - can't use "ICU_NORMALIZER_FILTER_NAME; return -1; } diff -r 00240870d0bd -r 7bd196b84518 src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Sat May 09 15:06:45 2015 +0300 +++ b/src/lib-fts/fts-filter.c Sat May 09 15:07:13 2015 +0300 @@ -90,12 +90,8 @@ fp->v->destroy(fp); } -/* TODO: Avoid multiple allocations by using a buffer in v->filter? - Do this non-recursively? */ -int -fts_filter_filter(struct fts_filter *filter, const char **token, - const char **error_r) - +int fts_filter_filter(struct fts_filter *filter, const char **token, + const char **error_r) { int ret = 0; diff -r 00240870d0bd -r 7bd196b84518 src/lib-fts/fts-filter.h --- a/src/lib-fts/fts-filter.h Sat May 09 15:06:45 2015 +0300 +++ b/src/lib-fts/fts-filter.h Sat May 09 15:07:13 2015 +0300 @@ -58,8 +58,7 @@ out (*token is also set to NULL) and -1 on error. Input is also given via *token. */ -int -fts_filter_filter(struct fts_filter *filter, const char **token, - const char **error_r); +int fts_filter_filter(struct fts_filter *filter, const char **token, + const char **error_r); #endif diff -r 00240870d0bd -r 7bd196b84518 src/lib-fts/fts-tokenizer-address.c --- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 15:06:45 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 15:07:13 2015 +0300 @@ -9,9 +9,6 @@ #define IS_DTEXT(c) \ (rfc822_atext_chars[(int)(unsigned char)(c)] == 2) -#define FTS_DEFAULT_NO_PARENT FALSE -#define FTS_DEFAULT_SEARCH FALSE - enum email_address_parser_state { EMAIL_ADDRESS_PARSER_STATE_NONE = 0, EMAIL_ADDRESS_PARSER_STATE_LOCALPART, @@ -33,7 +30,7 @@ const char **error_r) { struct email_address_fts_tokenizer *tok; - bool search = FTS_DEFAULT_SEARCH; + bool search = FALSE; unsigned int i; for (i = 0; settings[i] != NULL; i += 2) { @@ -120,18 +117,18 @@ size_t *skip_r) { size_t pos = 0; - bool at = FALSE; + bool seen_at = FALSE; while (pos < size && (IS_ATEXT(data[pos]) || data[pos] == '@' || data[pos] == '.')) { if (data[pos] == '@') - at = TRUE; + seen_at = TRUE; pos++; - if (at) + if (seen_at) break; } /* localpart and @ */ - if (at && (pos > 1 || str_len(tok->last_word) > 0)) { + if (seen_at && (pos > 1 || str_len(tok->last_word) > 0)) { str_append_n(tok->last_word, data, pos); *skip_r = pos; return EMAIL_ADDRESS_PARSER_STATE_DOMAIN; @@ -164,12 +161,9 @@ size_t *skip_r) { size_t pos = 0; - const unsigned char *p = data; - while (pos < size && (IS_DTEXT(*p) || *p == '.')) { + while (pos < size && (IS_DTEXT(data[pos]) || data[pos] == '.')) pos++; - p++; - } /* A complete domain name */ if ((pos > 1 && pos < size) || /* non-atext after atext in this data*/ (pos < size && !domain_is_empty(tok))) { /* non-atext after previous atext */ @@ -183,7 +177,7 @@ return EMAIL_ADDRESS_PARSER_STATE_DOMAIN; } /* not a domain. skip past no-good chars. */ - pos += skip_nonlocal_part(p, size - pos); + pos += skip_nonlocal_part(data + pos, size - pos); *skip_r = pos; return EMAIL_ADDRESS_PARSER_STATE_NONE; } diff -r 00240870d0bd -r 7bd196b84518 src/lib-fts/fts-tokenizer-generic-private.h --- a/src/lib-fts/fts-tokenizer-generic-private.h Sat May 09 15:06:45 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic-private.h Sat May 09 15:07:13 2015 +0300 @@ -31,8 +31,7 @@ enum boundary_algorithm { BOUNDARY_ALGORITHM_NONE = 0, BOUNDARY_ALGORITHM_SIMPLE, -#define ALGORITHM_SIMPLE_NAME "simple" /* TODO: could be public in - fts-tokenizer.h */ +#define ALGORITHM_SIMPLE_NAME "simple" BOUNDARY_ALGORITHM_TR29 #define ALGORITHM_TR29_NAME "tr29" }; diff -r 00240870d0bd -r 7bd196b84518 src/lib-fts/fts-tokenizer.c --- a/src/lib-fts/fts-tokenizer.c Sat May 09 15:06:45 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.c Sat May 09 15:07:13 2015 +0300 @@ -162,9 +162,9 @@ tok->v->reset(tok); } -int -fts_tokenizer_next(struct fts_tokenizer *tok, - const unsigned char *data, size_t size, const char **token_r) +int fts_tokenizer_next(struct fts_tokenizer *tok, + const unsigned char *data, size_t size, + const char **token_r) { int ret; diff -r 00240870d0bd -r 7bd196b84518 src/lib-fts/fts-tokenizer.h --- a/src/lib-fts/fts-tokenizer.h Sat May 09 15:06:45 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.h Sat May 09 15:07:13 2015 +0300 @@ -78,9 +78,9 @@ data must contain only valid complete UTF-8 sequences, but otherwise it may be broken into however small pieces. */ -int -fts_tokenizer_next(struct fts_tokenizer *tok, - const unsigned char *data, size_t size, const char **token_r); +int fts_tokenizer_next(struct fts_tokenizer *tok, + const unsigned char *data, size_t size, + const char **token_r); const char *fts_tokenizer_name(const struct fts_tokenizer *tok); From dovecot at dovecot.org Sat May 9 12:13:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 12:13:43 +0000 Subject: dovecot-2.2: fts: Prefer language-specific filter settings over ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b0ac652f9e2f changeset: 18594:b0ac652f9e2f user: Timo Sirainen date: Sat May 09 15:10:17 2015 +0300 description: fts: Prefer language-specific filter settings over global fts_filters setting. diffstat: src/plugins/fts/fts-user.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 7bd196b84518 -r b0ac652f9e2f src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Sat May 09 15:07:13 2015 +0300 +++ b/src/plugins/fts/fts-user.c Sat May 09 15:10:17 2015 +0300 @@ -64,10 +64,12 @@ unsigned int i; int ret = 0; - filters_key = "fts_filters"; + /* try to get the language-specific filters first */ + filters_key = t_strconcat("fts_filters_", lang->name, NULL); str = mail_user_plugin_getenv(user, filters_key); if (str == NULL) { - filters_key = t_strconcat("fts_filters_", lang->name, NULL); + /* fallback to global filters */ + filters_key = "fts_filters"; str = mail_user_plugin_getenv(user, filters_key); if (str == NULL) { *filter_r = NULL; From dovecot at dovecot.org Sat May 9 12:13:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 12:13:43 +0000 Subject: dovecot-2.2: fts: If fts_filters setting doesn't exist, use buil... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/461bb302bd03 changeset: 18595:461bb302bd03 user: Timo Sirainen date: Sat May 09 15:11:48 2015 +0300 description: fts: If fts_filters setting doesn't exist, use built-in defaults. diffstat: src/plugins/fts/fts-user.c | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diffs (33 lines): diff -r b0ac652f9e2f -r 461bb302bd03 src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Sat May 09 15:10:17 2015 +0300 +++ b/src/plugins/fts/fts-user.c Sat May 09 15:11:48 2015 +0300 @@ -11,6 +11,9 @@ #define FTS_USER_CONTEXT(obj) \ MODULE_CONTEXT(obj, fts_user_module) +#define FTS_DEFAULT_TOKENIZERS "generic email-address" +#define FTS_DEFAULT_FILTERS "normalizer-icu snowball" + struct fts_user { union mail_user_module_context module_ctx; @@ -72,8 +75,8 @@ filters_key = "fts_filters"; str = mail_user_plugin_getenv(user, filters_key); if (str == NULL) { - *filter_r = NULL; - return 0; + str = FTS_DEFAULT_FILTERS; + filters_key = "fts_filters(built-in default)"; } } @@ -134,7 +137,7 @@ tokenizers_key = "fts_tokenizers"; str = mail_user_plugin_getenv(user, tokenizers_key); if (str == NULL) - str = "generic email-address"; /* default tokenizers */ + str = FTS_DEFAULT_TOKENIZERS; tokenizers = t_strsplit_spaces(str, " "); From dovecot at dovecot.org Sat May 9 14:00:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 14:00:03 +0000 Subject: dovecot-2.2: fts: Use key=value instead of "key value" settings ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/82831550757b changeset: 18596:82831550757b user: Timo Sirainen date: Sat May 09 16:58:09 2015 +0300 description: fts: Use key=value instead of "key value" settings for tokenizers and filters. diffstat: src/plugins/fts/fts-user.c | 49 ++++++++++++++++++++++++++++++++------------- 1 files changed, 35 insertions(+), 14 deletions(-) diffs (89 lines): diff -r 461bb302bd03 -r 82831550757b src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Sat May 09 15:11:48 2015 +0300 +++ b/src/plugins/fts/fts-user.c Sat May 09 16:58:09 2015 +0300 @@ -26,6 +26,32 @@ static MODULE_CONTEXT_DEFINE_INIT(fts_user_module, &mail_user_module_register); +static const char *const *str_keyvalues_to_array(const char *str) +{ + const char *key, *value, *const *keyvalues; + ARRAY_TYPE(const_string) arr; + unsigned int i; + + if (str == NULL) + return NULL; + + t_array_init(&arr, 8); + keyvalues = t_strsplit_spaces(str, " "); + for (i = 0; keyvalues[i] != NULL; i++) { + value = strchr(keyvalues[i], '='); + if (value != NULL) + key = t_strdup_until(keyvalues[i], value++); + else { + key = keyvalues[i]; + value = ""; + } + array_append(&arr, &key, 1); + array_append(&arr, &value, 1); + } + array_append_zero(&arr); + return array_idx(&arr, 0); +} + static int fts_user_init_languages(struct mail_user *user, struct fts_user *fuser, const char **error_r) @@ -63,7 +89,7 @@ const struct fts_filter *filter_class; struct fts_filter *filter = NULL, *parent = NULL; const char *filters_key, *const *filters, *filter_set_name; - const char *str, *error, *set_key, *const *settings; + const char *str, *error, *set_key; unsigned int i; int ret = 0; @@ -99,9 +125,9 @@ set_key = t_strdup_printf("fts_filters_%s", filter_set_name); str = mail_user_plugin_getenv(user, set_key); } - settings = str == NULL ? NULL : t_strsplit_spaces(str, " "); - if (fts_filter_create(filter_class, parent, lang, settings, + if (fts_filter_create(filter_class, parent, lang, + str_keyvalues_to_array(str), &filter, &error) < 0) { *error_r = t_strdup_printf( "Filter '%s' init via settings '%s' failed: %s", @@ -130,7 +156,7 @@ const struct fts_tokenizer *tokenizer_class; struct fts_tokenizer *tokenizer = NULL, *parent = NULL; const char *tokenizers_key, *const *tokenizers, *tokenizer_set_name; - const char *str, *error, *set_key, *const *settings; + const char *str, *error, *set_key; unsigned int i; int ret = 0; @@ -156,17 +182,12 @@ /* tell the tokenizers that we're tokenizing a search string (instead of tokenizing indexed data) */ - if (search) { - if (str == NULL) - str = "search yes"; - else - str = t_strconcat(str, " search yes", NULL); - } + if (search) + str = t_strconcat("search=yes ", str, NULL); - settings = str == NULL ? NULL : t_strsplit_spaces(str, " "); - - if (fts_tokenizer_create(tokenizer_class, parent, settings, - &tokenizer, &error) < 0) { + if (fts_tokenizer_create(tokenizer_class, parent, + str_keyvalues_to_array(str), + &tokenizer, &error) < 0) { *error_r = t_strdup_printf( "Tokenizer '%s' init via settings '%s' failed: %s", tokenizers[i], set_key, error); From dovecot at dovecot.org Sat May 9 14:07:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 14:07:13 +0000 Subject: dovecot-2.2: fts: Simplify error messages a little bit to be mor... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cfe29a5d66ed changeset: 18597:cfe29a5d66ed user: Timo Sirainen date: Sat May 09 17:00:12 2015 +0300 description: fts: Simplify error messages a little bit to be more consistent with other such errors. Only the setting name is really important. diffstat: src/plugins/fts/fts-user.c | 8 ++------ 1 files changed, 2 insertions(+), 6 deletions(-) diffs (25 lines): diff -r 82831550757b -r cfe29a5d66ed src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Sat May 09 16:58:09 2015 +0300 +++ b/src/plugins/fts/fts-user.c Sat May 09 17:00:12 2015 +0300 @@ -129,9 +129,7 @@ if (fts_filter_create(filter_class, parent, lang, str_keyvalues_to_array(str), &filter, &error) < 0) { - *error_r = t_strdup_printf( - "Filter '%s' init via settings '%s' failed: %s", - filters[i], set_key, error); + *error_r = t_strdup_printf("%s: %s", set_key, error); ret = -1; break; } @@ -188,9 +186,7 @@ if (fts_tokenizer_create(tokenizer_class, parent, str_keyvalues_to_array(str), &tokenizer, &error) < 0) { - *error_r = t_strdup_printf( - "Tokenizer '%s' init via settings '%s' failed: %s", - tokenizers[i], set_key, error); + *error_r = t_strdup_printf("%s: %s", set_key, error); ret = -1; break; } From dovecot at dovecot.org Sat May 9 14:07:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 14:07:13 +0000 Subject: dovecot-2.2: fts: Renamed filter and tokenizer specific setting ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f5cebd75975f changeset: 18598:f5cebd75975f user: Timo Sirainen date: Sat May 09 17:05:17 2015 +0300 description: fts: Renamed filter and tokenizer specific setting keys. Having the plural in the name didn't make a lot of sense, so all the settings are now: fts_filters = name1 name2 fts_filters_ = name3 name 4 fts_filter_ = key1=value1 fts_filter__ = key2=value2 fts_tokenizers = name1 name2 fts_tokenizer_ = key1=value1 So this also removes the confusion in settings if there existed a filter with one of the language names. diffstat: src/plugins/fts/fts-user.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (26 lines): diff -r cfe29a5d66ed -r f5cebd75975f src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Sat May 09 17:00:12 2015 +0300 +++ b/src/plugins/fts/fts-user.c Sat May 09 17:05:17 2015 +0300 @@ -118,11 +118,11 @@ /* try the language-specific setting first */ filter_set_name = t_str_replace(filters[i], '-', '_'); - set_key = t_strdup_printf("fts_filters_%s_%s", + set_key = t_strdup_printf("fts_filter_%s_%s", lang->name, filter_set_name); str = mail_user_plugin_getenv(user, set_key); if (str == NULL) { - set_key = t_strdup_printf("fts_filters_%s", filter_set_name); + set_key = t_strdup_printf("fts_filter_%s", filter_set_name); str = mail_user_plugin_getenv(user, set_key); } @@ -175,7 +175,7 @@ } tokenizer_set_name = t_str_replace(tokenizers[i], '-', '_'); - set_key = t_strdup_printf("fts_tokenizers_%s", tokenizer_set_name); + set_key = t_strdup_printf("fts_tokenizer_%s", tokenizer_set_name); str = mail_user_plugin_getenv(user, set_key); /* tell the tokenizers that we're tokenizing a search string From dovecot at dovecot.org Sat May 9 14:08:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 14:08:35 +0000 Subject: dovecot-2.2: lib-storage: Run each storage hook inside its own d... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/de8561d67b38 changeset: 18599:de8561d67b38 user: Timo Sirainen date: Sat May 09 17:06:42 2015 +0300 description: lib-storage: Run each storage hook inside its own data stack frame. diffstat: src/lib-storage/mail-storage-hooks.c | 32 ++++++++++++++++++-------------- 1 files changed, 18 insertions(+), 14 deletions(-) diffs (112 lines): diff -r f5cebd75975f -r de8561d67b38 src/lib-storage/mail-storage-hooks.c --- a/src/lib-storage/mail-storage-hooks.c Sat May 09 17:05:17 2015 +0300 +++ b/src/lib-storage/mail-storage-hooks.c Sat May 09 17:06:42 2015 +0300 @@ -262,10 +262,10 @@ ctx = hook_build_init((void *)&user->v, sizeof(user->v)); user->vlast = &user->v; array_foreach(&user->hooks, hooks) { - if ((*hooks)->mail_user_created != NULL) { + if ((*hooks)->mail_user_created != NULL) T_BEGIN { (*hooks)->mail_user_created(user); hook_build_update(ctx, user->vlast); - } + } T_END; } pool_unref(&ctx->pool); } @@ -275,8 +275,9 @@ const struct mail_storage_hooks *const *hooks; array_foreach(&ns->user->hooks, hooks) { - if ((*hooks)->mail_namespace_storage_added != NULL) + if ((*hooks)->mail_namespace_storage_added != NULL) T_BEGIN { (*hooks)->mail_namespace_storage_added(ns); + } T_END; } } @@ -285,8 +286,9 @@ const struct mail_storage_hooks *const *hooks; array_foreach(&namespaces->user->hooks, hooks) { - if ((*hooks)->mail_namespaces_created != NULL) + if ((*hooks)->mail_namespaces_created != NULL) T_BEGIN { (*hooks)->mail_namespaces_created(namespaces); + } T_END; } } @@ -295,8 +297,9 @@ const struct mail_storage_hooks *const *hooks; array_foreach(&namespaces->user->hooks, hooks) { - if ((*hooks)->mail_namespaces_added != NULL) + if ((*hooks)->mail_namespaces_added != NULL) T_BEGIN { (*hooks)->mail_namespaces_added(namespaces); + } T_END; } } @@ -308,10 +311,10 @@ ctx = hook_build_init((void *)&storage->v, sizeof(storage->v)); storage->vlast = &storage->v; array_foreach(&storage->user->hooks, hooks) { - if ((*hooks)->mail_storage_created != NULL) { + if ((*hooks)->mail_storage_created != NULL) T_BEGIN { (*hooks)->mail_storage_created(storage); hook_build_update(ctx, storage->vlast); - } + } T_END; } pool_unref(&ctx->pool); } @@ -324,10 +327,10 @@ ctx = hook_build_init((void *)&list->v, sizeof(list->v)); list->vlast = &list->v; array_foreach(&list->ns->user->hooks, hooks) { - if ((*hooks)->mailbox_list_created != NULL) { + if ((*hooks)->mailbox_list_created != NULL) T_BEGIN { (*hooks)->mailbox_list_created(list); hook_build_update(ctx, list->vlast); - } + } T_END; } pool_unref(&ctx->pool); } @@ -340,10 +343,10 @@ ctx = hook_build_init((void *)&box->v, sizeof(box->v)); box->vlast = &box->v; array_foreach(&box->storage->user->hooks, hooks) { - if ((*hooks)->mailbox_allocated != NULL) { + if ((*hooks)->mailbox_allocated != NULL) T_BEGIN { (*hooks)->mailbox_allocated(box); hook_build_update(ctx, box->vlast); - } + } T_END; } pool_unref(&ctx->pool); } @@ -353,8 +356,9 @@ const struct mail_storage_hooks *const *hooks; array_foreach(&box->storage->user->hooks, hooks) { - if ((*hooks)->mailbox_opened != NULL) + if ((*hooks)->mailbox_opened != NULL) T_BEGIN { (*hooks)->mailbox_opened(box); + } T_END; } } @@ -367,10 +371,10 @@ ctx = hook_build_init((void *)&pmail->v, sizeof(pmail->v)); pmail->vlast = &pmail->v; array_foreach(&mail->box->storage->user->hooks, hooks) { - if ((*hooks)->mail_allocated != NULL) { + if ((*hooks)->mail_allocated != NULL) T_BEGIN { (*hooks)->mail_allocated(mail); hook_build_update(ctx, pmail->vlast); - } + } T_END; } pool_unref(&ctx->pool); } From dovecot at dovecot.org Sat May 9 14:09:26 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 14:09:26 +0000 Subject: dovecot-2.2: lib-fts: Fixed compiling without libicu Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/99ad974a3828 changeset: 18600:99ad974a3828 user: Timo Sirainen date: Sat May 09 17:07:32 2015 +0300 description: lib-fts: Fixed compiling without libicu diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r de8561d67b38 -r 99ad974a3828 src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 17:06:42 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Sat May 09 17:07:32 2015 +0300 @@ -241,7 +241,7 @@ struct fts_filter **filter_r ATTR_UNUSED, const char **error_r) { - *error_r = "libicu support not built in - can't use "ICU_NORMALIZER_FILTER_NAME; + *error_r = "libicu support not built in"; return -1; } From dovecot at dovecot.org Sat May 9 15:29:58 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 15:29:58 +0000 Subject: dovecot-2.2: lib-fts: Fixed token truncation. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dd04199a689f changeset: 18601:dd04199a689f user: Timo Sirainen date: Sat May 09 17:34:59 2015 +0300 description: lib-fts: Fixed token truncation. diffstat: src/lib-fts/fts-tokenizer-generic.c | 38 +++++++++++++++++++++++++----------- 1 files changed, 26 insertions(+), 12 deletions(-) diffs (98 lines): diff -r 99ad974a3828 -r dd04199a689f src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Sat May 09 17:07:32 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Sat May 09 17:34:59 2015 +0300 @@ -85,11 +85,20 @@ i_free(tok); } +static const char *fts_uni_strndup(const unsigned char *data, size_t size) +{ + size_t pos; + + /* if input is truncated with a partial UTF-8 character, drop it */ + (void)uni_utf8_partial_strlen_n(data, size, &pos); + return t_strndup(data, pos); +} + static int fts_tokenizer_generic_simple_current_token(struct generic_fts_tokenizer *tok, const char **token_r) { - *token_r = t_strndup(tok->token->data, I_MIN(tok->token->used, tok->max_length)); + *token_r = fts_uni_strndup(tok->token->data, tok->token->used); buffer_set_used_size(tok->token, 0); return 1; } @@ -148,6 +157,15 @@ buffer_set_used_size(tok->token, 0); } +static void tok_append_truncated(struct generic_fts_tokenizer *tok, + const unsigned char *data, size_t size) +{ + i_assert(tok->max_length >= tok->token->used); + + buffer_append(tok->token, data, + I_MIN(size, tok->max_length - tok->token->used)); +} + static int fts_tokenizer_generic_next_simple(struct fts_tokenizer *_tok, const unsigned char *data, size_t size, @@ -161,7 +179,7 @@ char_start_i = i; if (data_is_word_boundary(data, size, &i)) { len = char_start_i - start; - buffer_append(tok->token, data + start, len); + tok_append_truncated(tok, data + start, len); if (tok->token->used == 0) { /* no text read yet */ start = i + 1; @@ -174,16 +192,12 @@ } /* word boundary not found yet */ len = i - start; - buffer_append(tok->token, data + start, len); + tok_append_truncated(tok, data + start, len); *skip_r = i; /* return the last token */ if (size == 0 && tok->token->used > 0) return fts_tokenizer_generic_simple_current_token(tok, token_r); - - /* token too long */ - if (tok->token->used > tok->max_length) - return fts_tokenizer_generic_simple_current_token(tok, token_r); return 0; } @@ -488,9 +502,9 @@ if (is_one_past_end(tok)) end_skip = tok->last_size; - len = I_MIN(tok->token->used, tok->max_length) - end_skip; + len = tok->token->used - end_skip; i_assert(len > 0); - *token_r = t_strndup(tok->token->data, len); + *token_r = fts_uni_strndup(tok->token->data, len); buffer_set_used_size(tok->token, 0); tok->prev_prev_letter = LETTER_TYPE_NONE; tok->prev_letter = LETTER_TYPE_NONE; @@ -575,14 +589,14 @@ } if (uni_found_word_boundary(tok, lt)) { i_assert(char_start_i >= start_skip && size >= start_skip); - buffer_append(tok->token, data + start_skip, - char_start_i - start_skip); + tok_append_truncated(tok, data + start_skip, + char_start_i - start_skip); *skip_r = i + 1; return fts_tokenizer_generic_tr29_current_token(tok, token_r); } } i_assert(i >= start_skip && size >= start_skip); - buffer_append(tok->token, data + start_skip, i - start_skip); + tok_append_truncated(tok, data + start_skip, i - start_skip); *skip_r = i; if (size == 0 && tok->token->used > 0) { From dovecot at dovecot.org Sat May 9 15:29:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 15:29:59 +0000 Subject: dovecot-2.2: lib-fts: Added fts_tokenizer_final() as a convenien... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7542e3be6721 changeset: 18602:7542e3be6721 user: Timo Sirainen date: Sat May 09 18:00:58 2015 +0300 description: lib-fts: Added fts_tokenizer_final() as a convenience wrapper. diffstat: src/lib-fts/fts-tokenizer.c | 5 +++++ src/lib-fts/fts-tokenizer.h | 15 +++++++-------- src/plugins/fts/fts-search-args.c | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diffs (57 lines): diff -r dd04199a689f -r 7542e3be6721 src/lib-fts/fts-tokenizer.c --- a/src/lib-fts/fts-tokenizer.c Sat May 09 17:34:59 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.c Sat May 09 18:00:58 2015 +0300 @@ -196,3 +196,8 @@ i_unreached(); } } + +int fts_tokenizer_final(struct fts_tokenizer *tok, const char **token_r) +{ + return fts_tokenizer_next(tok, NULL, 0, token_r); +} diff -r dd04199a689f -r 7542e3be6721 src/lib-fts/fts-tokenizer.h --- a/src/lib-fts/fts-tokenizer.h Sat May 09 17:34:59 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.h Sat May 09 18:00:58 2015 +0300 @@ -65,22 +65,21 @@ void fts_tokenizer_reset(struct fts_tokenizer *tok); /* - Returns 1 if token was returned, 0 if input was non-blocking and - more data is needed, -1 if EOF/error. - - Returns the next token into *token_r, or NULL if more data is - needed for the next token. + Returns 1 if *token_r was returned, 0 if more data is needed, -1 on error. This function should be called with the same data+size until it - returns 0. When the input is finished, this function should be - still be called with size=0 to flush out the final token(s). + returns 0. After that fts_tokenizer_final() should be called until it + returns 0 to flush out the final token(s). data must contain only valid complete UTF-8 sequences, but otherwise it - may be broken into however small pieces. */ + may be broken into however small pieces. (Input to this function typically + comes from message-decoder, which returns only complete UTF-8 sequences.) */ int fts_tokenizer_next(struct fts_tokenizer *tok, const unsigned char *data, size_t size, const char **token_r); +/* Returns same as fts_tokenizer_next(). */ +int fts_tokenizer_final(struct fts_tokenizer *tok, const char **token_r); const char *fts_tokenizer_name(const struct fts_tokenizer *tok); diff -r dd04199a689f -r 7542e3be6721 src/plugins/fts/fts-search-args.c --- a/src/plugins/fts/fts-search-args.c Sat May 09 17:34:59 2015 +0300 +++ b/src/plugins/fts/fts-search-args.c Sat May 09 18:00:58 2015 +0300 @@ -125,7 +125,7 @@ token) < 0) return -1; } - while (fts_tokenizer_next(tokenizer, NULL, 0, &token) > 0) { + while (fts_tokenizer_final(tokenizer, &token) > 0) { if (fts_backend_dovecot_expand_lang_tokens(languages, pool, and_arg, orig_arg, orig_token, token) < 0) From dovecot at dovecot.org Sat May 9 15:30:04 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 15:30:04 +0000 Subject: dovecot-2.2: lib-fts: Various improvements to test-fts-tokenizer Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e4b62ba0fb5a changeset: 18603:e4b62ba0fb5a user: Timo Sirainen date: Sat May 09 18:28:04 2015 +0300 description: lib-fts: Various improvements to test-fts-tokenizer diffstat: src/lib-fts/test-fts-tokenizer.c | 565 +++++++++----------------------------- 1 files changed, 134 insertions(+), 431 deletions(-) diffs (truncated from 725 to 300 lines): diff -r 7542e3be6721 -r e4b62ba0fb5a src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Sat May 09 18:00:58 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Sat May 09 18:28:04 2015 +0300 @@ -1,16 +1,30 @@ /* Copyright (c) 2014-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" -#include "sha2.h" -#include "hex-binary.h" +#include "unichar.h" #include "test-common.h" #include "fts-tokenizer.h" #include "fts-tokenizer-private.h" -/* TODO: fix including and linking of this. */ -/* #include "fts-tokenizer-generic-private.h" */ +#include "fts-tokenizer-generic-private.h" #include +#define TEST_INPUT_TEXT \ + "hello world\r\n\nAnd there\twas: text galore, " \ + "abc at example.com, " \ + "Bar Baz , " \ + "foo at domain " \ + "1234567890123456789012345678?," \ + "12345678901234567890123456789?," \ + "123456789012345678901234567890?," \ + "and longlonglongabcdefghijklmnopqrstuvwxyz more.\n\n " \ + "(\"Hello world\")3.14 3,14 last" +#define TEST_INPUT_ADDRESS \ + "@invalid invalid@ Abc Dfg , " \ + "Bar Baz " \ + "Foo Bar (comment)foo.bar at host.example.org " \ + "foo, foo at domain" + static void test_fts_tokenizer_find(void) { test_begin("fts tokenizer find"); @@ -19,34 +33,79 @@ test_end(); } +static void +test_tokenizer_inputoutput(struct fts_tokenizer *tok, const char *_input, + const char *const *expected_output) +{ + const unsigned char *input = (const unsigned char *)_input; + const char *token; + unsigned int i, max, outi, char_len, input_len = strlen(_input); + + /* test all input at once */ + outi = 0; + while (fts_tokenizer_next(tok, input, input_len, &token) > 0) { + test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); + outi++; + } + while (fts_tokenizer_next(tok, NULL, 0, &token) > 0) { + test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); + outi++; + } + test_assert(expected_output[outi] == NULL); + + /* test input one byte at a time */ + for (i = outi = 0; i < input_len; i += char_len) { + char_len = uni_utf8_char_bytes(input[i]); + while (fts_tokenizer_next(tok, input+i, char_len, &token) > 0) { + test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); + outi++; + } + } + while (fts_tokenizer_final(tok, &token) > 0) { + test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); + outi++; + } + test_assert(expected_output[outi] == NULL); + + /* test input in random chunks */ + for (i = outi = 0; i < input_len; i += char_len) { + max = rand() % (input_len - i) + 1; + for (char_len = 0; char_len < max; ) + char_len += uni_utf8_char_bytes(input[i+char_len]); + while (fts_tokenizer_next(tok, input+i, char_len, &token) > 0) { + test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); + outi++; + } + } + while (fts_tokenizer_final(tok, &token) > 0) { + test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); + outi++; + } + test_assert(expected_output[outi] == NULL); +} + static void test_fts_tokenizer_generic_only(void) { - static const unsigned char input[] = - "hello world\r\nAnd there\twas: text " - "galore, and longlonglongabcdefghijklmnopqrstuvwxyz more.\n\n (\"Hello world\")last "; + static const char input[] = TEST_INPUT_TEXT; static const char *const expected_output[] = { "hello", "world", "And", "there", "was", "text", "galore", + "abc", "example", "com", "Bar", "Baz", + "bar", "example", "org", "foo", "domain", + "1234567890123456789012345678?", + "12345678901234567890123456789", + "123456789012345678901234567890", "and", "longlonglongabcdefghijklmnopqr", - "more", "Hello", "world", "last", NULL + "more", "Hello", "world", "3", "14", "3", "14", "last", NULL }; struct fts_tokenizer *tok; - const char * const *eopp = expected_output; - const char *token, *error; + const char *error; test_begin("fts tokenizer generic simple"); test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &tok, &error) == 0); -/*TODO: Uncomment when fts-tokenizer-generic-private.h inclusion is fixed */ -/*test_assert(((struct generic_fts_tokenizer *) tok)->algorithm == BOUNDARY_ALGORITHM_SIMPLE);*/ - while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) { - test_assert(strcmp(token, *eopp) == 0); - eopp++; - } - while (fts_tokenizer_next(tok, NULL, 0, &token) > 0) { - test_assert(strcmp(token, *eopp) == 0); - eopp++; - } - test_assert(*eopp == NULL); + test_assert(((struct generic_fts_tokenizer *) tok)->algorithm == BOUNDARY_ALGORITHM_SIMPLE); + + test_tokenizer_inputoutput(tok, input, expected_output); fts_tokenizer_unref(&tok); test_end(); } @@ -55,7 +114,7 @@ { /* with Unicode(utf8) U+FF01(ef bc 81)(U+2000(e2 80 80) and U+205A(e2 81 9a) and U+205F(e2 81 9f )*/ - static const unsigned char input[] = + static const char input[] = "hello\xEF\xBC\x81world\r\nAnd\xE2\x80\x80there\twas: text " "galore\xE2\x81\x9F""and\xE2\x81\x9Amore.\n\n"; static const char *const expected_output[] = { @@ -64,61 +123,12 @@ "and", "more", NULL }; struct fts_tokenizer *tok; - const char * const *eopp = expected_output; - const char *token, *error; + const char *error; test_begin("fts tokenizer generic simple with Unicode whitespace"); - fts_tokenizer_register(fts_tokenizer_generic); test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &tok, &error) == 0); - while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) { - test_assert(strcmp(token, *eopp) == 0); - eopp++; - } - while (fts_tokenizer_next(tok, NULL, 0, &token) > 0) { - test_assert(strcmp(token, *eopp) == 0); - eopp++; - } - test_assert(*eopp == NULL); + test_tokenizer_inputoutput(tok, input, expected_output); fts_tokenizer_unref(&tok); - fts_tokenizer_unregister(fts_tokenizer_generic); - test_end(); -} - -static void test_fts_tokenizer_char_generic_only(void) -{ - static const unsigned char input[] = - "abc at example.com, " - "Bar Baz , " - "foo at domain"; - static const char *const expected_output[] = { - "abc", "example", "com", "Bar", "Baz", - "bar", "example", "org", "foo", "domain", NULL - }; - struct fts_tokenizer *tok; - const char * const *eopp = expected_output; - const char *token, *error; - unsigned int i; - int ret; - - test_begin("fts tokenizer generic simple input one character at a time"); - fts_tokenizer_register(fts_tokenizer_generic); - - test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &tok, &error) == 0); - - for (i = 0; i <= sizeof(input)-1; ) { - ret = i < sizeof(input)-1 ? - fts_tokenizer_next(tok, &input[i], 1, &token) : - fts_tokenizer_next(tok, NULL, 0, &token); - if (ret == 0) { - i++; - continue; - } - test_assert(null_strcmp(token, *eopp) == 0); - eopp++; - } - test_assert(*eopp == NULL); - fts_tokenizer_unref(&tok); - fts_tokenizer_unregister(fts_tokenizer_generic); test_end(); } @@ -126,34 +136,25 @@ static void test_fts_tokenizer_generic_tr29_only(void) { - static const unsigned char input[] = - "hello world\r\n\nAnd there\twas: text " - "galore, and more.\n\n (\"Hello world\")3.14 3,14 last" - " longlonglongabcdefghijklmnopqrstuvwxyz 1."; + static const char input[] = TEST_INPUT_TEXT; static const char *const expected_output[] = { "hello", "world", "And", "there", "was", "text", "galore", - "and", "more", "Hello", "world", "3.14", - "3,14", "last", "longlonglongabcdefghijklmnopqr", "1", NULL + "abc", "example.com", "Bar", "Baz", + "bar", "example.org", "foo", "domain", + "1234567890123456789012345678?", + "12345678901234567890123456789", + "123456789012345678901234567890", + "and", "longlonglongabcdefghijklmnopqr", + "more", "Hello", "world", "3.14", "3,14", "last", NULL }; struct fts_tokenizer *tok; - const char * const *eopp = expected_output; - const char *token, *error; + const char *error; test_begin("fts tokenizer generic TR29"); - fts_tokenizer_register(fts_tokenizer_generic); test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0); - while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) { - test_assert(strcmp(token, *eopp) == 0); - eopp++; - } - while (fts_tokenizer_next(tok, NULL, 0, &token) > 0) { - test_assert(strcmp(token, *eopp) == 0); - eopp++; - } - test_assert(*eopp == NULL); + test_tokenizer_inputoutput(tok, input, expected_output); fts_tokenizer_unref(&tok); - fts_tokenizer_unregister(fts_tokenizer_generic); test_end(); } @@ -163,7 +164,7 @@ { /* with Unicode(utf8) U+2000(e2 80 80) and U+205A(e2 81 9a) and U+205F(e2 81 9f)*/ - static const unsigned char input[] = + static const char input[] = "hello world\r\nAnd\xE2\x80\x80there\twas: text " "galore\xE2\x81\x9F""and\xE2\x81\x9Amore.\n\n"; static const char *const expected_output[] = { @@ -172,404 +173,112 @@ "and", "more", NULL }; struct fts_tokenizer *tok; - const char * const *eopp = expected_output; - const char *token, *error; + const char *error; test_begin("fts tokenizer generic TR29 with Unicode whitespace"); - fts_tokenizer_register(fts_tokenizer_generic); test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0); - while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) { - test_assert(strcmp(token, *eopp) == 0); - eopp++; - } - while (fts_tokenizer_next(tok, NULL, 0, &token) > 0) { - test_assert(strcmp(token, *eopp) == 0); - eopp++; - } - test_assert(*eopp == NULL); + test_tokenizer_inputoutput(tok, input, expected_output); fts_tokenizer_unref(&tok); - fts_tokenizer_unregister(fts_tokenizer_generic); test_end(); } static void test_fts_tokenizer_generic_tr29_midnumlet_end(void) { /* u+FF0E is EF BC 8E */ - static const unsigned char input[] = + static const char input[] = "hello world\xEF\xBC\x8E"; static const char *const expected_output[] = { "hello", "world", NULL }; struct fts_tokenizer *tok; - const char * const *eopp = expected_output; - const char *token, *error; From dovecot at dovecot.org Sat May 9 15:36:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 15:36:09 +0000 Subject: dovecot-2.2: man: Added description of option `-F file'. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c469d8f4cde7 changeset: 18604:c469d8f4cde7 user: Pascal Volk date: Sat May 09 15:29:21 2015 +0000 description: man: Added description of option `-F file'. diffstat: doc/man/Makefile.am | 1 + doc/man/doveadm-acl.1.in | 22 ++++++++++++---------- doc/man/doveadm-altmove.1.in | 15 +++++++++++++-- doc/man/doveadm-batch.1.in | 17 +++++++++++++++-- doc/man/doveadm-deduplicate.1.in | 8 ++++++-- doc/man/doveadm-expunge.1.in | 18 ++++++++++++++++-- doc/man/doveadm-fetch.1.in | 11 ++++++++++- doc/man/doveadm-flags.1.in | 21 +++++++++++++++++---- doc/man/doveadm-force-resync.1.in | 9 ++++++++- doc/man/doveadm-fts.1.in | 8 +++++--- doc/man/doveadm-import.1.in | 16 ++++++++++++++-- doc/man/doveadm-index.1.in | 11 ++++++++++- doc/man/doveadm-mailbox.1.in | 18 ++++++++++-------- doc/man/doveadm-move.1.in | 29 +++++++++++++++++++++++++++-- doc/man/doveadm-purge.1.in | 15 +++++++++++++-- doc/man/doveadm-quota.1.in | 20 +++++++++++++++++--- doc/man/doveadm-search.1.in | 17 +++++++++++++++-- doc/man/doveadm-sync.1.in | 8 +++++--- doc/man/option-F-file.inc | 15 +++++++++++++++ doc/man/option-u-user.inc | 4 +++- doc/man/sed.sh | 4 ++++ 21 files changed, 236 insertions(+), 51 deletions(-) diffs (truncated from 904 to 300 lines): diff -r e4b62ba0fb5a -r c469d8f4cde7 doc/man/Makefile.am --- a/doc/man/Makefile.am Sat May 09 18:28:04 2015 +0300 +++ b/doc/man/Makefile.am Sat May 09 15:29:21 2015 +0000 @@ -56,6 +56,7 @@ $(srcdir)/global-options-formatter.inc \ $(srcdir)/global-options.inc \ $(srcdir)/option-A.inc \ + $(srcdir)/option-F-file.inc \ $(srcdir)/option-S-socket.inc \ $(srcdir)/option-u-user.inc \ $(srcdir)/reporting-bugs.inc diff -r e4b62ba0fb5a -r c469d8f4cde7 doc/man/doveadm-acl.1.in --- a/doc/man/doveadm-acl.1.in Sat May 09 18:28:04 2015 +0300 +++ b/doc/man/doveadm-acl.1.in Sat May 09 15:29:21 2015 +0000 @@ -1,5 +1,5 @@ .\" Copyright (c) 2014-2015 Dovecot authors, see the included COPYING file -.TH DOVEADM\-ACL 1 "2014-10-09" "Dovecot v2.2" "Dovecot" +.TH DOVEADM\-ACL 1 "2015-05-09" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-acl \- Manage Access Control List (ACL) .\"------------------------------------------------------------------------ @@ -26,6 +26,8 @@ .\"------------------------------------- @INCLUDE:option-A@ .\"------------------------------------- + at INCLUDE:option-F-file@ +.\"------------------------------------- @INCLUDE:option-S-socket@ .\"------------------------------------- @INCLUDE:option-u-user@ @@ -80,7 +82,7 @@ .fi .PP Now if timo is a member of the tempdisabled group, he has no access to the -mailbox. +mailbox. This wouldn\(aqt be possible with a normal group identifier, because the .B user=timo would override it. @@ -171,7 +173,7 @@ .SH COMMANDS .SS acl add .B doveadm acl add -[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-u\fP \fIuser\fP|\fB\-A\fP|\fB\-F\fP \fIfile\fP] [\fB\-S\fP \fIsocket_path\fP] .I mailbox id right .RI [ right " ...]" @@ -184,7 +186,7 @@ .\"------------------------------------- .SS acl debug .B doveadm acl debug -[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-u\fP \fIuser\fP|\fB\-A\fP|\fB\-F\fP \fIfile\fP] [\fB\-S\fP \fIsocket_path\fP] .I mailbox .PP @@ -194,7 +196,7 @@ .\"------------------------------------- .SS acl delete .B doveadm acl delete -[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-u\fP \fIuser\fP|\fB\-A\fP|\fB\-F\fP \fIfile\fP] [\fB\-S\fP \fIsocket_path\fP] .I mailbox id .PP @@ -203,7 +205,7 @@ .\"------------------------------------- .SS acl get .B doveadm acl get -[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-u\fP \fIuser\fP|\fB\-A\fP|\fB\-F\fP \fIfile\fP] [\fB\-S\fP \fIsocket_path\fP] .RB [ \-m ] .I mailbox @@ -213,7 +215,7 @@ .\"------------------------------------- .SS acl recalc .B doveadm acl recalc -[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-u\fP \fIuser\fP|\fB\-A\fP|\fB\-F\fP \fIfile\fP] [\fB\-S\fP \fIsocket_path\fP] .PP Make sure the @@ -223,7 +225,7 @@ .\"------------------------------------- .SS acl remove .B doveadm acl remove -[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-u\fP \fIuser\fP|\fB\-A\fP|\fB\-F\fP \fIfile\fP] [\fB\-S\fP \fIsocket_path\fP] .I mailbox id right .RI [ right " ...]" @@ -234,7 +236,7 @@ .\"------------------------------------- .SS acl rights .B doveadm acl rights -[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-u\fP \fIuser\fP|\fB\-A\fP|\fB\-F\fP \fIfile\fP] [\fB\-S\fP \fIsocket_path\fP] .I mailbox .PP @@ -245,7 +247,7 @@ .\"------------------------------------- .SS acl set .B doveadm acl set -[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-u\fP \fIuser\fP|\fB\-A\fP|\fB\-F\fP \fIfile\fP] [\fB\-S\fP \fIsocket_path\fP] .I mailbox id right .RI [ right " ...]" diff -r e4b62ba0fb5a -r c469d8f4cde7 doc/man/doveadm-altmove.1.in --- a/doc/man/doveadm-altmove.1.in Sat May 09 18:28:04 2015 +0300 +++ b/doc/man/doveadm-altmove.1.in Sat May 09 15:29:21 2015 +0000 @@ -1,5 +1,5 @@ .\" Copyright (c) 2010-2015 Dovecot authors, see the included COPYING file -.TH DOVEADM\-ALTMOVE 1 "2014-10-19" "Dovecot v2.2" "Dovecot" +.TH DOVEADM\-ALTMOVE 1 "2015-05-09" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-altmove \- Move matching mails to the alternative storage (dbox\-only) .\"------------------------------------------------------------------------ @@ -15,6 +15,11 @@ .\"------------------------------------- .BR doveadm " [" \-Dv "] " altmove " [" \-r "] ["\-S .IR socket_path "] " +.BI \-F " file search_query" +.br +.\"------------------------------------- +.BR doveadm " [" \-Dv "] " altmove " [" \-r "] ["\-S +.IR socket_path "] " .BI \-u " user search_query" .\"------------------------------------------------------------------------ .SH DESCRIPTION @@ -30,7 +35,11 @@ .PP In the second form, the command will be performed for all users. .PP -In the third form, only matching mails of the given +In the third form, the command will be performed for all users listed in +the given +.IR file . +.PP +In the fourth form, only matching mails of the given .IR user (s) will be moved to the alternative storage. .\"------------------------------------------------------------------------ @@ -42,6 +51,8 @@ .\"------------------------------------- @INCLUDE:option-A@ .\"------------------------------------- + at INCLUDE:option-F-file@ +.\"------------------------------------- .TP .B \-r When the diff -r e4b62ba0fb5a -r c469d8f4cde7 doc/man/doveadm-batch.1.in --- a/doc/man/doveadm-batch.1.in Sat May 09 18:28:04 2015 +0300 +++ b/doc/man/doveadm-batch.1.in Sat May 09 15:29:21 2015 +0000 @@ -1,5 +1,5 @@ .\" Copyright (c) 2013-2015 Dovecot authors, see the included COPYING file -.TH DOVEADM\-BATCH 1 "2014-10-19" "Dovecot v2.2" "Dovecot" +.TH DOVEADM\-BATCH 1 "2015-05-09" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-batch \- Execute multiple commands for multiple users .\"------------------------------------------------------------------------ @@ -12,6 +12,12 @@ .\"------------------------------------- .BR doveadm " [" \-Dv "] " batch " [" \-S .IR socket_path "] " +.BI \-F " file sep command sep command" +[...] +.br +.\"------------------------------------- +.BR doveadm " [" \-Dv "] " batch " [" \-S +.IR socket_path "] " .BI \-u " usermask sep command sep command" [...] .\"------------------------------------------------------------------------ @@ -34,7 +40,12 @@ .IR command s for each of them. .PP -In the second form the +In the second form +.BR doveadm (1) +will loop over all users, listed in the given +.IR file . +.PP +In the third form the .IR command s will be executed for each user matching the given .IR usermask . @@ -47,6 +58,8 @@ .\"------------------------------------- @INCLUDE:option-A@ .\"------------------------------------- + at INCLUDE:option-F-file@ +.\"------------------------------------- @INCLUDE:option-S-socket@ .\"------------------------------------- @INCLUDE:option-u-user@ diff -r e4b62ba0fb5a -r c469d8f4cde7 doc/man/doveadm-deduplicate.1.in --- a/doc/man/doveadm-deduplicate.1.in Sat May 09 18:28:04 2015 +0300 +++ b/doc/man/doveadm-deduplicate.1.in Sat May 09 15:29:21 2015 +0000 @@ -1,12 +1,14 @@ .\" Copyright (c) 2013-2015 Dovecot authors, see the included COPYING file -.TH DOVEADM\-DEDUPLICATE 1 "2014-10-19" "Dovecot v2.2" "Dovecot" +.TH DOVEADM\-DEDUPLICATE 1 "2015-05-09" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-deduplicate \- expunge duplicate messages .\"------------------------------------------------------------------------ .SH SYNOPSIS .BR doveadm " [" \-Dv "] " deduplicate " [" \-u .IR user |\c -.BR \-A "] [" \-S +.BR \-A |\c +.BI \-F " file" \c +.RB "] [" \-S .IR socket_path "] ["\c .BR \-m ] .I search_query @@ -31,6 +33,8 @@ .\"------------------------------------- @INCLUDE:option-A@ .\"------------------------------------- + at INCLUDE:option-F-file@ +.\"------------------------------------- @INCLUDE:option-S-socket@ .\"------------------------------------- .TP diff -r e4b62ba0fb5a -r c469d8f4cde7 doc/man/doveadm-expunge.1.in --- a/doc/man/doveadm-expunge.1.in Sat May 09 18:28:04 2015 +0300 +++ b/doc/man/doveadm-expunge.1.in Sat May 09 15:29:21 2015 +0000 @@ -1,5 +1,5 @@ .\" Copyright (c) 2010-2015 Dovecot authors, see the included COPYING file -.TH DOVEADM\-EXPUNGE 1 "2014-10-19" "Dovecot v2.2" "Dovecot" +.TH DOVEADM\-EXPUNGE 1 "2015-05-09" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-expunge \- Expunge messages matching given search query .\"------------------------------------------------------------------------ @@ -19,6 +19,12 @@ .BR doveadm " [" \-Dv "] " expunge " [" \-S .IR socket_path ] .RB [ \-d ] +.BI \-F " file search_query" +.br +.\"------------------------------------- +.BR doveadm " [" \-Dv "] " expunge " [" \-S +.IR socket_path ] +.RB [ \-d ] .BI \-u " user search_query" .\"------------------------------------------------------------------------ .SH DESCRIPTION @@ -39,7 +45,13 @@ .PP In the second form, the command will be performed for all users. .PP -In the third form, only matching mails of the given +In the third form, +.BR doveadm (1) +will expunge messages of the users listed +in the given +.IR file . +.PP +In the fourth form, only matching mails of the given .IR user (s) will be expunged. .\"------------------------------------------------------------------------ @@ -51,6 +63,8 @@ .\"------------------------------------- @INCLUDE:option-A@ .\"------------------------------------- + at INCLUDE:option-F-file@ +.\"------------------------------------- .TP .B \-d Delete the mailbox if it is empty after expunging. diff -r e4b62ba0fb5a -r c469d8f4cde7 doc/man/doveadm-fetch.1.in --- a/doc/man/doveadm-fetch.1.in Sat May 09 18:28:04 2015 +0300 +++ b/doc/man/doveadm-fetch.1.in Sat May 09 15:29:21 2015 +0000 @@ -1,5 +1,5 @@ .\" Copyright (c) 2010-2015 Dovecot authors, see the included COPYING file -.TH DOVEADM\-FETCH 1 "2012-02-13" "Dovecot v2.2" "Dovecot" +.TH DOVEADM\-FETCH 1 "2015-05-09" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-fetch \- Fetch partial/full messages or message information .\"------------------------------------------------------------------------ @@ -21,6 +21,13 @@ .IR formatter ] .BR fetch " [" \-S .IR socket_path "]" +.BI \-F " file fields search_query" +.br +.\"------------------------------------- +.BR doveadm " [" \-Dv "] [" \-f +.IR formatter ] +.BR fetch " [" \-S +.IR socket_path "]" .BI \-u " user fields search_query" .\"------------------------------------------------------------------------ From dovecot at dovecot.org Sat May 9 15:54:57 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 15:54:57 +0000 Subject: dovecot-2.2: lib-compression: Fixed dependency tracking in Makefile Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ceb9c13eb3c0 changeset: 18605:ceb9c13eb3c0 user: Timo Sirainen date: Sat May 09 18:53:02 2015 +0300 description: lib-compression: Fixed dependency tracking in Makefile diffstat: src/lib-compression/Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r c469d8f4cde7 -r ceb9c13eb3c0 src/lib-compression/Makefile.am --- a/src/lib-compression/Makefile.am Sat May 09 15:29:21 2015 +0000 +++ b/src/lib-compression/Makefile.am Sat May 09 18:53:02 2015 +0300 @@ -37,7 +37,7 @@ libfs_compress_la_SOURCES = fs-compress.c libfs_compress_la_LIBADD = libdovecot-compression.la -libfs_compress_la_DEPS = libdovecot-compression.la +libfs_compress_la_DEPENDENCIES = libdovecot-compression.la libfs_compress_la_LDFLAGS = -module -avoid-version test_programs = \ From dovecot at dovecot.org Sat May 9 16:02:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 16:02:34 +0000 Subject: dovecot-2.2: lib-fs: Minor code cleanup Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/24c2adaded28 changeset: 18606:24c2adaded28 user: Timo Sirainen date: Sat May 09 19:00:36 2015 +0300 description: lib-fs: Minor code cleanup Both temp_output and super_output are NULL so it doesn't matter which we use, but this change makes it use the intended variable. diffstat: src/lib-fs/fs-metawrap.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r ceb9c13eb3c0 -r 24c2adaded28 src/lib-fs/fs-metawrap.c --- a/src/lib-fs/fs-metawrap.c Sat May 09 18:53:02 2015 +0300 +++ b/src/lib-fs/fs-metawrap.c Sat May 09 19:00:36 2015 +0300 @@ -389,7 +389,7 @@ if (file->temp_output == NULL) { /* finishing up */ i_assert(file->super_output == NULL); - return fs_write_stream_finish(file->super, &file->super_output); + return fs_write_stream_finish(file->super, &file->temp_output); } /* finish writing the temporary file */ input = iostream_temp_finish(&file->temp_output, IO_BLOCK_SIZE); From dovecot at dovecot.org Sat May 9 16:18:49 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 16:18:49 +0000 Subject: dovecot-2.2: virtual: Don't crash if trying to save mail to a vi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0a1726e7015b changeset: 18607:0a1726e7015b user: Timo Sirainen date: Sat May 09 19:04:11 2015 +0300 description: virtual: Don't crash if trying to save mail to a virtual mailbox without configured save-mailbox. Found by Coverity diffstat: src/plugins/virtual/virtual-save.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 24c2adaded28 -r 0a1726e7015b src/plugins/virtual/virtual-save.c --- a/src/plugins/virtual/virtual-save.c Sat May 09 19:00:36 2015 +0300 +++ b/src/plugins/virtual/virtual-save.c Sat May 09 19:04:11 2015 +0300 @@ -46,8 +46,8 @@ virtual_transaction_get(_t, mbox->save_bbox->box); ctx->backend_save_ctx = mailbox_save_alloc(backend_trans); } + virtual_backend_box_accessed(mbox, mbox->save_bbox); } - virtual_backend_box_accessed(mbox, mbox->save_bbox); return _t->save_ctx; } From dovecot at dovecot.org Sat May 9 16:18:54 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 16:18:54 +0000 Subject: dovecot-2.2: lib-fts: fts_language_list_init() API changed to re... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1fc7ae2640b0 changeset: 18608:1fc7ae2640b0 user: Timo Sirainen date: Sat May 09 19:14:07 2015 +0300 description: lib-fts: fts_language_list_init() API changed to return errors. diffstat: src/lib-fts/fts-language.c | 22 +++++++++++----------- src/lib-fts/fts-language.h | 6 +++++- src/lib-fts/test-fts-language.c | 35 ++++++++++++++--------------------- src/plugins/fts/fts-user.c | 3 ++- 4 files changed, 32 insertions(+), 34 deletions(-) diffs (181 lines): diff -r 0a1726e7015b -r 1fc7ae2640b0 src/lib-fts/fts-language.c --- a/src/lib-fts/fts-language.c Sat May 09 19:04:11 2015 +0300 +++ b/src/lib-fts/fts-language.c Sat May 09 19:14:07 2015 +0300 @@ -51,26 +51,25 @@ return NULL; } -struct fts_language_list * -fts_language_list_init(const char *const *settings) +int fts_language_list_init(const char *const *settings, + struct fts_language_list **list_r, + const char **error_r) { struct fts_language_list *lp; pool_t pool; unsigned int i; - const char *conf = NULL; - const char *data = NULL; + const char *conf = NULL, *data = NULL; for (i = 0; settings[i] != NULL; i += 2) { const char *key = settings[i], *value = settings[i+1]; - if (strcmp(key, "fts_language_config") == 0) { + if (strcmp(key, "fts_language_config") == 0) conf = value; - } - else if (strcmp(key, "fts_language_data") == 0) { + else if (strcmp(key, "fts_language_data") == 0) data = value; - } else { - i_debug("Unknown setting: %s", key); - return NULL; + else { + *error_r = t_strdup_printf("Unknown setting: %s", key); + return -1; } } @@ -86,7 +85,8 @@ else lp->textcat_datadir = NULL; p_array_init(&lp->languages, pool, 32); - return lp; + *list_r = lp; + return 0; } void fts_language_list_deinit(struct fts_language_list **list) diff -r 0a1726e7015b -r 1fc7ae2640b0 src/lib-fts/fts-language.h --- a/src/lib-fts/fts-language.h Sat May 09 19:04:11 2015 +0300 +++ b/src/lib-fts/fts-language.h Sat May 09 19:14:07 2015 +0300 @@ -1,6 +1,8 @@ #ifndef FTS_LANGUAGE_H #define FTS_LANGUAGE_H +struct fts_language_list; + enum fts_language_result { /* Provided sample is too short. */ FTS_LANGUAGE_RESULT_SHORT, @@ -26,7 +28,9 @@ supported languages. */ const struct fts_language *fts_language_find(const char *name); -struct fts_language_list *fts_language_list_init(const char *const *settings); +int fts_language_list_init(const char *const *settings, + struct fts_language_list **list_r, + const char **error_r); void fts_language_list_deinit(struct fts_language_list **list); /* Add a language to the list of wanted languages. */ diff -r 0a1726e7015b -r 1fc7ae2640b0 src/lib-fts/test-fts-language.c --- a/src/lib-fts/test-fts-language.c Sat May 09 19:04:11 2015 +0300 +++ b/src/lib-fts/test-fts-language.c Sat May 09 19:14:07 2015 +0300 @@ -24,10 +24,9 @@ "vastaan. Kahdeksan maata pid\xC3\xA4ttyi "\ "\xC3\xA4\xC3\xA4nest\xC3\xA4m\xC3\xA4st\xC3\xA4."; const char names[] = "de, fi, en"; - const char *unknown; + const char *unknown, *error; test_begin("fts language detect Finnish"); - lp = fts_language_list_init(settings); - test_assert(lp != NULL); + test_assert(fts_language_list_init(settings, &lp, &error) == 0); test_assert(fts_language_list_add_names(lp, names, &unknown) == TRUE); test_assert(fts_language_detect(lp, finnish, sizeof(finnish)-1, &lang_r) == FTS_LANGUAGE_RESULT_OK); @@ -52,10 +51,9 @@ "of the common people, "; const char names[] = "fi, de, fr, en"; - const char *unknown; + const char *unknown, *error; test_begin("fts language detect English"); - lp = fts_language_list_init(settings); - test_assert(lp != NULL); + test_assert(fts_language_list_init(settings, &lp, &error) == 0); test_assert(fts_language_list_add_names(lp, names, &unknown) == TRUE); test_assert(fts_language_detect(lp, english, sizeof(english)-1, &lang_r) == FTS_LANGUAGE_RESULT_OK); @@ -88,10 +86,9 @@ const char names[] = "de, fi, fr, en"; - const char *unknown; + const char *unknown, *error; test_begin("fts language detect French"); - lp = fts_language_list_init(settings); - test_assert(lp != NULL); + test_assert(fts_language_list_init(settings, &lp, &error) == 0); test_assert(fts_language_list_add_names(lp, names, &unknown) == TRUE); test_assert(fts_language_detect(lp, french, sizeof(french)-1, &lang_r) == FTS_LANGUAGE_RESULT_OK); @@ -126,10 +123,9 @@ const char names[] = "fi, de, fr, en"; - const char *unknown; + const char *unknown, *error; test_begin("fts language detect German"); - lp = fts_language_list_init(settings); - test_assert(lp != NULL); + test_assert(fts_language_list_init(settings, &lp, &error) == 0); test_assert(fts_language_list_add_names(lp, names, &unknown) == TRUE); test_assert(fts_language_detect(lp, german, sizeof(german)-1, &lang_r) == FTS_LANGUAGE_RESULT_OK); @@ -152,10 +148,9 @@ "vastaan. Kahdeksan maata pid\xC3\xA4ttyi "\ "\xC3\xA4\xC3\xA4nest\xC3\xA4m\xC3\xA4st\xC3\xA4."; const char names[] = "en"; - const char *unknown; + const char *unknown, *error; test_begin("fts language detect Finnish as English"); - lp = fts_language_list_init(settings); - test_assert(lp != NULL); + test_assert(fts_language_list_init(settings, &lp, &error) == 0); test_assert(fts_language_list_add_names(lp, names, &unknown) == TRUE); test_assert(fts_language_detect(lp, finnish, sizeof(finnish)-1, &lang_r) == FTS_LANGUAGE_RESULT_OK); @@ -180,10 +175,9 @@ "of the common people, "; const char names[] = "fi, de, fr"; - const char *unknown; + const char *unknown, *error; test_begin("fts language detect not available"); - lp = fts_language_list_init(settings); - test_assert(lp != NULL); + test_assert(fts_language_list_init(settings, &lp, &error) == 0); test_assert(fts_language_list_add_names(lp, names, &unknown) == TRUE); test_assert(fts_language_detect(lp, english, sizeof(english)-1, &lang_r) == FTS_LANGUAGE_RESULT_UNKNOWN); @@ -201,10 +195,9 @@ "SeH'eghtaHghach'a'na'chajmo'."; const char names[] = "fi, de, fr"; - const char *unknown; + const char *unknown, *error; test_begin("fts language detect unknown"); - lp = fts_language_list_init(settings); - test_assert(lp != NULL); + test_assert(fts_language_list_init(settings, &lp, &error) == 0); test_assert(fts_language_list_add_names(lp, names, &unknown) == TRUE); test_assert(fts_language_detect(lp, klingon, sizeof(klingon), &lang_r) == FTS_LANGUAGE_RESULT_UNKNOWN); diff -r 0a1726e7015b -r 1fc7ae2640b0 src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Sat May 09 19:04:11 2015 +0300 +++ b/src/plugins/fts/fts-user.c Sat May 09 19:14:07 2015 +0300 @@ -66,9 +66,10 @@ } lang_config[1] = mail_user_plugin_getenv(user, "fts_language_config"); - fuser->lang_list = fts_language_list_init(lang_config); if (lang_config[1] != NULL) lang_config[0] = "fts_language_config"; + if (fts_language_list_init(lang_config, &fuser->lang_list, error_r) < 0) + return -1; if (!fts_language_list_add_names(fuser->lang_list, languages, &unknown)) { *error_r = t_strdup_printf( From dovecot at dovecot.org Sat May 9 16:23:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 16:23:39 +0000 Subject: dovecot-2.2: lib-fts: Minor code cleanup - avoid functions alway... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fa55a06ffae2 changeset: 18609:fa55a06ffae2 user: Timo Sirainen date: Sat May 09 19:21:45 2015 +0300 description: lib-fts: Minor code cleanup - avoid functions always returning same value Makes it clearer to see in the code that the function can't return any failures. diffstat: src/lib-fts/fts-tokenizer-address.c | 18 ++++++++++-------- src/lib-fts/fts-tokenizer-generic.c | 22 ++++++++++++---------- 2 files changed, 22 insertions(+), 18 deletions(-) diffs (141 lines): diff -r 1fc7ae2640b0 -r fa55a06ffae2 src/lib-fts/fts-tokenizer-address.c --- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 19:14:07 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 19:21:45 2015 +0300 @@ -63,14 +63,13 @@ i_free(tok); } -static int +static void fts_tokenizer_address_current_token(struct email_address_fts_tokenizer *tok, const char **token_r) { tok->tokenizer.skip_parents = TRUE; tok->state = EMAIL_ADDRESS_PARSER_STATE_NONE; *token_r = t_strdup(str_c(tok->last_word)); - return 1; } static bool @@ -215,7 +214,8 @@ if (tok->state == EMAIL_ADDRESS_PARSER_STATE_COMPLETE) { *skip_r = pos; - return fts_tokenizer_address_current_token(tok, token_r); + fts_tokenizer_address_current_token(tok, token_r); + return 1; } /* end of data, output lingering tokens. first the parents data, then @@ -231,8 +231,10 @@ if (fts_tokenizer_address_parent_data(tok, token_r)) return 1; - if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN) - return fts_tokenizer_address_current_token(tok, token_r); + if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN) { + fts_tokenizer_address_current_token(tok, token_r); + return 1; + } tok->state = EMAIL_ADDRESS_PARSER_STATE_NONE; } @@ -279,9 +281,9 @@ break; case EMAIL_ADDRESS_PARSER_STATE_COMPLETE: *skip_r = pos; - if (fts_tokenizer_address_parent_data(tok, token_r)) - return 1; - return fts_tokenizer_address_current_token(tok, token_r); + if (!fts_tokenizer_address_parent_data(tok, token_r)) + fts_tokenizer_address_current_token(tok, token_r); + return 1; default: i_unreached(); } diff -r 1fc7ae2640b0 -r fa55a06ffae2 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Sat May 09 19:14:07 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Sat May 09 19:21:45 2015 +0300 @@ -94,13 +94,12 @@ return t_strndup(data, pos); } -static int +static void fts_tokenizer_generic_simple_current_token(struct generic_fts_tokenizer *tok, const char **token_r) { *token_r = fts_uni_strndup(tok->token->data, tok->token->used); buffer_set_used_size(tok->token, 0); - return 1; } /* TODO: This is duplicated from unichar.c */ @@ -187,7 +186,8 @@ } /* word boundary found - return a new token */ *skip_r = i + 1; - return fts_tokenizer_generic_simple_current_token(tok, token_r); + fts_tokenizer_generic_simple_current_token(tok, token_r); + return 1; } } /* word boundary not found yet */ @@ -196,8 +196,10 @@ *skip_r = i; /* return the last token */ - if (size == 0 && tok->token->used > 0) - return fts_tokenizer_generic_simple_current_token(tok, token_r); + if (size == 0 && tok->token->used > 0) { + fts_tokenizer_generic_simple_current_token(tok, token_r); + return 1; + } return 0; } @@ -492,7 +494,7 @@ return FALSE; } -static int +static void fts_tokenizer_generic_tr29_current_token(struct generic_fts_tokenizer *tok, const char **token_r) { @@ -508,7 +510,6 @@ buffer_set_used_size(tok->token, 0); tok->prev_prev_letter = LETTER_TYPE_NONE; tok->prev_letter = LETTER_TYPE_NONE; - return 1; } struct letter_fn { @@ -569,7 +570,6 @@ { struct generic_fts_tokenizer *tok = (struct generic_fts_tokenizer *)_tok; - unichar_t c; size_t i, char_start_i, start_skip = 0; enum letter_type lt; @@ -592,7 +592,8 @@ tok_append_truncated(tok, data + start_skip, char_start_i - start_skip); *skip_r = i + 1; - return fts_tokenizer_generic_tr29_current_token(tok, token_r); + fts_tokenizer_generic_tr29_current_token(tok, token_r); + return 1; } } i_assert(i >= start_skip && size >= start_skip); @@ -602,7 +603,8 @@ if (size == 0 && tok->token->used > 0) { /* return the last token */ *skip_r = 0; - return fts_tokenizer_generic_tr29_current_token(tok, token_r); + fts_tokenizer_generic_tr29_current_token(tok, token_r); + return 1; } return 0; } From dovecot at dovecot.org Sat May 9 16:28:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 16:28:07 +0000 Subject: dovecot-2.2: lib-fts: Changed fts_tokenizer_next/final() to retu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7f151aca47ac changeset: 18610:7f151aca47ac user: Timo Sirainen date: Sat May 09 19:26:01 2015 +0300 description: lib-fts: Changed fts_tokenizer_next/final() to return error string. The current tokenizers can't fail, but if we're doing tokenization via external services they could fail. diffstat: src/lib-fts/fts-tokenizer-address.c | 3 +- src/lib-fts/fts-tokenizer-generic.c | 11 ++++++--- src/lib-fts/fts-tokenizer-private.h | 3 +- src/lib-fts/fts-tokenizer.c | 22 +++++++++++--------- src/lib-fts/fts-tokenizer.h | 5 ++- src/lib-fts/test-fts-tokenizer.c | 40 ++++++++++++++++++------------------ src/plugins/fts/fts-build-mail.c | 2 +- src/plugins/fts/fts-search-args.c | 18 +++++++++++----- 8 files changed, 59 insertions(+), 45 deletions(-) diffs (truncated from 314 to 300 lines): diff -r fa55a06ffae2 -r 7f151aca47ac src/lib-fts/fts-tokenizer-address.c --- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 19:21:45 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 19:26:01 2015 +0300 @@ -203,7 +203,8 @@ static int fts_tokenizer_email_address_next(struct fts_tokenizer *_tok, const unsigned char *data, size_t size, - size_t *skip_r, const char **token_r) + size_t *skip_r, const char **token_r, + const char **error_r ATTR_UNUSED) { struct email_address_fts_tokenizer *tok = (struct email_address_fts_tokenizer *)_tok; diff -r fa55a06ffae2 -r 7f151aca47ac src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Sat May 09 19:21:45 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Sat May 09 19:26:01 2015 +0300 @@ -168,7 +168,8 @@ static int fts_tokenizer_generic_next_simple(struct fts_tokenizer *_tok, const unsigned char *data, size_t size, - size_t *skip_r, const char **token_r) + size_t *skip_r, const char **token_r, + const char **error_r ATTR_UNUSED) { struct generic_fts_tokenizer *tok = (struct generic_fts_tokenizer *)_tok; @@ -565,8 +566,9 @@ static int fts_tokenizer_generic_next_tr29(struct fts_tokenizer *_tok, - const unsigned char *data, size_t size, - size_t *skip_r, const char **token_r) + const unsigned char *data, size_t size, + size_t *skip_r, const char **token_r, + const char **error_r ATTR_UNUSED) { struct generic_fts_tokenizer *tok = (struct generic_fts_tokenizer *)_tok; @@ -614,7 +616,8 @@ const unsigned char *data ATTR_UNUSED, size_t size ATTR_UNUSED, size_t *skip_r ATTR_UNUSED, - const char **token_r ATTR_UNUSED) + const char **token_r ATTR_UNUSED, + const char **error_r ATTR_UNUSED) { i_unreached(); } diff -r fa55a06ffae2 -r 7f151aca47ac src/lib-fts/fts-tokenizer-private.h --- a/src/lib-fts/fts-tokenizer-private.h Sat May 09 19:21:45 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-private.h Sat May 09 19:26:01 2015 +0300 @@ -12,7 +12,8 @@ void (*reset)(struct fts_tokenizer *tok); int (*next)(struct fts_tokenizer *tok, const unsigned char *data, - size_t size, size_t *skip_r, const char **token_r); + size_t size, size_t *skip_r, const char **token_r, + const char **error_r); }; enum fts_tokenizer_parent_state { diff -r fa55a06ffae2 -r 7f151aca47ac src/lib-fts/fts-tokenizer.c --- a/src/lib-fts/fts-tokenizer.c Sat May 09 19:21:45 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.c Sat May 09 19:26:01 2015 +0300 @@ -123,7 +123,7 @@ static int fts_tokenizer_next_self(struct fts_tokenizer *tok, const unsigned char *data, size_t size, - const char **token_r) + const char **token_r, const char **error_r) { int ret = 0; size_t skip = 0; @@ -133,12 +133,13 @@ if (tok->prev_reply_finished) { /* whole new data */ - ret = tok->v->next(tok, data, size, &skip, token_r); + ret = tok->v->next(tok, data, size, &skip, token_r, error_r); } else { /* continuing previous data */ i_assert(tok->prev_skip <= size); ret = tok->v->next(tok, data + tok->prev_skip, - size - tok->prev_skip, &skip, token_r); + size - tok->prev_skip, &skip, + token_r, error_r); } if (ret > 0) { @@ -164,13 +165,13 @@ int fts_tokenizer_next(struct fts_tokenizer *tok, const unsigned char *data, size_t size, - const char **token_r) + const char **token_r, const char **error_r) { int ret; switch (tok->parent_state) { case FTS_TOKENIZER_PARENT_STATE_ADD_DATA: - ret = fts_tokenizer_next_self(tok, data, size, token_r); + ret = fts_tokenizer_next_self(tok, data, size, token_r, error_r); if (ret <= 0 || tok->parent == NULL || tok->skip_parents) return ret; buffer_set_used_size(tok->parent_input, 0); @@ -179,25 +180,26 @@ /* fall through */ case FTS_TOKENIZER_PARENT_STATE_NEXT_OUTPUT: ret = fts_tokenizer_next(tok->parent, tok->parent_input->data, - tok->parent_input->used, token_r); + tok->parent_input->used, token_r, error_r); if (ret != 0) return ret; tok->parent_state++; /* fall through */ case FTS_TOKENIZER_PARENT_STATE_FINALIZE: - ret = fts_tokenizer_next(tok->parent, NULL, 0, token_r); + ret = fts_tokenizer_next(tok->parent, NULL, 0, token_r, error_r); if (ret != 0) return ret; /* we're finished sending this token to parent tokenizer. see if our own tokenizer has more tokens available */ tok->parent_state = FTS_TOKENIZER_PARENT_STATE_ADD_DATA; - return fts_tokenizer_next(tok, data, size, token_r); + return fts_tokenizer_next(tok, data, size, token_r, error_r); default: i_unreached(); } } -int fts_tokenizer_final(struct fts_tokenizer *tok, const char **token_r) +int fts_tokenizer_final(struct fts_tokenizer *tok, const char **token_r, + const char **error_r) { - return fts_tokenizer_next(tok, NULL, 0, token_r); + return fts_tokenizer_next(tok, NULL, 0, token_r, error_r); } diff -r fa55a06ffae2 -r 7f151aca47ac src/lib-fts/fts-tokenizer.h --- a/src/lib-fts/fts-tokenizer.h Sat May 09 19:21:45 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.h Sat May 09 19:26:01 2015 +0300 @@ -77,9 +77,10 @@ int fts_tokenizer_next(struct fts_tokenizer *tok, const unsigned char *data, size_t size, - const char **token_r); + const char **token_r, const char **error_r); /* Returns same as fts_tokenizer_next(). */ -int fts_tokenizer_final(struct fts_tokenizer *tok, const char **token_r); +int fts_tokenizer_final(struct fts_tokenizer *tok, const char **token_r, + const char **error_r); const char *fts_tokenizer_name(const struct fts_tokenizer *tok); diff -r fa55a06ffae2 -r 7f151aca47ac src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Sat May 09 19:21:45 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Sat May 09 19:26:01 2015 +0300 @@ -38,16 +38,16 @@ const char *const *expected_output) { const unsigned char *input = (const unsigned char *)_input; - const char *token; + const char *token, *error; unsigned int i, max, outi, char_len, input_len = strlen(_input); /* test all input at once */ outi = 0; - while (fts_tokenizer_next(tok, input, input_len, &token) > 0) { + while (fts_tokenizer_next(tok, input, input_len, &token, &error) > 0) { test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); outi++; } - while (fts_tokenizer_next(tok, NULL, 0, &token) > 0) { + while (fts_tokenizer_next(tok, NULL, 0, &token, &error) > 0) { test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); outi++; } @@ -56,12 +56,12 @@ /* test input one byte at a time */ for (i = outi = 0; i < input_len; i += char_len) { char_len = uni_utf8_char_bytes(input[i]); - while (fts_tokenizer_next(tok, input+i, char_len, &token) > 0) { + while (fts_tokenizer_next(tok, input+i, char_len, &token, &error) > 0) { test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); outi++; } } - while (fts_tokenizer_final(tok, &token) > 0) { + while (fts_tokenizer_final(tok, &token, &error) > 0) { test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); outi++; } @@ -72,12 +72,12 @@ max = rand() % (input_len - i) + 1; for (char_len = 0; char_len < max; ) char_len += uni_utf8_char_bytes(input[i+char_len]); - while (fts_tokenizer_next(tok, input+i, char_len, &token) > 0) { + while (fts_tokenizer_next(tok, input+i, char_len, &token, &error) > 0) { test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); outi++; } } - while (fts_tokenizer_final(tok, &token) > 0) { + while (fts_tokenizer_final(tok, &token, &error) > 0) { test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); outi++; } @@ -257,28 +257,28 @@ test_tokenizer_inputoutput(tok, input, expected_output); /* make sure state is forgotten at EOF */ - test_assert(fts_tokenizer_next(tok, (const void *)"foo", 3, &token) == 0); - test_assert(fts_tokenizer_final(tok, &token) > 0 && + test_assert(fts_tokenizer_next(tok, (const void *)"foo", 3, &token, &error) == 0); + test_assert(fts_tokenizer_final(tok, &token, &error) > 0 && strcmp(token, "foo") == 0); - test_assert(fts_tokenizer_final(tok, &token) == 0); + test_assert(fts_tokenizer_final(tok, &token, &error) == 0); - test_assert(fts_tokenizer_next(tok, (const void *)"bar at baz", 7, &token) == 0); - test_assert(fts_tokenizer_final(tok, &token) > 0 && + test_assert(fts_tokenizer_next(tok, (const void *)"bar at baz", 7, &token, &error) == 0); + test_assert(fts_tokenizer_final(tok, &token, &error) > 0 && strcmp(token, "bar at baz") == 0); - test_assert(fts_tokenizer_final(tok, &token) == 0); + test_assert(fts_tokenizer_final(tok, &token, &error) == 0); - test_assert(fts_tokenizer_next(tok, (const void *)"foo@", 4, &token) == 0); - test_assert(fts_tokenizer_final(tok, &token) > 0 && + test_assert(fts_tokenizer_next(tok, (const void *)"foo@", 4, &token, &error) == 0); + test_assert(fts_tokenizer_final(tok, &token, &error) > 0 && strcmp(token, "foo") == 0); - test_assert(fts_tokenizer_final(tok, &token) == 0); + test_assert(fts_tokenizer_final(tok, &token, &error) == 0); /* test reset explicitly */ - test_assert(fts_tokenizer_next(tok, (const void *)"a", 1, &token) == 0); + test_assert(fts_tokenizer_next(tok, (const void *)"a", 1, &token, &error) == 0); fts_tokenizer_reset(tok); - test_assert(fts_tokenizer_next(tok, (const void *)"b at c", 3, &token) == 0); - test_assert(fts_tokenizer_final(tok, &token) > 0 && + test_assert(fts_tokenizer_next(tok, (const void *)"b at c", 3, &token, &error) == 0); + test_assert(fts_tokenizer_final(tok, &token, &error) > 0 && strcmp(token, "b at c") == 0); - test_assert(fts_tokenizer_final(tok, &token) == 0); + test_assert(fts_tokenizer_final(tok, &token, &error) == 0); fts_tokenizer_unref(&tok); fts_tokenizer_unref(&gen_tok); diff -r fa55a06ffae2 -r 7f151aca47ac src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Sat May 09 19:21:45 2015 +0300 +++ b/src/plugins/fts/fts-build-mail.c Sat May 09 19:26:01 2015 +0300 @@ -261,7 +261,7 @@ int ret; tokenizer = fts_user_get_index_tokenizer(ctx->update_ctx->backend->ns->user); - while ((ret = fts_tokenizer_next(tokenizer, data, size, &token)) > 0) { + while ((ret = fts_tokenizer_next(tokenizer, data, size, &token, &error)) > 0) { if (filter != NULL) { ret = fts_filter_filter(filter, &token, &error); if (ret == 0) diff -r fa55a06ffae2 -r 7f151aca47ac src/plugins/fts/fts-search-args.c --- a/src/plugins/fts/fts-search-args.c Sat May 09 19:21:45 2015 +0300 +++ b/src/plugins/fts/fts-search-args.c Sat May 09 19:26:01 2015 +0300 @@ -81,7 +81,7 @@ token2 = t_strdup(token2); array_append(&tokens, &token2, 1); } else if (ret < 0) { - i_error("fts: Couldn't create search tokens: %s", error); + i_error("fts: Couldn't filter search tokens: %s", error); return -1; } } @@ -99,9 +99,10 @@ { const ARRAY_TYPE(fts_user_language) *languages; struct mail_search_arg *and_arg, *orig_arg = *argp; - const char *token, *orig_token = orig_arg->value.str; + const char *error, *token, *orig_token = orig_arg->value.str; unsigned int orig_token_len = strlen(orig_token); struct fts_tokenizer *tokenizer; + int ret; languages = fts_user_get_all_languages(backend->ns->user); tokenizer = fts_user_get_search_tokenizer(backend->ns->user); @@ -117,20 +118,25 @@ /* reset tokenizer between search args in case there's any state left from some previous failure */ fts_tokenizer_reset(tokenizer); - while (fts_tokenizer_next(tokenizer, - (const void *)orig_token, - orig_token_len, &token) > 0) { + while ((ret = fts_tokenizer_next(tokenizer, + (const void *)orig_token, + orig_token_len, &token, &error)) > 0) { if (fts_backend_dovecot_expand_lang_tokens(languages, pool, and_arg, orig_arg, orig_token, token) < 0) return -1; } - while (fts_tokenizer_final(tokenizer, &token) > 0) { From dovecot at dovecot.org Sat May 9 16:33:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 16:33:16 +0000 Subject: dovecot-2.2: indexer-worker: Don't crash if we failed to get mai... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a982d64d1727 changeset: 18611:a982d64d1727 user: Timo Sirainen date: Sat May 09 19:28:46 2015 +0300 description: indexer-worker: Don't crash if we failed to get mailbox path. Found by Coverity diffstat: src/indexer/master-connection.c | 13 +++++++------ 1 files changed, 7 insertions(+), 6 deletions(-) diffs (25 lines): diff -r 7f151aca47ac -r a982d64d1727 src/indexer/master-connection.c --- a/src/indexer/master-connection.c Sat May 09 19:26:01 2015 +0300 +++ b/src/indexer/master-connection.c Sat May 09 19:28:46 2015 +0300 @@ -139,14 +139,15 @@ ns = mail_namespace_find(user->namespaces, mailbox); box = mailbox_alloc(ns->list, mailbox, 0); ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path); - if (ret <= 0) { + if (ret < 0) { + i_error("Getting path to mailbox %s failed: %s", + mailbox, mailbox_get_last_error(box, NULL)); mailbox_free(&box); - if (ret < 0) { - i_error("Getting path to mailbox %s failed: %s", - mailbox, mailbox_get_last_error(box, NULL)); - return -1; - } + return -1; + } + if (ret == 0) { i_info("Indexes disabled for mailbox %s, skipping", mailbox); + mailbox_free(&box); return 0; } ret = 0; From dovecot at dovecot.org Sat May 9 16:33:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 16:33:16 +0000 Subject: dovecot-2.2: lib-mail: Fixed crash in istream-attachment-extract... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d547eee8a8c7 changeset: 18612:d547eee8a8c7 user: Timo Sirainen date: Sat May 09 19:31:05 2015 +0300 description: lib-mail: Fixed crash in istream-attachment-extractor error handling path diffstat: src/lib-mail/istream-attachment-extractor.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r a982d64d1727 -r d547eee8a8c7 src/lib-mail/istream-attachment-extractor.c --- a/src/lib-mail/istream-attachment-extractor.c Sat May 09 19:28:46 2015 +0300 +++ b/src/lib-mail/istream-attachment-extractor.c Sat May 09 19:31:05 2015 +0300 @@ -416,7 +416,7 @@ i_assert(ret == -1); if (input->stream_errno != 0) { i_error("istream-attachment: read(%s) failed: %m", - i_stream_get_name(base64_input)); + i_stream_get_name(input)); failed = TRUE; } } From dovecot at dovecot.org Sat May 9 16:33:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 16:33:16 +0000 Subject: dovecot-2.2: lib-mail: Improved istream-attachment-extractor err... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/024fb73d7d62 changeset: 18613:024fb73d7d62 user: Timo Sirainen date: Sat May 09 19:31:17 2015 +0300 description: lib-mail: Improved istream-attachment-extractor error logging. diffstat: src/lib-mail/istream-attachment-extractor.c | 14 ++++++++------ 1 files changed, 8 insertions(+), 6 deletions(-) diffs (34 lines): diff -r d547eee8a8c7 -r 024fb73d7d62 src/lib-mail/istream-attachment-extractor.c --- a/src/lib-mail/istream-attachment-extractor.c Sat May 09 19:31:05 2015 +0300 +++ b/src/lib-mail/istream-attachment-extractor.c Sat May 09 19:31:17 2015 +0300 @@ -392,13 +392,14 @@ if (ret != -1) { i_assert(failed); } else if (base64_input->stream_errno != 0) { - i_error("istream-attachment: read(%s) failed: %m", - i_stream_get_name(base64_input)); + i_error("istream-attachment: read(%s) failed: %s", + i_stream_get_name(base64_input), + i_stream_get_error(base64_input)); failed = TRUE; } if (o_stream_nfinish(output) < 0) { - i_error("istream-attachment: write(%s) failed: %m", - o_stream_get_name(output)); + i_error("istream-attachment: write(%s) failed: %s", + o_stream_get_name(output), o_stream_get_error(output)); failed = TRUE; } @@ -415,8 +416,9 @@ } i_assert(ret == -1); if (input->stream_errno != 0) { - i_error("istream-attachment: read(%s) failed: %m", - i_stream_get_name(input)); + i_error("istream-attachment: read(%s) failed: %s", + i_stream_get_name(input), + i_stream_get_error(input)); failed = TRUE; } } From dovecot at dovecot.org Sat May 9 16:37:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 16:37:25 +0000 Subject: dovecot-2.2: imap: Don't crash if APPEND command is given with i... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1c2e42bf8825 changeset: 18614:1c2e42bf8825 user: Timo Sirainen date: Sat May 09 19:35:31 2015 +0300 description: imap: Don't crash if APPEND command is given with invalid parameters. Found by Coverity. diffstat: src/imap/cmd-append.c | 21 +++++++++++---------- 1 files changed, 11 insertions(+), 10 deletions(-) diffs (37 lines): diff -r 024fb73d7d62 -r 1c2e42bf8825 src/imap/cmd-append.c --- a/src/imap/cmd-append.c Sat May 09 19:31:17 2015 +0300 +++ b/src/imap/cmd-append.c Sat May 09 19:35:31 2015 +0300 @@ -493,22 +493,23 @@ valid = FALSE; *nonsync_r = FALSE; ctx->catenate = FALSE; - if (imap_arg_atom_equals(args, "CATENATE")) { - args++; - if (imap_arg_get_list(args, &cat_list)) { - valid = TRUE; - ctx->catenate = TRUE; - } + if (imap_arg_get_literal_size(args, &ctx->literal_size)) { + *nonsync_r = args->type == IMAP_ARG_LITERAL_SIZE_NONSYNC; + ctx->binary_input = args->literal8; + valid = TRUE; + } else if (!imap_arg_atom_equals(args, "CATENATE")) { + /* invalid */ + } else if (!imap_arg_get_list(++args, &cat_list)) { + /* invalid */ + } else { + valid = TRUE; + ctx->catenate = TRUE; /* We'll do BINARY conversion only if the CATENATE's first part is a literal8. If it doesn't and a literal8 is seen later we'll abort the append with UNKNOWN-CTE. */ ctx->binary_input = imap_arg_atom_equals(&cat_list[0], "TEXT") && cat_list[1].literal8; - } else if (imap_arg_get_literal_size(args, &ctx->literal_size)) { - *nonsync_r = args->type == IMAP_ARG_LITERAL_SIZE_NONSYNC; - ctx->binary_input = args->literal8; - valid = TRUE; } if (!IMAP_ARG_IS_EOL(&args[1])) valid = FALSE; From dovecot at dovecot.org Sat May 9 16:41:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 16:41:32 +0000 Subject: dovecot-2.2: doveadm: Removed unused doveadm_mailbox_find_and_sy... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cff670d68f69 changeset: 18615:cff670d68f69 user: Timo Sirainen date: Sat May 09 19:39:39 2015 +0300 description: doveadm: Removed unused doveadm_mailbox_find_and_sync() function diffstat: src/doveadm/doveadm-mail.c | 32 -------------------------------- src/doveadm/doveadm-mail.h | 2 -- 2 files changed, 0 insertions(+), 34 deletions(-) diffs (54 lines): diff -r 1c2e42bf8825 -r cff670d68f69 src/doveadm/doveadm-mail.c --- a/src/doveadm/doveadm-mail.c Sat May 09 19:35:31 2015 +0300 +++ b/src/doveadm/doveadm-mail.c Sat May 09 19:39:39 2015 +0300 @@ -243,38 +243,6 @@ return mailbox_alloc(ns->list, mailbox, MAILBOX_FLAG_IGNORE_ACLS); } -static int -doveadm_mailbox_find_and_open(struct mail_user *user, const char *mailbox, - struct mailbox **box_r) -{ - struct mailbox *box; - - box = doveadm_mailbox_find(user, mailbox); - if (mailbox_open(box) < 0) { - i_error("Opening mailbox %s failed: %s", mailbox, - mailbox_get_last_error(box, NULL)); - mailbox_free(&box); - return -1; - } - *box_r = box; - return 0; -} - -int doveadm_mailbox_find_and_sync(struct mail_user *user, const char *mailbox, - struct mailbox **box_r) -{ - if (doveadm_mailbox_find_and_open(user, mailbox, box_r) < 0) - return -1; - if (mailbox_sync(*box_r, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { - i_error("Syncing mailbox %s failed: %s", mailbox, - mailbox_get_last_error(*box_r, NULL)); - mailbox_free(box_r); - mailbox_free(box_r); - return -1; - } - return 0; -} - struct mail_search_args * doveadm_mail_build_search_args(const char *const args[]) { diff -r 1c2e42bf8825 -r cff670d68f69 src/doveadm/doveadm-mail.h --- a/src/doveadm/doveadm-mail.h Sat May 09 19:35:31 2015 +0300 +++ b/src/doveadm/doveadm-mail.h Sat May 09 19:39:39 2015 +0300 @@ -146,8 +146,6 @@ struct mailbox * doveadm_mailbox_find(struct mail_user *user, const char *mailbox); -int doveadm_mailbox_find_and_sync(struct mail_user *user, const char *mailbox, - struct mailbox **box_r); struct mail_search_args * doveadm_mail_build_search_args(const char *const args[]); void doveadm_mailbox_args_check(const char *const args[]); From dovecot at dovecot.org Sat May 9 16:43:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 16:43:39 +0000 Subject: dovecot-2.2: doveadm-server: Fixed potential crash if doveadm cl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e702b7a745e0 changeset: 18616:e702b7a745e0 user: Timo Sirainen date: Sat May 09 19:41:45 2015 +0300 description: doveadm-server: Fixed potential crash if doveadm client disconnected Found by Coverity diffstat: src/doveadm/server-connection.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (14 lines): diff -r cff670d68f69 -r e702b7a745e0 src/doveadm/server-connection.c --- a/src/doveadm/server-connection.c Sat May 09 19:39:39 2015 +0300 +++ b/src/doveadm/server-connection.c Sat May 09 19:41:45 2015 +0300 @@ -127,9 +127,9 @@ ret = o_stream_flush(conn->output); if (ret > 0 && conn->cmd_input != NULL && conn->delayed_cmd == NULL) ret = server_connection_send_cmd_input_more(conn); + o_stream_uncork(conn->output); if (ret < 0) server_connection_destroy(&conn); - o_stream_uncork(conn->output); return ret; } From dovecot at dovecot.org Sat May 9 16:52:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 16:52:10 +0000 Subject: dovecot-2.2: anvil: Penalty tracking was moving last checksums w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/398d94f08407 changeset: 18617:398d94f08407 user: Timo Sirainen date: Sat May 09 19:50:14 2015 +0300 description: anvil: Penalty tracking was moving last checksums wrongly with memcpy(). Depending on the OS/etc this could have caused the checksum tracking to go wrong. Found by Coverity. diffstat: src/anvil/penalty.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r e702b7a745e0 -r 398d94f08407 src/anvil/penalty.c --- a/src/anvil/penalty.c Sat May 09 19:41:45 2015 +0300 +++ b/src/anvil/penalty.c Sat May 09 19:50:14 2015 +0300 @@ -105,8 +105,8 @@ for (i = 0; i < count; i++) { if (checksums[i] == checksum) { if (i > 0) { - memcpy(checksums + 1, checksums, - sizeof(checksums[0]) * i); + memmove(checksums + 1, checksums, + sizeof(checksums[0]) * i); checksums[0] = checksum; } return TRUE; From dovecot at dovecot.org Sat May 9 16:59:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 16:59:23 +0000 Subject: dovecot-2.2: quota-dict: It wasn't possible to use multiple quot... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8b452eb97422 changeset: 18618:8b452eb97422 user: Timo Sirainen date: Sat May 09 19:57:28 2015 +0300 description: quota-dict: It wasn't possible to use multiple quota settings. Found by Coverity diffstat: src/plugins/quota/quota-dict.c | 19 +++++++------------ 1 files changed, 7 insertions(+), 12 deletions(-) diffs (46 lines): diff -r 398d94f08407 -r 8b452eb97422 src/plugins/quota/quota-dict.c --- a/src/plugins/quota/quota-dict.c Sat May 09 19:50:14 2015 +0300 +++ b/src/plugins/quota/quota-dict.c Sat May 09 19:57:28 2015 +0300 @@ -44,25 +44,19 @@ username = t_strdup_until(args, p); args = p+1; - do { + for (;;) { /* FIXME: pretty ugly in here. the parameters should have been designed to be extensible. do it in a future version */ if (strncmp(args, "noenforcing:", 12) == 0) { _root->no_enforcing = TRUE; args += 12; - continue; - } - if (strncmp(args, "hidden:", 7) == 0) { + } else if (strncmp(args, "hidden:", 7) == 0) { _root->hidden = TRUE; args += 7; - continue; - } - if (strncmp(args, "ignoreunlimited:", 16) == 0) { + } else if (strncmp(args, "ignoreunlimited:", 16) == 0) { _root->disable_unlimited_tracking = TRUE; args += 16; - continue; - } - if (strncmp(args, "ns=", 3) == 0) { + } else if (strncmp(args, "ns=", 3) == 0) { p = strchr(args, ':'); if (p == NULL) break; @@ -70,9 +64,10 @@ _root->ns_prefix = p_strdup_until(_root->pool, args + 3, p); args = p + 1; - continue; + } else { + break; } - } while (0); + } if (*username == '\0') username = _root->quota->user->username; From dovecot at dovecot.org Sat May 9 17:01:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 17:01:07 +0000 Subject: dovecot-2.2: Make Coverity happier. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/49f5131a502e changeset: 18619:49f5131a502e user: Timo Sirainen date: Sat May 09 19:58:59 2015 +0300 description: Make Coverity happier. diffstat: src/lib-http/http-server-connection.c | 2 ++ src/lib-index/mail-transaction-log-file.c | 1 + src/lib-mail/ostream-dot.c | 1 + src/lib-storage/index/index-mail.c | 2 ++ src/lib-storage/index/maildir/maildir-uidlist.c | 1 + 5 files changed, 7 insertions(+), 0 deletions(-) diffs (62 lines): diff -r 8b452eb97422 -r 49f5131a502e src/lib-http/http-server-connection.c --- a/src/lib-http/http-server-connection.c Sat May 09 19:57:28 2015 +0300 +++ b/src/lib-http/http-server-connection.c Sat May 09 19:58:59 2015 +0300 @@ -541,11 +541,13 @@ switch (error_code) { case HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST: conn->input_broken = TRUE; + /* fall through */ case HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST: http_server_request_fail(req, 400, "Bad Request"); break; case HTTP_REQUEST_PARSE_ERROR_METHOD_TOO_LONG: conn->input_broken = TRUE; + /* fall through */ case HTTP_REQUEST_PARSE_ERROR_NOT_IMPLEMENTED: http_server_request_fail(req, 501, "Not Implemented"); break; diff -r 8b452eb97422 -r 49f5131a502e src/lib-index/mail-transaction-log-file.c --- a/src/lib-index/mail-transaction-log-file.c Sat May 09 19:57:28 2015 +0300 +++ b/src/lib-index/mail-transaction-log-file.c Sat May 09 19:58:59 2015 +0300 @@ -1620,6 +1620,7 @@ if (file->mmap_base == NULL) return; + i_assert(file->buffer != NULL); if (munmap(file->mmap_base, file->mmap_size) < 0) log_file_set_syscall_error(file, "munmap()"); file->mmap_base = NULL; diff -r 8b452eb97422 -r 49f5131a502e src/lib-mail/ostream-dot.c --- a/src/lib-mail/ostream-dot.c Sat May 09 19:57:28 2015 +0300 +++ b/src/lib-mail/ostream-dot.c Sat May 09 19:58:59 2015 +0300 @@ -139,6 +139,7 @@ case '.': /* add dot */ add = '.'; + /* fall through */ default: dstream->state = STREAM_STATE_NONE; break; diff -r 8b452eb97422 -r 49f5131a502e src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Sat May 09 19:57:28 2015 +0300 +++ b/src/lib-storage/index/index-mail.c Sat May 09 19:58:59 2015 +0300 @@ -920,6 +920,8 @@ old_offset = mail->data.stream == NULL ? 0 : mail->data.stream->v_offset; if (mail_get_stream(&mail->mail.mail, NULL, NULL, &input) < 0) return -1; + i_assert(mail->data.stream != NULL); + i_stream_seek(input, part->physical_pos); input = i_stream_create_limit(input, part->header_size.physical_size + part->body_size.physical_size); diff -r 8b452eb97422 -r 49f5131a502e src/lib-storage/index/maildir/maildir-uidlist.c --- a/src/lib-storage/index/maildir/maildir-uidlist.c Sat May 09 19:57:28 2015 +0300 +++ b/src/lib-storage/index/maildir/maildir-uidlist.c Sat May 09 19:58:59 2015 +0300 @@ -1976,6 +1976,7 @@ array_free(&uidlist->records); uidlist->records = ctx->records; ctx->records.arr.buffer = NULL; + i_assert(array_is_created(&uidlist->records)); hash_table_destroy(&uidlist->files); uidlist->files = ctx->files; From dovecot at dovecot.org Sat May 9 17:45:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 17:45:13 +0000 Subject: dovecot-2.2: Moved fs-compress to a separate plugin directory. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7799886e017a changeset: 18620:7799886e017a user: Timo Sirainen date: Sat May 09 20:26:06 2015 +0300 description: Moved fs-compress to a separate plugin directory. Mainly because I couldn't figure out how to make automake dependencies work on "make install" stage. It was trying to link fs-compress.so using -ldovecot-storage, but libdovecot-storage.so was also concurrently being installed. diffstat: configure.ac | 1 + src/lib-compression/Makefile.am | 10 - src/lib-compression/fs-compress.c | 418 ---------------------------------- src/plugins/Makefile.am | 3 +- src/plugins/fs-compress/Makefile.am | 13 + src/plugins/fs-compress/fs-compress.c | 418 ++++++++++++++++++++++++++++++++++ 6 files changed, 434 insertions(+), 429 deletions(-) diffs (truncated from 910 to 300 lines): diff -r 49f5131a502e -r 7799886e017a configure.ac --- a/configure.ac Sat May 09 19:58:59 2015 +0300 +++ b/configure.ac Sat May 09 20:26:06 2015 +0300 @@ -2944,6 +2944,7 @@ src/plugins/imap-acl/Makefile src/plugins/autocreate/Makefile src/plugins/expire/Makefile +src/plugins/fs-compress/Makefile src/plugins/fts/Makefile src/plugins/fts-lucene/Makefile src/plugins/fts-solr/Makefile diff -r 49f5131a502e -r 7799886e017a src/lib-compression/Makefile.am --- a/src/lib-compression/Makefile.am Sat May 09 19:58:59 2015 +0300 +++ b/src/lib-compression/Makefile.am Sat May 09 20:26:06 2015 +0300 @@ -1,12 +1,7 @@ noinst_LTLIBRARIES = libcompression.la -fs_moduledir = $(moduledir) -fs_module_LTLIBRARIES = \ - libfs_compress.la - AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ - -I$(top_srcdir)/src/lib-fs \ -I$(top_srcdir)/src/lib-test libcompression_la_SOURCES = \ @@ -35,11 +30,6 @@ libdovecot_compression_la_DEPENDENCIES = libcompression.la ../lib-dovecot/libdovecot.la libdovecot_compression_la_LDFLAGS = -export-dynamic -libfs_compress_la_SOURCES = fs-compress.c -libfs_compress_la_LIBADD = libdovecot-compression.la -libfs_compress_la_DEPENDENCIES = libdovecot-compression.la -libfs_compress_la_LDFLAGS = -module -avoid-version - test_programs = \ test-compression diff -r 49f5131a502e -r 7799886e017a src/lib-compression/fs-compress.c --- a/src/lib-compression/fs-compress.c Sat May 09 19:58:59 2015 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,418 +0,0 @@ -/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "istream.h" -#include "ostream.h" -#include "iostream-temp.h" -#include "compression.h" -#include "fs-api-private.h" - -struct compress_fs { - struct fs fs; - const struct compression_handler *handler; - unsigned int compress_level; -}; - -struct compress_fs_file { - struct fs_file file; - struct compress_fs *fs; - struct fs_file *super, *super_read; - enum fs_open_mode open_mode; - struct istream *input; - - struct ostream *super_output; - struct ostream *temp_output; -}; - -struct compress_fs_iter { - struct fs_iter iter; - struct fs_iter *super; -}; - -extern const struct fs fs_class_compress; - -static struct fs *fs_compress_alloc(void) -{ - struct compress_fs *fs; - - fs = i_new(struct compress_fs, 1); - fs->fs = fs_class_compress; - return &fs->fs; -} - -static int -fs_compress_init(struct fs *_fs, const char *args, const - struct fs_settings *set) -{ - struct compress_fs *fs = (struct compress_fs *)_fs; - const char *p, *compression_name, *level_str, *error; - const char *parent_name, *parent_args; - - /* get compression handler name */ - p = strchr(args, ':'); - if (p == NULL) { - fs_set_error(_fs, "Compression method not given as parameter"); - return -1; - } - compression_name = t_strdup_until(args, p++); - args = p; - - /* get compression level */ - p = strchr(args, ':'); - if (p == NULL || p[1] == '\0') { - fs_set_error(_fs, "Parent filesystem not given as parameter"); - return -1; - } - - level_str = t_strdup_until(args, p++); - if (str_to_uint(level_str, &fs->compress_level) < 0 || - fs->compress_level < 1 || fs->compress_level > 9) { - fs_set_error(_fs, "Invalid compression level parameter '%s'", level_str); - return -1; - } - args = p; - - fs->handler = compression_lookup_handler(compression_name); - if (fs->handler == NULL) { - fs_set_error(_fs, "Compression method '%s' not support", compression_name); - return -1; - } - - parent_args = strchr(args, ':'); - if (parent_args == NULL) { - parent_name = args; - parent_args = ""; - } else { - parent_name = t_strdup_until(args, parent_args); - parent_args++; - } - if (fs_init(parent_name, parent_args, set, &_fs->parent, &error) < 0) { - fs_set_error(_fs, "%s: %s", parent_name, error); - return -1; - } - return 0; -} - -static void fs_compress_deinit(struct fs *_fs) -{ - struct compress_fs *fs = (struct compress_fs *)_fs; - - if (_fs->parent != NULL) - fs_deinit(&_fs->parent); - i_free(fs); -} - -static enum fs_properties fs_compress_get_properties(struct fs *_fs) -{ - return fs_get_properties(_fs->parent); -} - -static struct fs_file * -fs_compress_file_init(struct fs *_fs, const char *path, - enum fs_open_mode mode, enum fs_open_flags flags) -{ - struct compress_fs *fs = (struct compress_fs *)_fs; - struct compress_fs_file *file; - - file = i_new(struct compress_fs_file, 1); - file->file.fs = _fs; - file->file.path = i_strdup(path); - file->fs = fs; - file->open_mode = mode; - - /* avoid unnecessarily creating two seekable streams */ - flags &= ~FS_OPEN_FLAG_SEEKABLE; - - file->super = fs_file_init(_fs->parent, path, mode | flags); - if (mode == FS_OPEN_MODE_READONLY && - (flags & FS_OPEN_FLAG_ASYNC) == 0) { - /* use async stream for super, so fs_read_stream() won't create - another seekable stream unneededly */ - file->super_read = fs_file_init(_fs->parent, path, mode | flags | - FS_OPEN_FLAG_ASYNC); - } else { - file->super_read = file->super; - } - return &file->file; -} - -static void fs_compress_file_deinit(struct fs_file *_file) -{ - struct compress_fs_file *file = (struct compress_fs_file *)_file; - - if (file->super_read != file->super && file->super_read != NULL) - fs_file_deinit(&file->super_read); - fs_file_deinit(&file->super); - i_free(file->file.path); - i_free(file); -} - -static void fs_compress_file_close(struct fs_file *_file) -{ - struct compress_fs_file *file = (struct compress_fs_file *)_file; - - if (file->input != NULL) - i_stream_unref(&file->input); - if (file->super_read != NULL) - fs_file_close(file->super_read); - if (file->super != NULL) - fs_file_close(file->super); -} - -static const char *fs_compress_file_get_path(struct fs_file *_file) -{ - struct compress_fs_file *file = (struct compress_fs_file *)_file; - - return fs_file_path(file->super); -} - -static void -fs_compress_set_async_callback(struct fs_file *_file, - fs_file_async_callback_t *callback, - void *context) -{ - struct compress_fs_file *file = (struct compress_fs_file *)_file; - - fs_file_set_async_callback(file->super, callback, context); -} - -static int fs_compress_wait_async(struct fs *_fs) -{ - return fs_wait_async(_fs->parent); -} - -static void -fs_compress_set_metadata(struct fs_file *_file, const char *key, - const char *value) -{ - struct compress_fs_file *file = (struct compress_fs_file *)_file; - - fs_set_metadata(file->super, key, value); -} - -static int -fs_compress_get_metadata(struct fs_file *_file, - const ARRAY_TYPE(fs_metadata) **metadata_r) -{ - struct compress_fs_file *file = (struct compress_fs_file *)_file; - - return fs_get_metadata(file->super, metadata_r); -} - -static bool fs_compress_prefetch(struct fs_file *_file, uoff_t length) -{ - struct compress_fs_file *file = (struct compress_fs_file *)_file; - - return fs_prefetch(file->super, length); -} - -static struct istream * -fs_compress_read_stream(struct fs_file *_file, size_t max_buffer_size) -{ - struct compress_fs_file *file = (struct compress_fs_file *)_file; - struct istream *input; - - if (file->input != NULL) { - i_stream_ref(file->input); - i_stream_seek(file->input, 0); - return file->input; - } - - input = fs_read_stream(file->super_read, max_buffer_size); - file->input = file->fs->handler->create_istream(input, FALSE); - i_stream_unref(&input); - i_stream_ref(file->input); - return file->input; -} - -static void fs_compress_write_stream(struct fs_file *_file) -{ - struct compress_fs_file *file = (struct compress_fs_file *)_file; - - i_assert(_file->output == NULL); - - file->temp_output = - iostream_temp_create_named(_file->fs->temp_path_prefix, - IOSTREAM_TEMP_FLAG_TRY_FD_DUP, - fs_file_path(_file)); - _file->output = file->fs->handler-> - create_ostream(file->temp_output, file->fs->compress_level); -} - -static int fs_compress_write_stream_finish(struct fs_file *_file, bool success) -{ - struct compress_fs_file *file = (struct compress_fs_file *)_file; - struct istream *input; - int ret; - - if (_file->output != NULL) { - if (_file->output->closed) - success = FALSE; - if (_file->output == file->super_output) - _file->output = NULL; - else - o_stream_unref(&_file->output); - } - if (!success) { From dovecot at dovecot.org Sat May 9 17:45:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 17:45:13 +0000 Subject: dovecot-2.2: Released v2.2.17.rc1. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/da685736985a changeset: 18621:da685736985a user: Timo Sirainen date: Sat May 09 20:29:52 2015 +0300 description: Released v2.2.17.rc1. diffstat: NEWS | 47 +++++++++++++++++++++++++++++++++++++++++++++++ configure.ac | 4 ++-- 2 files changed, 49 insertions(+), 2 deletions(-) diffs (68 lines): diff -r 7799886e017a -r da685736985a NEWS --- a/NEWS Sat May 09 20:26:06 2015 +0300 +++ b/NEWS Sat May 09 20:29:52 2015 +0300 @@ -1,3 +1,50 @@ +v2.2.17 2015-05-xx Timo Sirainen + + * Dovecot no longer checks or warns if a mountpoint is removed. This + was causing more trouble than it was worth. Make sure that all the + mountpoints that Dovecot accesses aren't writable by mail processes + when they're unmounted. + * dict server wasn't properly escaping/unescaping data. Fixing this + broke backwards compatibility with data that contains line feeds. + This hopefully affects only very few installations. If you're using + dict to save multiline data (Sieve scripts to SQL), you may be + affected. + * imap: SPECIAL-USE capability is no longer advertised if there are + no special_use flags specified for any mailboxes. + + + lmtp: Added lmtp_hdr_delivery_address setting to specify whether + to include email address in Delivered-To: and Received: headers. + + Added initial version of full text search library, which includes + language-specific text normalization and filtering. This is still + in development, but it's already possible to use for testing with + fts-lucene and fts-solr. + + lda, lmtp: deliver_log_format can now include %{delivery_time}, + which expands to how many milliseconds it took to deliver the mail. + With LMTP %{session_time} also expands to how many milliseconds the + LMTP session took, not including the delivery time. + + lmtp proxy: Mail delivery logging includes timing information. + + imap: Most IMAP commands now include in the tagged reply how many + milliseconds it took to run the command (not counting the time spent + on waiting for the IMAP client to read/write data). + + director: Implemented director_proxy_maybe passdb extra field to + be able to run director and backend in the same Dovecot instance. + (LMTP doesn't support mixed proxy/non-proxy destinations currently.) + + doveadm: Added -F parameter to read a list of users from the + given file and run the command for all the users. This is similar to + -A parameter reading the list of users from userdb lookup. + - auth: If auth_master_user_separator was set, auth process could be + crashed by trying to log in with empty master username. + - imap-login, pop3-login: Fixed crash on handshake failures with new + OpenSSL versions (v1.0.2) when SSLv3 was disabled. + - auth: If one passdb fails allow_nets check, it shouldn't have failed + all the other passdb checks later on. + - imap: Server METADATA couldn't be accessed + - imapc: Fixed \Muted label handling in gmail-migration. + - imapc: Various bugfixes and improvements. + - Trash plugin fixes by Alexei Gradinari + - mbox: Fixed crash/corruption in some situations when the first mail + was expunged. + v2.2.16 2015-03-12 Timo Sirainen * dbox: Resyncing (e.g. doveadm force-resync) no longer deletes diff -r 7799886e017a -r da685736985a configure.ac --- a/configure.ac Sat May 09 20:26:06 2015 +0300 +++ b/configure.ac Sat May 09 20:29:52 2015 +0300 @@ -2,8 +2,8 @@ # Be sure to update ABI version also if anything changes that might require # recompiling plugins. Most importantly that means if any structs are changed. -AC_INIT([Dovecot],[2.2.16],[dovecot at dovecot.org]) -AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv16($PACKAGE_VERSION)", [Dovecot ABI version]) +AC_INIT([Dovecot],[2.2.17.rc1],[dovecot at dovecot.org]) +AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv17($PACKAGE_VERSION)", [Dovecot ABI version]) AC_CONFIG_SRCDIR([src]) From dovecot at dovecot.org Sat May 9 17:45:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 17:45:14 +0000 Subject: dovecot-2.2: Added tag 2.2.17.rc1 for changeset da685736985a Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8e9dfdfea3ce changeset: 18622:8e9dfdfea3ce user: Timo Sirainen date: Sat May 09 20:29:52 2015 +0300 description: Added tag 2.2.17.rc1 for changeset da685736985a diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r da685736985a -r 8e9dfdfea3ce .hgtags --- a/.hgtags Sat May 09 20:29:52 2015 +0300 +++ b/.hgtags Sat May 09 20:29:52 2015 +0300 @@ -123,3 +123,4 @@ e8b793f2c4091f76075e66d5166b38cf82e598bd 2.2.15 a3c27cec411261b3ff2947795b3b8ff96a5320a1 2.2.16.rc1 0f8f0d8ee60745f6ce7ff379da7f2660aec7c4d4 2.2.16 +da685736985a17c2b6b1ff24c4f762a4009ce598 2.2.17.rc1 From dovecot at dovecot.org Sat May 9 17:45:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 17:45:14 +0000 Subject: dovecot-2.2: Added signature for changeset da685736985a Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8b20d717a3e9 changeset: 18623:8b20d717a3e9 user: Timo Sirainen date: Sat May 09 20:29:55 2015 +0300 description: Added signature for changeset da685736985a diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 8e9dfdfea3ce -r 8b20d717a3e9 .hgsigs --- a/.hgsigs Sat May 09 20:29:52 2015 +0300 +++ b/.hgsigs Sat May 09 20:29:55 2015 +0300 @@ -86,3 +86,4 @@ e8b793f2c4091f76075e66d5166b38cf82e598bd 0 iEYEABECAAYFAlRLH5gACgkQyUhSUUBVisn+6gCfcGV1kM2OS9udaFxY7MVtqaaGwYAAnRxUc/RGeUhvJGdg9LvSSzBFPCfz a3c27cec411261b3ff2947795b3b8ff96a5320a1 0 iEYEABECAAYFAlT5zBYACgkQyUhSUUBVisnQ4gCdFb/4T+WZYCAxRKjfup+xIiP26bkAn2CbIlsJBU1f3WnAJX9KsAmhphlY 0f8f0d8ee60745f6ce7ff379da7f2660aec7c4d4 0 iEYEABECAAYFAlUBs5IACgkQyUhSUUBVisn/yACcCUC3xDOCPFAgTKd72uyDrEUv7wcAn1tNcf+AyIUmM9ksDeB2fuZZZJFT +da685736985a17c2b6b1ff24c4f762a4009ce598 0 iEYEABECAAYFAlVORBAACgkQyUhSUUBViskBnQCfRmUjoFty2pUJrrrdIvkEgOCiVEUAnRK0OYM4z5bxs9xcBlXZn8n2qyFx From dovecot at dovecot.org Sat May 9 18:24:45 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 09 May 2015 18:24:45 +0000 Subject: dovecot-2.2: pop3-migration: Added more debug and error logging. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3903badc4ee0 changeset: 18624:3903badc4ee0 user: Timo Sirainen date: Sat May 09 21:22:50 2015 +0300 description: pop3-migration: Added more debug and error logging. diffstat: src/plugins/pop3-migration/pop3-migration-plugin.c | 29 ++++++++++++++++++--- 1 files changed, 24 insertions(+), 5 deletions(-) diffs (86 lines): diff -r 8b20d717a3e9 -r 3903badc4ee0 src/plugins/pop3-migration/pop3-migration-plugin.c --- a/src/plugins/pop3-migration/pop3-migration-plugin.c Sat May 09 20:29:55 2015 +0300 +++ b/src/plugins/pop3-migration/pop3-migration-plugin.c Sat May 09 21:22:50 2015 +0300 @@ -286,8 +286,11 @@ map->size = size; } - if (mailbox_search_deinit(&ctx) < 0) + if (mailbox_search_deinit(&ctx) < 0) { + i_error("pop3_migration: Failed to search all POP3 mails: %s", + mailbox_get_last_error(pop3_box, NULL)); ret = -1; + } (void)mailbox_transaction_commit(&t); return ret; } @@ -331,8 +334,11 @@ map->hdr_sha1_set = TRUE; } - if (mailbox_search_deinit(&ctx) < 0) + if (mailbox_search_deinit(&ctx) < 0) { + i_error("pop3_migration: Failed to search all POP3 mail hashes: %s", + mailbox_get_last_error(pop3_box, NULL)); ret = -1; + } (void)mailbox_transaction_commit(&t); if (ret == 0 && first_seq == 1) mstorage->pop3_all_hdr_sha1_set = TRUE; @@ -377,8 +383,11 @@ map->psize = psize; } - if (mailbox_search_deinit(&ctx) < 0) + if (mailbox_search_deinit(&ctx) < 0) { + i_error("pop3_migration: Failed to search all IMAP mails: %s", + mailbox_get_last_error(box, NULL)); ret = -1; + } (void)mailbox_transaction_commit(&t); return ret; } @@ -410,8 +419,11 @@ map->hdr_sha1_set = TRUE; } - if (mailbox_search_deinit(&ctx) < 0) + if (mailbox_search_deinit(&ctx) < 0) { + i_error("pop3_migration: Failed to search all IMAP mail hashes: %s", + mailbox_get_last_error(box, NULL)); ret = -1; + } (void)mailbox_transaction_commit(&t); return ret; } @@ -443,6 +455,8 @@ imap_map[i].pop3_seq = pop3_map[i].pop3_seq; } mbox->first_unfound_idx = i; + if (box->storage->user->mail_debug) + i_debug("pop3_migration: %u/%u mails matched by size", i, count); return i == count; } @@ -511,6 +525,8 @@ } i_warning("pop3_migration: %u POP3 messages have no " "matching IMAP messages", missing_uids_count); + } else if (box->storage->user->mail_debug) { + i_debug("pop3_migration: %u mails matched by headers", pop3_count); } array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_pop3_seq_cmp); array_sort(&mbox->imap_msg_map, imap_msg_map_uid_cmp); @@ -666,8 +682,11 @@ pop3_box_vname = mail_user_plugin_getenv(storage->user, "pop3_migration_mailbox"); - if (pop3_box_vname == NULL) + if (pop3_box_vname == NULL) { + if (storage->user->mail_debug) + i_debug("pop3_migration: No pop3_migration_mailbox setting - disabled"); return; + } mstorage = p_new(storage->pool, struct pop3_migration_mail_storage, 1); mstorage->module_ctx.super = *v; From dovecot at dovecot.org Sun May 10 08:22:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 10 May 2015 08:22:34 +0000 Subject: dovecot-2.2: lib-http: http-server now always creates a payload ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/05ee9da60255 changeset: 18625:05ee9da60255 user: Timo Sirainen date: Sun May 10 11:20:37 2015 +0300 description: lib-http: http-server now always creates a payload istream, even empty one. This simplifies the caller's logic so that it doesn't need to explicitly check if payload is NULL everywhere. diffstat: src/lib-http/http-server-connection.c | 13 ++++++++----- 1 files changed, 8 insertions(+), 5 deletions(-) diffs (24 lines): diff -r 3903badc4ee0 -r 05ee9da60255 src/lib-http/http-server-connection.c --- a/src/lib-http/http-server-connection.c Sat May 09 21:22:50 2015 +0300 +++ b/src/lib-http/http-server-connection.c Sun May 10 11:20:37 2015 +0300 @@ -296,12 +296,15 @@ actual payload stream. */ conn->incoming_payload = req->req.payload = i_stream_create_limit(req->req.payload, (uoff_t)-1); - i_stream_add_destroy_callback(req->req.payload, - http_server_payload_destroyed, req); - /* the callback may add its own I/O, so we need to remove - our one before calling it */ - http_server_connection_input_halt(conn); + } else { + conn->incoming_payload = req->req.payload = + i_stream_create_from_data("", 0); } + i_stream_add_destroy_callback(req->req.payload, + http_server_payload_destroyed, req); + /* the callback may add its own I/O, so we need to remove + our one before calling it */ + http_server_connection_input_halt(conn); http_server_connection_request_callback(conn, req); if (conn->closed) { From dovecot at dovecot.org Sun May 10 09:07:05 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 10 May 2015 09:07:05 +0000 Subject: dovecot-2.2: fts: Fixed crash when not using lib-fts Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fddd3dbdf987 changeset: 18626:fddd3dbdf987 user: Timo Sirainen date: Sun May 10 12:05:06 2015 +0300 description: fts: Fixed crash when not using lib-fts diffstat: src/plugins/fts/fts-build-mail.c | 31 ++++++++++++++++++++----------- 1 files changed, 20 insertions(+), 11 deletions(-) diffs (48 lines): diff -r 05ee9da60255 -r fddd3dbdf987 src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Sun May 10 11:20:37 2015 +0300 +++ b/src/plugins/fts/fts-build-mail.c Sun May 10 12:05:06 2015 +0300 @@ -133,6 +133,23 @@ return FALSE; } +static void +fts_build_tokenized_hdr_update_lang(struct fts_mail_build_context *ctx, + const struct message_header_line *hdr) +{ + /* Headers that don't contain any human language will only be + translated to lowercase - no stemming or other filtering. There's + unfortunately no pefect way of detecting which headers contain + human languages, so we have a list of some hardcoded header names + and we'll also assume that if there's any 8bit content it's a human + language. */ + if (header_has_language(hdr->name) || + data_has_8bit(hdr->full_value, hdr->full_value_len)) + ctx->cur_user_lang = NULL; + else + ctx->cur_user_lang = fts_user_get_data_lang(ctx->update_ctx->backend->ns->user); +} + static void fts_build_mail_header(struct fts_mail_build_context *ctx, const struct message_block *block) { @@ -151,17 +168,9 @@ key.part = block->part; key.hdr_name = hdr->name; - /* Headers that don't contain any human language will only be - translated to lowercase - no stemming or other filtering. There's - unfortunately no pefect way of detecting which headers contain - human languages, so we have a list of some hardcoded header names - and we'll also assume that if there's any 8bit content it's a human - language. */ - if (header_has_language(key.hdr_name) || - data_has_8bit(hdr->full_value, hdr->full_value_len)) - ctx->cur_user_lang = NULL; - else - ctx->cur_user_lang = fts_user_get_data_lang(ctx->update_ctx->backend->ns->user); + if ((ctx->update_ctx->backend->flags & + FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) + fts_build_tokenized_hdr_update_lang(ctx, hdr); if (!fts_backend_update_set_build_key(ctx->update_ctx, &key)) return; From dovecot at dovecot.org Sun May 10 09:31:47 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 10 May 2015 09:31:47 +0000 Subject: dovecot-2.2: lib-index: If header is corrupted after syncing, lo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/69630e6048fd changeset: 18627:69630e6048fd user: Timo Sirainen date: Sun May 10 12:28:09 2015 +0300 description: lib-index: If header is corrupted after syncing, log the reason why. diffstat: src/lib-index/mail-index-map-hdr.c | 102 ++++++++++++++++++++++---------- src/lib-index/mail-index-map-read.c | 22 +++++- src/lib-index/mail-index-private.h | 10 ++- src/lib-index/mail-index-sync-update.c | 8 +- 4 files changed, 100 insertions(+), 42 deletions(-) diffs (299 lines): diff -r fddd3dbdf987 -r 69630e6048fd src/lib-index/mail-index-map-hdr.c --- a/src/lib-index/mail-index-map-hdr.c Sun May 10 12:05:06 2015 +0300 +++ b/src/lib-index/mail-index-map-hdr.c Sun May 10 12:28:09 2015 +0300 @@ -167,7 +167,7 @@ bool mail_index_check_header_compat(struct mail_index *index, const struct mail_index_header *hdr, - uoff_t file_size) + uoff_t file_size, const char **error_r) { enum mail_index_header_compat_flags compat_flags = 0; @@ -176,35 +176,34 @@ #endif if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) { - /* major version change - handle silently(?) */ + /* major version change */ + *error_r = t_strdup_printf("Major version changed (%u != %u)", + hdr->major_version, MAIL_INDEX_MAJOR_VERSION); return FALSE; } if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) { /* we've already complained about it */ + *error_r = "Header's corrupted flag is set"; return FALSE; } if (hdr->compat_flags != compat_flags) { /* architecture change */ - mail_index_set_error(index, "Rebuilding index file %s: " - "CPU architecture changed", - index->filepath); + *error_r = "CPU architecture changed"; return FALSE; } if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE || hdr->header_size < hdr->base_header_size) { - mail_index_set_error(index, "Corrupted index file %s: " - "Corrupted header sizes (base %u, full %u)", - index->filepath, hdr->base_header_size, - hdr->header_size); + *error_r = t_strdup_printf( + "Corrupted header sizes (base %u, full %u)", + hdr->base_header_size, hdr->header_size); return FALSE; } if (hdr->header_size > file_size) { - mail_index_set_error(index, "Corrupted index file %s: " - "Corrupted header size (%u > %"PRIuUOFF_T")", - index->filepath, hdr->header_size, - file_size); + *error_r = t_strdup_printf( + "Header size is larger than file (%u > %"PRIuUOFF_T")", + hdr->header_size, file_size); return FALSE; } @@ -218,7 +217,6 @@ index->indexid = hdr->indexid; mail_transaction_log_indexid_changed(index->log); } - return TRUE; } @@ -233,33 +231,51 @@ } } -int mail_index_map_check_header(struct mail_index_map *map) +int mail_index_map_check_header(struct mail_index_map *map, + const char **error_r) { struct mail_index *index = map->index; const struct mail_index_header *hdr = &map->hdr; - if (!mail_index_check_header_compat(index, hdr, (uoff_t)-1)) + if (!mail_index_check_header_compat(index, hdr, (uoff_t)-1, error_r)) return -1; /* following some extra checks that only take a bit of CPU */ if (hdr->record_size < sizeof(struct mail_index_record)) { - mail_index_set_error(index, "Corrupted index file %s: " - "record_size too small: %u < %"PRIuSIZE_T, - index->filepath, hdr->record_size, - sizeof(struct mail_index_record)); + *error_r = t_strdup_printf( + "record_size too small (%u < %"PRIuSIZE_T")", + hdr->record_size, sizeof(struct mail_index_record)); return -1; } - if (hdr->uid_validity == 0 && hdr->next_uid != 1) + if (hdr->uid_validity == 0 && hdr->next_uid != 1) { + *error_r = t_strdup_printf( + "uidvalidity=0, but next_uid=%u", hdr->next_uid); return 0; - if (hdr->next_uid == 0) + } + if (hdr->next_uid == 0) { + *error_r = "next_uid=0"; return 0; - if (hdr->messages_count > map->rec_map->records_count) + } + if (hdr->messages_count > map->rec_map->records_count) { + *error_r = t_strdup_printf( + "messages_count is higher in header than record map (%u > %u)", + hdr->messages_count, map->rec_map->records_count); return 0; + } - if (hdr->seen_messages_count > hdr->messages_count || - hdr->deleted_messages_count > hdr->messages_count) + if (hdr->seen_messages_count > hdr->messages_count) { + *error_r = t_strdup_printf( + "seen_messages_count %u > messages_count %u", + hdr->seen_messages_count, hdr->messages_count); return 0; + } + if (hdr->deleted_messages_count > hdr->messages_count) { + *error_r = t_strdup_printf( + "deleted_messages_count %u > messages_count %u", + hdr->deleted_messages_count, hdr->messages_count); + return 0; + } switch (hdr->minor_version) { case 0: /* upgrade silently from v1.0 */ @@ -279,11 +295,28 @@ map->hdr.unused_old_sync_size = 0; map->hdr.unused_old_sync_stamp = 0; } - if (hdr->first_recent_uid == 0 || - hdr->first_recent_uid > hdr->next_uid || - hdr->first_unseen_uid_lowwater > hdr->next_uid || - hdr->first_deleted_uid_lowwater > hdr->next_uid) + if (hdr->first_recent_uid == 0) { + *error_r = "first_recent_uid=0"; return 0; + } + if (hdr->first_recent_uid > hdr->next_uid) { + *error_r = t_strdup_printf( + "first_recent_uid %u > next_uid %u", + hdr->first_recent_uid, hdr->next_uid); + return 0; + } + if (hdr->first_unseen_uid_lowwater > hdr->next_uid) { + *error_r = t_strdup_printf( + "first_unseen_uid_lowwater %u > next_uid %u", + hdr->first_unseen_uid_lowwater, hdr->next_uid); + return 0; + } + if (hdr->first_deleted_uid_lowwater > hdr->next_uid) { + *error_r = t_strdup_printf( + "first_deleted_uid_lowwater %u > next_uid %u", + hdr->first_deleted_uid_lowwater, hdr->next_uid); + return 0; + } if (hdr->messages_count > 0) { /* last message's UID must be smaller than next_uid. @@ -291,9 +324,16 @@ const struct mail_index_record *rec; rec = MAIL_INDEX_REC_AT_SEQ(map, hdr->messages_count); - if (rec->uid == 0 || rec->uid >= hdr->next_uid) + if (rec->uid == 0) { + *error_r = "last message has uid=0"; + return -1; + } + if (rec->uid >= hdr->next_uid) { + *error_r = t_strdup_printf( + "last message uid %u >= next_uid %u", + rec->uid, hdr->next_uid); return 0; + } } - return 1; } diff -r fddd3dbdf987 -r 69630e6048fd src/lib-index/mail-index-map-read.c --- a/src/lib-index/mail-index-map-read.c Sun May 10 12:05:06 2015 +0300 +++ b/src/lib-index/mail-index-map-read.c Sun May 10 12:28:09 2015 +0300 @@ -31,6 +31,7 @@ struct mail_index *index = map->index; struct mail_index_record_map *rec_map = map->rec_map; const struct mail_index_header *hdr; + const char *error; i_assert(rec_map->mmap_base == NULL); @@ -66,9 +67,11 @@ return 0; } - if (!mail_index_check_header_compat(index, hdr, rec_map->mmap_size)) { + if (!mail_index_check_header_compat(index, hdr, rec_map->mmap_size, &error)) { /* Can't use this file */ - return 0; + mail_index_set_error(index, "Corrupted index file %s: %s", + index->filepath, error); + return -1; } rec_map->mmap_used_size = hdr->header_size + @@ -126,6 +129,7 @@ struct mail_index *index = map->index; const struct mail_index_header *hdr; unsigned char read_buf[IO_BLOCK_SIZE]; + const char *error; const void *buf; void *data = NULL; ssize_t ret; @@ -146,9 +150,11 @@ if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE && (ret > 0 || pos >= hdr->base_header_size)) { - if (!mail_index_check_header_compat(index, hdr, file_size)) { + if (!mail_index_check_header_compat(index, hdr, file_size, &error)) { /* Can't use this file */ - return 0; + mail_index_set_error(index, "Corrupted index file %s: %s", + index->filepath, error); + return -1; } initial_buf_pos = pos; @@ -295,6 +301,7 @@ struct stat st; uoff_t file_size; bool use_mmap, unusable = FALSE; + const char *error; int ret, try; ret = mail_index_reopen_if_changed(index); @@ -339,7 +346,12 @@ for (try = 0; ret > 0; try++) { /* make sure the header is ok before using this mapping */ - ret = mail_index_map_check_header(new_map); + ret = mail_index_map_check_header(new_map, &error); + if (ret < 0) { + mail_index_set_error(index, + "Corrupted index file %s: %s", + index->filepath, error); + } if (ret > 0) T_BEGIN { if (mail_index_map_parse_extensions(new_map) < 0) ret = 0; diff -r fddd3dbdf987 -r 69630e6048fd src/lib-index/mail-index-private.h --- a/src/lib-index/mail-index-private.h Sun May 10 12:05:06 2015 +0300 +++ b/src/lib-index/mail-index-private.h Sun May 10 12:28:09 2015 +0300 @@ -307,10 +307,16 @@ uint32_t *first_seq_r, uint32_t *last_seq_r); -int mail_index_map_check_header(struct mail_index_map *map); +/* Returns 1 on success, 0 on non-critical errors we want to silently fix, + -1 if map isn't usable. The caller is responsible for logging the errors + if -1 is returned. */ +int mail_index_map_check_header(struct mail_index_map *map, + const char **error_r); +/* Returns 1 if header is usable, 0 or -1 if not. The caller should log an + error if -1 is returned, but not if 0 is returned. */ bool mail_index_check_header_compat(struct mail_index *index, const struct mail_index_header *hdr, - uoff_t file_size); + uoff_t file_size, const char **error_r); int mail_index_map_parse_extensions(struct mail_index_map *map); int mail_index_map_parse_keywords(struct mail_index_map *map); diff -r fddd3dbdf987 -r 69630e6048fd src/lib-index/mail-index-sync-update.c --- a/src/lib-index/mail-index-sync-update.c Sun May 10 12:05:06 2015 +0300 +++ b/src/lib-index/mail-index-sync-update.c Sun May 10 12:28:09 2015 +0300 @@ -911,7 +911,7 @@ const void *tdata; uint32_t prev_seq; uoff_t start_offset, prev_offset; - const char *reason; + const char *reason, *error; int ret; bool had_dirty, reset; @@ -1075,10 +1075,10 @@ i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW); - if (mail_index_map_check_header(map) <= 0) { + if (mail_index_map_check_header(map, &error) <= 0) { mail_index_set_error(index, - "Synchronization corrupted index header: %s", - index->filepath); + "Synchronization corrupted index header %s: %s", + index->filepath, error); (void)mail_index_fsck(index); map = index->map; } else if (sync_map_ctx.errors) { From pigeonhole at rename-it.nl Sun May 10 20:52:17 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 10 May 2015 22:52:17 +0200 Subject: dovecot-2.2-pigeonhole: Updated example configuration. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/d9ae92effdf2 changeset: 2050:d9ae92effdf2 user: Stephan Bosch date: Sun May 10 22:52:09 2015 +0200 description: Updated example configuration. diffstat: doc/example-config/conf.d/90-sieve.conf | 12 ++++++++---- 1 files changed, 8 insertions(+), 4 deletions(-) diffs (49 lines): diff -r f050c1de127c -r d9ae92effdf2 doc/example-config/conf.d/90-sieve.conf --- a/doc/example-config/conf.d/90-sieve.conf Thu May 07 22:05:23 2015 +0200 +++ b/doc/example-config/conf.d/90-sieve.conf Sun May 10 22:52:09 2015 +0200 @@ -39,13 +39,17 @@ sieve = file:~/sieve;active=~/.dovecot.sieve # The default Sieve script when the user has none. This is the location of a - # global sieve script file, which gets executed ONLY if user's personal Sieve + # global sieve script file, which gets executed ONLY if user's personal Sieve # script doesn't exist. Be sure to pre-compile this script manually using the # sievec command line tool if the binary is not stored in a global location. # --> See sieve_before for executing scripts before the user's personal # script. #sieve_default = /var/lib/dovecot/sieve/default.sieve + # The name by which the default Sieve script (as configured by the + # sieve_default setting) is visible to the user through ManageSieve. + #sieve_default_name = + # Location for ":global" include scripts as used by the "include" extension. #sieve_global = @@ -66,7 +70,7 @@ # Identical to sieve_before, only the specified scripts are executed after the # user's script (only when keep is still in effect!). Multiple script - # locations can be specified by appending an increasing number. + # locations can be specified by appending an increasing number. #sieve_after = #sieve_after2 = #sieve_after2 = (etc...) @@ -77,7 +81,7 @@ # to disable certain Sieve extensions or enable those that are not available # by default. This setting can use '+' and '-' to specify differences relative # to the default. For example `sieve_extensions = +imapflags' will enable the - # deprecated imapflags extension in addition to all extensions were already + # deprecated imapflags extension in addition to all extensions were already # enabled by default. #sieve_extensions = +notify +imapflags @@ -97,7 +101,7 @@ # setting, the used plugins can be specified. Check the Dovecot wiki # (wiki2.dovecot.org) or the pigeonhole website # (http://pigeonhole.dovecot.org) for available plugins. - # The sieve_extprograms plugin is included in this release. + # The sieve_extprograms plugin is included in this release. #sieve_plugins = # The separator that is expected between the :user and :detail From pigeonhole at rename-it.nl Sun May 10 22:19:30 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 May 2015 00:19:30 +0200 Subject: dovecot-2.2-pigeonhole: Added tag 0.4.8.rc1 for changeset fcc97e... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/eb358910d318 changeset: 2052:eb358910d318 user: Stephan Bosch date: Mon May 11 00:19:16 2015 +0200 description: Added tag 0.4.8.rc1 for changeset fcc97e953584 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r fcc97e953584 -r eb358910d318 .hgtags --- a/.hgtags Mon May 11 00:19:00 2015 +0200 +++ b/.hgtags Mon May 11 00:19:16 2015 +0200 @@ -27,3 +27,4 @@ fd050f466314f911900027720dcafaa86e3cd537 0.4.7.rc2 87b2d3e43a447f000b06ce1ee1ab73456c7357b3 0.4.7.rc3 f2323a90c202c4e9708a0c87e81a1b06d267a5c5 0.4.7 +fcc97e953584269e5fa140363804f4016c33ef1c 0.4.8.rc1 From pigeonhole at rename-it.nl Sun May 10 22:19:30 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 May 2015 00:19:30 +0200 Subject: dovecot-2.2-pigeonhole: Released v0.4.8.rc1 for Dovecot v2.2.17.... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/fcc97e953584 changeset: 2051:fcc97e953584 user: Stephan Bosch date: Mon May 11 00:19:00 2015 +0200 description: Released v0.4.8.rc1 for Dovecot v2.2.17.rc1. diffstat: NEWS | 18 ++++++++++++++++++ configure.ac | 2 +- 2 files changed, 19 insertions(+), 1 deletions(-) diffs (34 lines): diff -r d9ae92effdf2 -r fcc97e953584 NEWS --- a/NEWS Sun May 10 22:52:09 2015 +0200 +++ b/NEWS Mon May 11 00:19:00 2015 +0200 @@ -1,3 +1,21 @@ +v0.4.8 xx-05-2015 Stephan Bosch + + * LDA Sieve plugin: Dovecot changed the deliver_log_format setting to include + %{delivery_time}. This prompted changes in Pigeonhole that make this release + dependent on Dovecot v2.2.17. + + Implemented magic to make sieve_default script visible from ManageSieve + under a configurable name. This way, users can see the default rules, edit + them and store a private adjusted version. This could also be achieved by + copying the default script into the user's script storage, but that way + updates to the global sieve_default script would be ignored. + - Fixed problem in address test: erroneously decoded mime-encoded words in + address headers. + - extprograms plugin: Fixed failure occurring when connecting to script + service without the need to read back the output from the external program. + - Fixed bug in script storage path normalization occurring with relative + symbolic links below root. + - Fixed and updated various parts of the documentation + v0.4.7 19-03-2015 Stephan Bosch * editheader extension: Made protection against addition and deletion of diff -r d9ae92effdf2 -r fcc97e953584 configure.ac --- a/configure.ac Sun May 10 22:52:09 2015 +0200 +++ b/configure.ac Mon May 11 00:19:00 2015 +0200 @@ -1,4 +1,4 @@ -AC_INIT([Pigeonhole], [0.4.7], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) +AC_INIT([Pigeonhole], [0.4.8.rc1], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_SRCDIR([src]) AC_CONFIG_MACRO_DIR([m4]) From pigeonhole at rename-it.nl Sun May 10 22:22:24 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 May 2015 00:22:24 +0200 Subject: dovecot-2.2-pigeonhole: Added signature for changeset fcc97e953584 Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/052cd67b423f changeset: 2053:052cd67b423f user: Stephan Bosch date: Mon May 11 00:23:22 2015 +0200 description: Added signature for changeset fcc97e953584 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r eb358910d318 -r 052cd67b423f .hgsigs --- a/.hgsigs Mon May 11 00:19:16 2015 +0200 +++ b/.hgsigs Mon May 11 00:23:22 2015 +0200 @@ -21,3 +21,4 @@ fd050f466314f911900027720dcafaa86e3cd537 0 iQEcBAABAgAGBQJVBMEkAAoJEATWKx49+7T0RcMH/2Hyfa0M7oBB0M8gY9jVtGoIXC2U37+crLduKSEsJWkchW2/SkuTLbxbVPb4oBXbJJOiDVNVVvjhHefqxbVDJlWVdcTz8hdZam0Wl2LKX4OlvpsLBiCil/gNoQSCABd5Q4zUCa18mBU9PIltlG61IgTxXtz0A5IiCI7N34l3FAwkxSvaCUVR82y5M7dTat7OEDOpVVUIucf9DPrLcGwTFJ/TuOxiI9lH/MKrbRVeOyrs+3hNAzcArBfvvMy7cVyUV85u8EqgFIL6V5mrp4YNOgkuqvvdTDThs14w1GV1ILl9Y4ycUzYDDqqbXnfK/j5mvTNSelnV/JBZC+moJozptmI= 87b2d3e43a447f000b06ce1ee1ab73456c7357b3 0 iQEcBAABAgAGBQJVB4QIAAoJEATWKx49+7T0MKcH/0Wg6xazPO9Q3v8rYZe4PFSlWoXv4P8ixy275pouHyLWKDltg2zVPq3ieDNLkOfNiQvBClWppQa5LPzgTN/9x5MmIYU/IpdXCKvl8i9G3qzRaGUL1oMtp3JocI8JyKx3R61V8u4VyJ+qBg0kLCs85ilNdo1ia9uFCYbZh7QOnM5FHJAsGzDPEriZyrcf3D/CTGDedYfoELEJXpSzW19Ov1qPIq501mBfrxXSd/MTUFkm8DExYW7i/YTBEsJNn58y1tsdC6FxuP8bZJRQlS/4PRR5euACxKpEV38Ig7b22a6P/MoZphCvZkIMBIxIB9RIEIljRyXSKESvssgUoA/v1Q8= f2323a90c202c4e9708a0c87e81a1b06d267a5c5 0 iQEcBAABAgAGBQJVC01gAAoJEATWKx49+7T0eNQH/iHUSapqKCaPoZByc1UHpp/FS7IEzbXUWTa71uTHN7z+/Gr6vKm4/xl0k10N6XhpY5aJImLt31dd8jSDg6hqhVwDlqIImJzCjf+flG4dZu5MIL2vibjEn7InYvMli5EcafQ08EkTWGm7IPOcqfnZheevU64piqaUr1YfNaR0g/cQcffvT9qXNovlqFQe+hDB2/wUj6zKiHx0rqUzAOh4dAcrfq1jCpXhpGChjezxyiGzFpLPopbEHQU9M9Fr6VFgY+v/mDVp6hj8DpE7Q84BxPvxJKi0il+h0/lliDZUA+XtZsVjckih5WBDC5tLgNRKujMzkAc2UNEU/1vJxkPkldI= +fcc97e953584269e5fa140363804f4016c33ef1c 0 iQEcBAABAgAGBQJVT9pVAAoJEATWKx49+7T0/14H/3iejYaCOqm5ftfQDMlebUhbMa9LGEMwOPRoBxnzGOBoJGEQ7MS/aqk8ARM1i016jFtDa0oVWkTDyWZhDwQHS6yA2HkXPxizKHf9+9Kc2Lxjew4GIyTGSNmkaJb34XUKGhKx4YMfDXQiFsc0G/vVSO8UoZzvqho6g7bUmuWeuCrXmEW+M+bUG4KMCF0abYba7iasYKaV+1fSnbAnh5kRQqwfUfs/ugdU4f9VZxmnGdJymf540KdoqEzePMaxhEp6OPhhr/eVGabfWhV+RJiRI7atWQl7I6uoEkvziObuUKx3rl0xVoSVcKeCC4APsJdQQT/HDoPmWfyaQCV+WORmz6U= From pigeonhole at rename-it.nl Mon May 11 08:03:08 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 May 2015 10:03:08 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: storage: Fixed segfault probl... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/c00f51f3c91a changeset: 2054:c00f51f3c91a user: Stephan Bosch date: Mon May 11 10:03:02 2015 +0200 description: lib-sieve: storage: Fixed segfault problem in main storage initialization. Caused by earlier sieve_default_name-related changes. diffstat: src/lib-sieve/sieve-storage.c | 25 ++++++++++++------------- 1 files changed, 12 insertions(+), 13 deletions(-) diffs (40 lines): diff -r 052cd67b423f -r c00f51f3c91a src/lib-sieve/sieve-storage.c --- a/src/lib-sieve/sieve-storage.c Mon May 11 00:23:22 2015 +0200 +++ b/src/lib-sieve/sieve-storage.c Mon May 11 10:03:02 2015 +0200 @@ -497,24 +497,23 @@ /* Attempt to locate user's main storage */ storage = sieve_storage_do_create_main(svinst, user, flags, error_r); - - set_default_name = - sieve_setting_get(svinst, "sieve_default_name"); - if ( set_default_name != NULL && *set_default_name != '\0' && - !sieve_script_name_is_valid(set_default_name) ) { - sieve_storage_sys_error(storage, - "Invalid script name `%s' for `sieve_default_name' setting.", - str_sanitize(set_default_name, 80)); - set_default_name = NULL; - } - storage->default_name = - p_strdup_empty(storage->pool, set_default_name); - if ( storage != NULL ) { /* Success; record default script location for later use */ storage->default_location = p_strdup_empty(storage->pool, set_default); + set_default_name = + sieve_setting_get(svinst, "sieve_default_name"); + if ( set_default_name != NULL && *set_default_name != '\0' && + !sieve_script_name_is_valid(set_default_name) ) { + sieve_storage_sys_error(storage, + "Invalid script name `%s' for `sieve_default_name' setting.", + str_sanitize(set_default_name, 80)); + set_default_name = NULL; + } + storage->default_name = + p_strdup_empty(storage->pool, set_default_name); + if (storage->default_location != NULL && storage->default_name != NULL) { sieve_storage_sys_debug(storage, From dovecot at dovecot.org Mon May 11 09:31:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 09:31:07 +0000 Subject: dovecot-2.2: lmtp: chdir() to base_dir shouldn't fail - log an e... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/413962f2b7e7 changeset: 18628:413962f2b7e7 user: Timo Sirainen date: Mon May 11 12:27:39 2015 +0300 description: lmtp: chdir() to base_dir shouldn't fail - log an error if it does diffstat: src/lmtp/commands.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r 69630e6048fd -r 413962f2b7e7 src/lmtp/commands.c --- a/src/lmtp/commands.c Sun May 10 12:28:09 2015 +0300 +++ b/src/lmtp/commands.c Mon May 11 12:27:39 2015 +0300 @@ -1037,7 +1037,9 @@ /* enable core dumping again. we need to chdir also to root-owned directory to get core dumps. */ restrict_access_allow_coredumps(TRUE); - (void)chdir(base_dir); + if (chdir(base_dir) < 0) + i_error("chdir(%s) failed: %m", base_dir); + } } } From dovecot at dovecot.org Mon May 11 09:31:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 09:31:07 +0000 Subject: dovecot-2.2: Compiler warning fixes Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9c4d1e1e252f changeset: 18629:9c4d1e1e252f user: Timo Sirainen date: Mon May 11 12:28:58 2015 +0300 description: Compiler warning fixes diffstat: src/imap-urlauth/imap-urlauth.c | 8 ++++++-- src/plugins/fts-lucene/lucene-wrapper.cc | 5 ++--- src/pop3/main.c | 4 +++- 3 files changed, 11 insertions(+), 6 deletions(-) diffs (76 lines): diff -r 413962f2b7e7 -r 9c4d1e1e252f src/imap-urlauth/imap-urlauth.c --- a/src/imap-urlauth/imap-urlauth.c Mon May 11 12:27:39 2015 +0300 +++ b/src/imap-urlauth/imap-urlauth.c Mon May 11 12:28:58 2015 +0300 @@ -143,7 +143,9 @@ i_error("Peer's credentials (uid=%ld) do not match " "the user that logged in (uid=%ld).", (long)cred.uid, (long)reply.uid); - (void)write(client->fd, msg, strlen(msg)); + if (write(client->fd, msg, strlen(msg)) < 0) { + /* ignored */ + } net_disconnect(client->fd); return; } @@ -159,7 +161,9 @@ const char *errormsg ATTR_UNUSED) { const char *msg = "NO\n"; - (void)write(client->fd, msg, strlen(msg)); + if (write(client->fd, msg, strlen(msg)) < 0) { + /* ignored */ + } } static void client_connected(struct master_service_connection *conn) diff -r 413962f2b7e7 -r 9c4d1e1e252f src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Mon May 11 12:27:39 2015 +0300 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Mon May 11 12:28:58 2015 +0300 @@ -99,7 +99,9 @@ }; static void *textcat = NULL; +#ifdef HAVE_FTS_TEXTCAT static bool textcat_broken = FALSE; +#endif static int textcat_refcount = 0; static void lucene_handle_error(struct lucene_index *index, CLuceneError &err, @@ -112,7 +114,6 @@ const struct fts_lucene_settings *set) { struct lucene_index *index; - unsigned int len; index = i_new(struct lucene_index, 1); index->path = i_strdup(path); @@ -410,7 +411,6 @@ static int lucene_settings_check(struct lucene_index *index) { - struct fts_index_header hdr; uint32_t set_checksum; int ret = 0; @@ -858,7 +858,6 @@ { static const TCHAR *sort_fields[] = { _T("box"), _T("uid"), NULL }; struct rescan_context ctx; - guid_128_t guid; bool failed = false; int ret; diff -r 413962f2b7e7 -r 9c4d1e1e252f src/pop3/main.c --- a/src/pop3/main.c Mon May 11 12:27:39 2015 +0300 +++ b/src/pop3/main.c Mon May 11 12:28:58 2015 +0300 @@ -107,7 +107,9 @@ if (mail_storage_service_lookup_next(storage_service, input, &user, &mail_user, error_r) <= 0) { - (void)write(fd_out, lookup_error_str, strlen(lookup_error_str)); + if (write(fd_out, lookup_error_str, strlen(lookup_error_str)) < 0) { + /* ignored */ + } return -1; } restrict_access_allow_coredumps(TRUE); From dovecot at dovecot.org Mon May 11 11:12:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 11:12:20 +0000 Subject: dovecot-2.2: lib-fts: Added asserts to make sure we don't return... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/13461d146be0 changeset: 18630:13461d146be0 user: Timo Sirainen date: Mon May 11 14:10:24 2015 +0300 description: lib-fts: Added asserts to make sure we don't return empty tokens. diffstat: src/lib-fts/fts-filter.c | 4 +++- src/lib-fts/fts-tokenizer.c | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diffs (52 lines): diff -r 9c4d1e1e252f -r 13461d146be0 src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Mon May 11 12:28:58 2015 +0300 +++ b/src/lib-fts/fts-filter.c Mon May 11 14:10:24 2015 +0300 @@ -105,7 +105,9 @@ if (ret <= 0) *token = NULL; - else + else { i_assert(*token != NULL); + i_assert((*token)[0] != '\0'); + } return ret; } diff -r 9c4d1e1e252f -r 13461d146be0 src/lib-fts/fts-tokenizer.c --- a/src/lib-fts/fts-tokenizer.c Mon May 11 12:28:58 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.c Mon May 11 14:10:24 2015 +0300 @@ -173,7 +173,7 @@ case FTS_TOKENIZER_PARENT_STATE_ADD_DATA: ret = fts_tokenizer_next_self(tok, data, size, token_r, error_r); if (ret <= 0 || tok->parent == NULL || tok->skip_parents) - return ret; + break; buffer_set_used_size(tok->parent_input, 0); buffer_append(tok->parent_input, *token_r, strlen(*token_r)); tok->parent_state++; @@ -182,13 +182,13 @@ ret = fts_tokenizer_next(tok->parent, tok->parent_input->data, tok->parent_input->used, token_r, error_r); if (ret != 0) - return ret; + break; tok->parent_state++; /* fall through */ case FTS_TOKENIZER_PARENT_STATE_FINALIZE: ret = fts_tokenizer_next(tok->parent, NULL, 0, token_r, error_r); if (ret != 0) - return ret; + break; /* we're finished sending this token to parent tokenizer. see if our own tokenizer has more tokens available */ tok->parent_state = FTS_TOKENIZER_PARENT_STATE_ADD_DATA; @@ -196,6 +196,9 @@ default: i_unreached(); } + /* we must not be returning empty tokens */ + i_assert(ret <= 0 || (*token_r)[0] != '\0'); + return ret; } int fts_tokenizer_final(struct fts_tokenizer *tok, const char **token_r, From dovecot at dovecot.org Mon May 11 11:24:01 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 11:24:01 +0000 Subject: dovecot-2.2: lmtp: Earlier compiler warning fix broke compiling ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/103f64df4e77 changeset: 18631:103f64df4e77 user: Timo Sirainen date: Mon May 11 14:22:05 2015 +0300 description: lmtp: Earlier compiler warning fix broke compiling completely.. diffstat: src/lmtp/commands.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diffs (11 lines): diff -r 13461d146be0 -r 103f64df4e77 src/lmtp/commands.c --- a/src/lmtp/commands.c Mon May 11 14:10:24 2015 +0300 +++ b/src/lmtp/commands.c Mon May 11 14:22:05 2015 +0300 @@ -1039,7 +1039,6 @@ restrict_access_allow_coredumps(TRUE); if (chdir(base_dir) < 0) i_error("chdir(%s) failed: %m", base_dir); - } } } From dovecot at dovecot.org Mon May 11 11:37:47 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 11:37:47 +0000 Subject: dovecot-2.2: lib-fts: Improved test-fts-tokenizer to run multipl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9f06c6054e3e changeset: 18632:9f06c6054e3e user: Timo Sirainen date: Mon May 11 14:34:50 2015 +0300 description: lib-fts: Improved test-fts-tokenizer to run multiple text inputs diffstat: src/lib-fts/test-fts-tokenizer.c | 168 ++++++++++++++++---------------------- 1 files changed, 73 insertions(+), 95 deletions(-) diffs (290 lines): diff -r 103f64df4e77 -r 9f06c6054e3e src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Mon May 11 14:22:05 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Mon May 11 14:34:50 2015 +0300 @@ -9,22 +9,34 @@ #include -#define TEST_INPUT_TEXT \ - "hello world\r\n\nAnd there\twas: text galore, " \ - "abc at example.com, " \ - "Bar Baz , " \ - "foo at domain " \ - "1234567890123456789012345678?," \ - "12345678901234567890123456789?," \ - "123456789012345678901234567890?," \ - "and longlonglongabcdefghijklmnopqrstuvwxyz more.\n\n " \ - "(\"Hello world\")3.14 3,14 last" #define TEST_INPUT_ADDRESS \ "@invalid invalid@ Abc Dfg , " \ "Bar Baz " \ "Foo Bar (comment)foo.bar at host.example.org " \ "foo, foo at domain" +static const char *test_inputs[] = { + /* generic things and word truncation: */ + "hello world\r\n\nAnd there\twas: text galore, " + "abc at example.com, " + "Bar Baz , " + "foo at domain " + "1234567890123456789012345678?," + "12345678901234567890123456789?," + "123456789012345678901234567890?," + "and longlonglongabcdefghijklmnopqrstuvwxyz more.\n\n " + "(\"Hello world\")3.14 3,14 last", + + /* whitespace: with Unicode(utf8) U+FF01(ef bc 81)(U+2000(e2 80 80) and + U+205A(e2 81 9a) and U+205F(e2 81 9f) */ + "hello\xEF\xBC\x81world\r\nAnd\xE2\x80\x80there\twas: text " + "galore\xE2\x81\x9F""and\xE2\x81\x9Amore.\n\n", + + /* TR29 MinNumLet U+FF0E at end: u+FF0E is EF BC 8E */ + "hello world\xEF\xBC\x8E" + +}; + static void test_fts_tokenizer_find(void) { test_begin("fts tokenizer find"); @@ -33,16 +45,17 @@ test_end(); } -static void +static unsigned int test_tokenizer_inputoutput(struct fts_tokenizer *tok, const char *_input, - const char *const *expected_output) + const char *const *expected_output, + unsigned int first_outi) { const unsigned char *input = (const unsigned char *)_input; const char *token, *error; - unsigned int i, max, outi, char_len, input_len = strlen(_input); + unsigned int i, outi, max, char_len, input_len = strlen(_input); /* test all input at once */ - outi = 0; + outi = first_outi; while (fts_tokenizer_next(tok, input, input_len, &token, &error) > 0) { test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); outi++; @@ -51,10 +64,11 @@ test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); outi++; } - test_assert(expected_output[outi] == NULL); + test_assert_idx(expected_output[outi] == NULL, outi); /* test input one byte at a time */ - for (i = outi = 0; i < input_len; i += char_len) { + outi = first_outi; + for (i = 0; i < input_len; i += char_len) { char_len = uni_utf8_char_bytes(input[i]); while (fts_tokenizer_next(tok, input+i, char_len, &token, &error) > 0) { test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); @@ -65,10 +79,11 @@ test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); outi++; } - test_assert(expected_output[outi] == NULL); + test_assert_idx(expected_output[outi] == NULL, outi); /* test input in random chunks */ - for (i = outi = 0; i < input_len; i += char_len) { + outi = first_outi; + for (i = 0; i < input_len; i += char_len) { max = rand() % (input_len - i) + 1; for (char_len = 0; char_len < max; ) char_len += uni_utf8_char_bytes(input[i+char_len]); @@ -81,12 +96,25 @@ test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); outi++; } - test_assert(expected_output[outi] == NULL); + test_assert_idx(expected_output[outi] == NULL, outi); + return outi+1; +} + +static void +test_tokenizer_inputs(struct fts_tokenizer *tok, + const char *const *expected_output) +{ + unsigned int i, outi = 0; + + for (i = 0; i < N_ELEMENTS(test_inputs); i++) { + outi = test_tokenizer_inputoutput(tok, test_inputs[i], + expected_output, outi); + } + test_assert_idx(expected_output[outi] == NULL, outi); } static void test_fts_tokenizer_generic_only(void) { - static const char input[] = TEST_INPUT_TEXT; static const char *const expected_output[] = { "hello", "world", "And", "there", "was", "text", "galore", @@ -96,7 +124,15 @@ "12345678901234567890123456789", "123456789012345678901234567890", "and", "longlonglongabcdefghijklmnopqr", - "more", "Hello", "world", "3", "14", "3", "14", "last", NULL + "more", "Hello", "world", "3", "14", "3", "14", "last", NULL, + + "hello", "world", "And", + "there", "was", "text", "galore", + "and", "more", NULL, + + "hello", "world", NULL, + + NULL }; struct fts_tokenizer *tok; const char *error; @@ -105,38 +141,17 @@ test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &tok, &error) == 0); test_assert(((struct generic_fts_tokenizer *) tok)->algorithm == BOUNDARY_ALGORITHM_SIMPLE); - test_tokenizer_inputoutput(tok, input, expected_output); - fts_tokenizer_unref(&tok); - test_end(); -} - -static void test_fts_tokenizer_generic_unicode_whitespace(void) -{ - /* with Unicode(utf8) U+FF01(ef bc 81)(U+2000(e2 80 80) and - U+205A(e2 81 9a) and U+205F(e2 81 9f )*/ - static const char input[] = - "hello\xEF\xBC\x81world\r\nAnd\xE2\x80\x80there\twas: text " - "galore\xE2\x81\x9F""and\xE2\x81\x9Amore.\n\n"; - static const char *const expected_output[] = { - "hello", "world", "And", - "there", "was", "text", "galore", - "and", "more", NULL - }; - struct fts_tokenizer *tok; - const char *error; - - test_begin("fts tokenizer generic simple with Unicode whitespace"); - test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &tok, &error) == 0); - test_tokenizer_inputoutput(tok, input, expected_output); + test_tokenizer_inputs(tok, expected_output); fts_tokenizer_unref(&tok); test_end(); } const char *const tr29_settings[] = {"algorithm", "tr29", NULL}; +/* TODO: U+206F is in "Format" and therefore currently not word break. + This definitely needs to be remapped. */ static void test_fts_tokenizer_generic_tr29_only(void) { - static const char input[] = TEST_INPUT_TEXT; static const char *const expected_output[] = { "hello", "world", "And", "there", "was", "text", "galore", @@ -146,56 +161,22 @@ "12345678901234567890123456789", "123456789012345678901234567890", "and", "longlonglongabcdefghijklmnopqr", - "more", "Hello", "world", "3.14", "3,14", "last", NULL + "more", "Hello", "world", "3.14", "3,14", "last", NULL, + + "hello", "world", "And", + "there", "was", "text", "galore", + "and", "more", NULL, + + "hello", "world", NULL, + + NULL }; struct fts_tokenizer *tok; const char *error; test_begin("fts tokenizer generic TR29"); test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0); - test_tokenizer_inputoutput(tok, input, expected_output); - fts_tokenizer_unref(&tok); - test_end(); -} - -/* TODO: U+206F is in "Format" and therefore currently not word break. - This definitely needs to be remapped. */ -static void test_fts_tokenizer_generic_tr29_unicode_whitespace(void) -{ - /* with Unicode(utf8) U+2000(e2 80 80) and U+205A(e2 81 9a) and U+205F(e2 - 81 9f)*/ - static const char input[] = - "hello world\r\nAnd\xE2\x80\x80there\twas: text " - "galore\xE2\x81\x9F""and\xE2\x81\x9Amore.\n\n"; - static const char *const expected_output[] = { - "hello", "world", "And", - "there", "was", "text", "galore", - "and", "more", NULL - }; - struct fts_tokenizer *tok; - const char *error; - - test_begin("fts tokenizer generic TR29 with Unicode whitespace"); - test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0); - test_tokenizer_inputoutput(tok, input, expected_output); - fts_tokenizer_unref(&tok); - test_end(); -} - -static void test_fts_tokenizer_generic_tr29_midnumlet_end(void) -{ - /* u+FF0E is EF BC 8E */ - static const char input[] = - "hello world\xEF\xBC\x8E"; - static const char *const expected_output[] = { - "hello", "world", NULL - }; - struct fts_tokenizer *tok; - const char *error; - - test_begin("fts tokenizer generic TR29 with MinNumLet U+FF0E at end"); - test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0); - test_tokenizer_inputoutput(tok, input, expected_output); + test_tokenizer_inputs(tok, expected_output); fts_tokenizer_unref(&tok); test_end(); } @@ -212,7 +193,7 @@ test_begin("fts tokenizer email address only"); test_assert(fts_tokenizer_create(fts_tokenizer_email_address, NULL, NULL, &tok, &error) == 0); - test_tokenizer_inputoutput(tok, input, expected_output); + test_tokenizer_inputoutput(tok, input, expected_output, 0); fts_tokenizer_unref(&tok); test_end(); } @@ -232,7 +213,7 @@ test_begin("fts tokenizer email address + parent"); test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &gen_tok, &error) == 0); test_assert(fts_tokenizer_create(fts_tokenizer_email_address, gen_tok, NULL, &tok, &error) == 0); - test_tokenizer_inputoutput(tok, input, expected_output); + test_tokenizer_inputoutput(tok, input, expected_output, 0); fts_tokenizer_unref(&tok); fts_tokenizer_unref(&gen_tok); test_end(); @@ -254,7 +235,7 @@ test_begin("fts tokenizer search email address + parent"); test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &gen_tok, &error) == 0); test_assert(fts_tokenizer_create(fts_tokenizer_email_address, gen_tok, settings, &tok, &error) == 0); - test_tokenizer_inputoutput(tok, input, expected_output); + test_tokenizer_inputoutput(tok, input, expected_output, 0); /* make sure state is forgotten at EOF */ test_assert(fts_tokenizer_next(tok, (const void *)"foo", 3, &token, &error) == 0); @@ -290,10 +271,7 @@ static void (*test_functions[])(void) = { test_fts_tokenizer_find, test_fts_tokenizer_generic_only, - test_fts_tokenizer_generic_unicode_whitespace, test_fts_tokenizer_generic_tr29_only, - test_fts_tokenizer_generic_tr29_unicode_whitespace, - test_fts_tokenizer_generic_tr29_midnumlet_end, test_fts_tokenizer_address_only, test_fts_tokenizer_address_parent, test_fts_tokenizer_address_search, From dovecot at dovecot.org Mon May 11 11:37:47 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 11:37:47 +0000 Subject: dovecot-2.2: lib-fts: Test trailing "number." for TR29 in test-f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2483039db977 changeset: 18633:2483039db977 user: Timo Sirainen date: Mon May 11 14:35:49 2015 +0300 description: lib-fts: Test trailing "number." for TR29 in test-fts-tokenizer diffstat: src/lib-fts/test-fts-tokenizer.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (30 lines): diff -r 9f06c6054e3e -r 2483039db977 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Mon May 11 14:34:50 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Mon May 11 14:35:49 2015 +0300 @@ -27,6 +27,8 @@ "and longlonglongabcdefghijklmnopqrstuvwxyz more.\n\n " "(\"Hello world\")3.14 3,14 last", + "1.", + /* whitespace: with Unicode(utf8) U+FF01(ef bc 81)(U+2000(e2 80 80) and U+205A(e2 81 9a) and U+205F(e2 81 9f) */ "hello\xEF\xBC\x81world\r\nAnd\xE2\x80\x80there\twas: text " @@ -126,6 +128,8 @@ "and", "longlonglongabcdefghijklmnopqr", "more", "Hello", "world", "3", "14", "3", "14", "last", NULL, + "1", NULL, + "hello", "world", "And", "there", "was", "text", "galore", "and", "more", NULL, @@ -163,6 +167,8 @@ "and", "longlonglongabcdefghijklmnopqr", "more", "Hello", "world", "3.14", "3,14", "last", NULL, + "1", NULL, + "hello", "world", "And", "there", "was", "text", "galore", "and", "more", NULL, From dovecot at dovecot.org Mon May 11 11:44:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 11:44:16 +0000 Subject: dovecot-2.2: lib-fts: Fixed assert-crash in fts-tokenizer-generic Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aa3374b9ce0f changeset: 18634:aa3374b9ce0f user: Timo Sirainen date: Mon May 11 14:42:18 2015 +0300 description: lib-fts: Fixed assert-crash in fts-tokenizer-generic diffstat: src/lib-fts/fts-tokenizer-generic.c | 20 +++++++++++--------- 1 files changed, 11 insertions(+), 9 deletions(-) diffs (60 lines): diff -r 2483039db977 -r aa3374b9ce0f src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon May 11 14:35:49 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon May 11 14:42:18 2015 +0300 @@ -91,6 +91,7 @@ /* if input is truncated with a partial UTF-8 character, drop it */ (void)uni_utf8_partial_strlen_n(data, size, &pos); + i_assert(pos > 0); return t_strndup(data, pos); } @@ -495,7 +496,7 @@ return FALSE; } -static void +static bool fts_tokenizer_generic_tr29_current_token(struct generic_fts_tokenizer *tok, const char **token_r) { @@ -505,12 +506,13 @@ if (is_one_past_end(tok)) end_skip = tok->last_size; - len = tok->token->used - end_skip; - i_assert(len > 0); - *token_r = fts_uni_strndup(tok->token->data, len); - buffer_set_used_size(tok->token, 0); tok->prev_prev_letter = LETTER_TYPE_NONE; tok->prev_letter = LETTER_TYPE_NONE; + + len = tok->token->used - end_skip; + *token_r = len == 0 ? "" : fts_uni_strndup(tok->token->data, len); + buffer_set_used_size(tok->token, 0); + return len > 0; } struct letter_fn { @@ -594,8 +596,8 @@ tok_append_truncated(tok, data + start_skip, char_start_i - start_skip); *skip_r = i + 1; - fts_tokenizer_generic_tr29_current_token(tok, token_r); - return 1; + if (fts_tokenizer_generic_tr29_current_token(tok, token_r)) + return 1; } } i_assert(i >= start_skip && size >= start_skip); @@ -605,8 +607,8 @@ if (size == 0 && tok->token->used > 0) { /* return the last token */ *skip_r = 0; - fts_tokenizer_generic_tr29_current_token(tok, token_r); - return 1; + if (fts_tokenizer_generic_tr29_current_token(tok, token_r)) + return 1; } return 0; } From dovecot at dovecot.org Mon May 11 13:03:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 13:03:09 +0000 Subject: dovecot-2.2: lib-fts: Added assert to fts_filter_filter() to mak... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cc59ac0f6a45 changeset: 18635:cc59ac0f6a45 user: Timo Sirainen date: Mon May 11 16:00:21 2015 +0300 description: lib-fts: Added assert to fts_filter_filter() to make sure input token isn't empty diffstat: src/lib-fts/fts-filter.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r aa3374b9ce0f -r cc59ac0f6a45 src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Mon May 11 14:42:18 2015 +0300 +++ b/src/lib-fts/fts-filter.c Mon May 11 16:00:21 2015 +0300 @@ -95,6 +95,8 @@ { int ret = 0; + i_assert((*token)[0] != '\0'); + /* Recurse to parent. */ if (filter->parent != NULL) ret = fts_filter_filter(filter->parent, token, error_r); From dovecot at dovecot.org Mon May 11 13:03:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 13:03:14 +0000 Subject: dovecot-2.2: lib-fts: normalizer-icu no longer returns empty tok... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d4541fd7a531 changeset: 18636:d4541fd7a531 user: Timo Sirainen date: Mon May 11 16:00:53 2015 +0300 description: lib-fts: normalizer-icu no longer returns empty tokens. diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r cc59ac0f6a45 -r d4541fd7a531 src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Mon May 11 16:00:21 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Mon May 11 16:00:53 2015 +0300 @@ -229,6 +229,9 @@ return -1; } + if (utext_len == 0) + return 0; + make_utf8(utext, token); return 1; } From dovecot at dovecot.org Mon May 11 13:03:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 13:03:14 +0000 Subject: dovecot-2.2: lib-fts: Fixed/improved test-fts-filter unit tests ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/44a87e17f988 changeset: 18637:44a87e17f988 user: Timo Sirainen date: Mon May 11 16:01:13 2015 +0300 description: lib-fts: Fixed/improved test-fts-filter unit tests for previous changes. diffstat: src/lib-fts/test-fts-filter.c | 49 +++++++++++++++++++++++++++--------------- 1 files changed, 31 insertions(+), 18 deletions(-) diffs (108 lines): diff -r d4541fd7a531 -r 44a87e17f988 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Mon May 11 16:00:53 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Mon May 11 16:01:13 2015 +0300 @@ -322,8 +322,6 @@ { struct fts_filter *norm = NULL; const char *input[] = { - NULL, - "", "Vem", "?", "???", @@ -331,8 +329,6 @@ "\xC3\x85\xC3\x84\xC3\x96\xC3\xB6\xC3\xA4\xC3\xA5" }; const char *expected_output[] = { - NULL, - "", "vem", "a", "aao", @@ -349,11 +345,9 @@ T_BEGIN { test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0); for (i = 0; i < N_ELEMENTS(input); i++) { - if (input[i] != NULL) { - token = input[i]; - test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i); - test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); - } + token = input[i]; + test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i); + test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); } fts_filter_unref(&norm); } T_END; @@ -365,8 +359,6 @@ { struct fts_filter *norm = NULL; const char *input[] = { - NULL, - "", "Vem", "?", "???", @@ -374,8 +366,6 @@ "\xC3\x85\xC3\x84\xC3\x96\xC3\xB6\xC3\xA4\xC3\xA5" }; const char *expected_output[] = { - NULL, - "", "vem", "a", "aao", @@ -390,11 +380,9 @@ T_BEGIN { test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, NULL, &norm, &error) == 0); for (i = 0; i < N_ELEMENTS(input); i++) { - if (input[i] != NULL) { - token = input[i]; - test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i); - test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); - } + token = input[i]; + test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i); + test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); } fts_filter_unref(&norm); } T_END; @@ -451,6 +439,30 @@ test_end(); } +static void test_fts_filter_normalizer_empty(void) +{ + /* test just a couple of these */ + static const char *empty_tokens[] = { + "\xCC\x80", /* U+0300 */ + "\xF3\xA0\x87\xAF", /* U+E01EF */ + "\xCC\x80\xF3\xA0\x87\xAF" /* U+0300 U+E01EF */ + }; + const char * const settings[] = + {"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove", NULL}; + struct fts_filter *norm; + const char *error; + unsigned int i; + + test_begin("fts filter normalizer empty tokens"); + test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0); + for (i = 0; i < N_ELEMENTS(empty_tokens); i++) { + const char *token = empty_tokens[i]; + test_assert_idx(fts_filter_filter(norm, &token, &error) == 0, i); + } + fts_filter_unref(&norm); + test_end(); +} + static void test_fts_filter_normalizer_invalid_id(void) { struct fts_filter *norm = NULL; @@ -543,6 +555,7 @@ test_fts_filter_normalizer_swedish_short, test_fts_filter_normalizer_swedish_short_default_id, test_fts_filter_normalizer_french, + test_fts_filter_normalizer_empty, test_fts_filter_normalizer_invalid_id, test_fts_filter_normalizer_stopwords_stemmer_eng, #endif From dovecot at dovecot.org Mon May 11 18:50:49 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 18:50:49 +0000 Subject: dovecot-2.2: lib-dict: Added DICT_ITERATE_FLAG_EXACT_KEY flag. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bb7d35fa9b43 changeset: 18638:bb7d35fa9b43 user: Timo Sirainen date: Mon May 11 19:12:45 2015 +0300 description: lib-dict: Added DICT_ITERATE_FLAG_EXACT_KEY flag. This is mainly useful with SQL for iterating through a result that has multiple rows. diffstat: src/lib-dict/dict-file.c | 12 +++++++++--- src/lib-dict/dict-fs.c | 1 + src/lib-dict/dict-sql.c | 8 ++++++-- src/lib-dict/dict.h | 10 +++++++++- 4 files changed, 25 insertions(+), 6 deletions(-) diffs (74 lines): diff -r 44a87e17f988 -r bb7d35fa9b43 src/lib-dict/dict-file.c --- a/src/lib-dict/dict-file.c Mon May 11 16:01:13 2015 +0300 +++ b/src/lib-dict/dict-file.c Mon May 11 19:12:45 2015 +0300 @@ -261,9 +261,15 @@ if (path == NULL) continue; - if ((ctx->flags & DICT_ITERATE_FLAG_RECURSE) == 0 && - strchr(key + path->len, '/') != NULL) - continue; + if ((ctx->flags & DICT_ITERATE_FLAG_RECURSE) != 0) { + /* match everything */ + } else if ((ctx->flags & DICT_ITERATE_FLAG_EXACT_KEY) != 0) { + if (key[path->len] != '\0') + continue; + } else { + if (strchr(key + path->len, '/') != NULL) + continue; + } *key_r = key; *value_r = value; diff -r 44a87e17f988 -r bb7d35fa9b43 src/lib-dict/dict-fs.c --- a/src/lib-dict/dict-fs.c Mon May 11 16:01:13 2015 +0300 +++ b/src/lib-dict/dict-fs.c Mon May 11 19:12:45 2015 +0300 @@ -125,6 +125,7 @@ /* these flags are not supported for now */ i_assert((flags & DICT_ITERATE_FLAG_RECURSE) == 0); + i_assert((flags & DICT_ITERATE_FLAG_EXACT_KEY) == 0); i_assert((flags & (DICT_ITERATE_FLAG_SORT_BY_KEY | DICT_ITERATE_FLAG_SORT_BY_VALUE)) == 0); diff -r 44a87e17f988 -r bb7d35fa9b43 src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Mon May 11 16:01:13 2015 +0300 +++ b/src/lib-dict/dict-sql.c Mon May 11 19:12:45 2015 +0300 @@ -377,8 +377,12 @@ str_printfa(query, " FROM %s", map->table); - recurse_type = (ctx->flags & DICT_ITERATE_FLAG_RECURSE) == 0 ? - SQL_DICT_RECURSE_ONE : SQL_DICT_RECURSE_FULL; + if ((ctx->flags & DICT_ITERATE_FLAG_RECURSE) != 0) + recurse_type = SQL_DICT_RECURSE_FULL; + else if ((ctx->flags & DICT_ITERATE_FLAG_EXACT_KEY) != 0) + recurse_type = SQL_DICT_RECURSE_NONE; + else + recurse_type = SQL_DICT_RECURSE_ONE; sql_dict_where_build(dict, map, &values, ctx->paths[ctx->path_idx][0], recurse_type, query); diff -r 44a87e17f988 -r bb7d35fa9b43 src/lib-dict/dict.h --- a/src/lib-dict/dict.h Mon May 11 16:01:13 2015 +0300 +++ b/src/lib-dict/dict.h Mon May 11 19:12:45 2015 +0300 @@ -7,10 +7,18 @@ struct dict; enum dict_iterate_flags { + /* Recurse to all the sub-hierarchies (e.g. iterating "foo/" will + return "foo/a", but should it return "foo/a/b"?) */ DICT_ITERATE_FLAG_RECURSE = 0x01, + /* Sort returned results by key */ DICT_ITERATE_FLAG_SORT_BY_KEY = 0x02, + /* Sort returned results by value */ DICT_ITERATE_FLAG_SORT_BY_VALUE = 0x04, - DICT_ITERATE_FLAG_NO_VALUE = 0x08 + /* Don't return values, only keys */ + DICT_ITERATE_FLAG_NO_VALUE = 0x08, + /* Don't recurse at all. This is basically the same as dict_lookup(), + but it'll return all the rows instead of only the first one. */ + DICT_ITERATE_FLAG_EXACT_KEY = 0x10 }; enum dict_data_type { From dovecot at dovecot.org Mon May 11 18:50:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 18:50:50 +0000 Subject: dovecot-2.2: dict-sql: If DICT_ITERATE_FLAG_EXACT_KEY is used, u... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7abdcf42b63b changeset: 18639:7abdcf42b63b user: Timo Sirainen date: Mon May 11 21:48:45 2015 +0300 description: dict-sql: If DICT_ITERATE_FLAG_EXACT_KEY is used, use only the first found map. diffstat: src/lib-dict/dict-sql.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (17 lines): diff -r bb7d35fa9b43 -r 7abdcf42b63b src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Mon May 11 19:12:45 2015 +0300 +++ b/src/lib-dict/dict-sql.c Mon May 11 21:48:45 2015 +0300 @@ -447,8 +447,11 @@ } while ((ret = sql_result_next_row(ctx->result)) == 0) { - /* see if there are more results in the next map */ - if (!sql_dict_iterate_next_query(ctx)) + /* see if there are more results in the next map. + don't do it if we're looking for an exact match, since we + already should have handled it. */ + if ((ctx->flags & DICT_ITERATE_FLAG_EXACT_KEY) != 0 || + !sql_dict_iterate_next_query(ctx)) return FALSE; } if (ret < 0) { From dovecot at dovecot.org Mon May 11 18:57:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 18:57:40 +0000 Subject: dovecot-2.2: lib-sql: Added support for Cassandra CQL as lib-sql... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3725c601dbaf changeset: 18640:3725c601dbaf user: Timo Sirainen date: Mon May 11 21:55:42 2015 +0300 description: lib-sql: Added support for Cassandra CQL as lib-sql backend. Implemented using DataStax's cpp-driver. Many things are still unimplemented. Column name specific functionality isn't even supported by the Cassandra library. So this can currently mainly be used as one of the dict backends for some simple functionality. diffstat: configure.ac | 39 +- src/lib-sql/Makefile.am | 18 +- src/lib-sql/driver-cassandra.c | 961 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1012 insertions(+), 6 deletions(-) diffs (truncated from 1110 to 300 lines): diff -r 7abdcf42b63b -r 3725c601dbaf configure.ac --- a/configure.ac Mon May 11 21:48:45 2015 +0300 +++ b/configure.ac Mon May 11 21:55:42 2015 +0300 @@ -153,6 +153,11 @@ TEST_WITH(sqlite, $withval), want_sqlite=no) +AC_ARG_WITH(cassandra, +AS_HELP_STRING([--with-cassandra], [Build with Cassandra driver support]), + TEST_WITH(cassandra, $withval), + want_cassandra=no) + AC_ARG_WITH(lucene, AS_HELP_STRING([--with-lucene], [Build with CLucene full text search support]), TEST_WITH(lucene, $withval), @@ -2319,14 +2324,32 @@ fi ]) fi + +if test $want_cassandra != no; then + AC_CHECK_LIB(cassandra, cass_session_new, [ + AC_CHECK_HEADER(cassandra.h, [ + CASSANDRA_LIBS="$CASSANDRA_LIBS -lcassandra" + + AC_DEFINE(HAVE_CASSANDRA,, Build with Cassandra support) + found_sql_drivers="$found_sql_drivers cassandra" + ], [ + if test $want_cassandra = yes; then + AC_ERROR([Can't build with Cassandra support: cassandra.h not found]) + fi + ]) + ], [ + if test $want_cassandra = yes; then + AC_ERROR([Can't build with Cassandra support: libcassandra not found]) + fi + ]) +fi -SQL_CFLAGS="$MYSQL_CFLAGS $PGSQL_CFLAGS $SQLITE_CFLAGS" +SQL_CFLAGS="$MYSQL_CFLAGS $PGSQL_CFLAGS $SQLITE_CFLAGS $CASSANDRA_CFLAGS" if test "$want_sql" != "plugin"; then - SQL_LIBS="$MYSQL_LIBS $PGSQL_LIBS $SQLITE_LIBS" + SQL_LIBS="$MYSQL_LIBS $PGSQL_LIBS $SQLITE_LIBS $CASSANDRA_LIBS" else AC_DEFINE(SQL_DRIVER_PLUGINS,, Build SQL drivers as plugins) fi - sql_drivers= not_sql_drivers= @@ -2444,6 +2467,8 @@ AC_SUBST(PGSQL_LIBS) AC_SUBST(SQLITE_CFLAGS) AC_SUBST(SQLITE_LIBS) +AC_SUBST(CASSANDRA_CFLAGS) +AC_SUBST(CASSANDRA_LIBS) AC_SUBST(DICT_LIBS) AC_SUBST(CDB_LIBS) @@ -2587,6 +2612,7 @@ build_pgsql=no build_mysql=no build_sqlite=no +build_cassandra=no for driver in $sql_drivers; do if test "$driver" = "pgsql"; then AC_DEFINE(BUILD_PGSQL,, Built-in PostgreSQL support) @@ -2597,6 +2623,9 @@ elif test "$driver" = "sqlite"; then AC_DEFINE(BUILD_SQLITE,, Built-in SQLite support) build_sqlite=yes + elif test "$driver" = "cassandra"; then + AC_DEFINE(BUILD_CASSANDRA,, Built-in Cassandra support) + build_cassandra=yes fi done if test $build_pgsql = no; then @@ -2608,11 +2637,15 @@ if test $build_sqlite = no; then not_sql_drivers="$not_sql_drivers sqlite" fi +if test $build_cassandra = no; then + not_sql_drivers="$not_sql_drivers cassandra" +fi AC_SUBST(sql_drivers) AM_CONDITIONAL(BUILD_PGSQL, test "$build_pgsql" = "yes") AM_CONDITIONAL(BUILD_MYSQL, test "$build_mysql" = "yes") AM_CONDITIONAL(BUILD_SQLITE, test "$build_sqlite" = "yes") +AM_CONDITIONAL(BUILD_CASSANDRA, test "$build_cassandra" = "yes") AM_CONDITIONAL(SQL_PLUGINS, test "$want_sql" = "plugin") dnl ** diff -r 7abdcf42b63b -r 3725c601dbaf src/lib-sql/Makefile.am --- a/src/lib-sql/Makefile.am Mon May 11 21:48:45 2015 +0300 +++ b/src/lib-sql/Makefile.am Mon May 11 21:55:42 2015 +0300 @@ -18,11 +18,16 @@ SQLITE_LIB = libdriver_sqlite.la SQL_DRIVER_PLUGINS += sqlite endif +if BUILD_CASSANDRA +SQLITE_LIB = libdriver_cassandra.la +SQL_DRIVER_PLUGINS += cassandra +endif sql_module_LTLIBRARIES = \ $(MYSQL_LIB) \ $(PGSQL_LIB) \ - $(SQLITE_LIB) + $(SQLITE_LIB) \ + $(CASSANDRA_LIB) sql_moduledir = $(moduledir) endif @@ -41,7 +46,8 @@ driver_sources = \ driver-mysql.c \ driver-pgsql.c \ - driver-sqlite.c + driver-sqlite.c \ + driver-cassandra.c endif libsql_la_SOURCES = \ @@ -69,12 +75,18 @@ libdriver_sqlite_la_CPPFLAGS = -I$(top_srcdir)/src/lib $(SQLITE_CFLAGS) libdriver_sqlite_la_SOURCES = driver-sqlite.c +libdriver_cassandra_la_LDFLAGS = -module -avoid-version +libdriver_cassandra_la_LIBADD = $(CASSANDRA_LIBS) +libdriver_cassandra_la_CPPFLAGS = -I$(top_srcdir)/src/lib $(CASSANDRA_CFLAGS) +libdriver_cassandra_la_SOURCES = driver-cassandra.c + sql_libs = else sql_libs = \ $(MYSQL_LIBS) \ $(PGSQL_LIBS) \ - $(SQLITE_LIBS) + $(SQLITE_LIBS) \ + $(CASSANDRA_LIBS) endif pkglib_LTLIBRARIES = libdovecot-sql.la diff -r 7abdcf42b63b -r 3725c601dbaf src/lib-sql/driver-cassandra.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sql/driver-cassandra.c Mon May 11 21:55:42 2015 +0300 @@ -0,0 +1,961 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "ioloop.h" +#include "write-full.h" +#include "sql-api-private.h" + +#ifdef BUILD_CASSANDRA +#include +#include + +#define IS_CONNECTED(db) \ + ((db)->api.state != SQL_DB_STATE_DISCONNECTED && \ + (db)->api.state != SQL_DB_STATE_CONNECTING) + +typedef void driver_cassandra_callback_t(CassFuture *future, void *context); + +struct cassandra_callback { + unsigned int id; + CassFuture *future; + struct cassandra_db *db; + driver_cassandra_callback_t *callback; + void *context; +}; + +struct cassandra_db { + struct sql_db api; + + char *hosts, *keyspace; + CassConsistency consistency; + + CassCluster *cluster; + CassSession *session; + + int fd_pipe[2]; + struct io *io_pipe; + ARRAY(struct cassandra_callback *) callbacks; + unsigned int callback_ids; + + struct cassandra_result *cur_result; + struct ioloop *ioloop, *orig_ioloop; + struct sql_result *sync_result; + + char *error; +}; + +struct cassandra_result { + struct sql_result api; + CassStatement *statement; + const CassResult *result; + CassIterator *iterator; + char *query; + char *error; + + pool_t row_pool; + ARRAY_TYPE(const_string) fields; + + sql_query_callback_t *callback; + void *context; + + unsigned int finished:1; +}; + +struct cassandra_transaction_context { + struct sql_transaction_context ctx; + int refcount; + + sql_commit_callback_t *callback; + void *context; + + pool_t query_pool; + const char *error; + + unsigned int begin_succeeded:1; + unsigned int begin_failed:1; + unsigned int failed:1; +}; + +extern const struct sql_db driver_cassandra_db; +extern const struct sql_result driver_cassandra_result; + +static struct { + CassConsistency consistency; + const char *name; +} cass_consistency_names[] = { + { CASS_CONSISTENCY_ANY, "any" }, + { CASS_CONSISTENCY_ONE, "one" }, + { CASS_CONSISTENCY_TWO, "two" }, + { CASS_CONSISTENCY_THREE, "three" }, + { CASS_CONSISTENCY_QUORUM, "" }, + { CASS_CONSISTENCY_ALL, "all" }, + { CASS_CONSISTENCY_QUORUM, "" }, + { CASS_CONSISTENCY_ALL, "all" }, + { CASS_CONSISTENCY_LOCAL_QUORUM, "local-quorum" }, + { CASS_CONSISTENCY_EACH_QUORUM, "each-quorum" }, + { CASS_CONSISTENCY_SERIAL, "serial" }, + { CASS_CONSISTENCY_LOCAL_SERIAL, "local-serial" }, + { CASS_CONSISTENCY_LOCAL_ONE, "local-one" } +}; + +static void result_finish(struct cassandra_result *result); + +static int consistency_parse(const char *str, CassConsistency *consistency_r) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(cass_consistency_names); i++) { + if (strcmp(cass_consistency_names[i].name, str) == 0) { + *consistency_r = cass_consistency_names[i].consistency; + return 0; + } + } + return -1; +} + +static void driver_cassandra_set_state(struct cassandra_db *db, enum sql_db_state state) +{ + i_assert(state == SQL_DB_STATE_BUSY || db->cur_result == NULL); + + /* switch back to original ioloop in case the caller wants to + add/remove timeouts */ + if (db->ioloop != NULL) + io_loop_set_current(db->orig_ioloop); + sql_db_set_state(&db->api, state); + if (db->ioloop != NULL) + io_loop_set_current(db->ioloop); +} + +static void driver_cassandra_close(struct cassandra_db *db) +{ + if (db->io_pipe != NULL) + io_remove(&db->io_pipe); + if (db->fd_pipe[0] != -1) { + i_close_fd(&db->fd_pipe[0]); + i_close_fd(&db->fd_pipe[1]); + } + driver_cassandra_set_state(db, SQL_DB_STATE_DISCONNECTED); + + if (db->ioloop != NULL) { + /* running a sync query, stop it */ + io_loop_stop(db->ioloop); + } +} + +static void driver_cassandra_log_error(CassFuture *future, const char *str) +{ + const char *message; + size_t size; + From pigeonhole at rename-it.nl Mon May 11 19:02:22 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 May 2015 21:02:22 +0200 Subject: dovecot-2.2-pigeonhole: Added tag 0.4.8.rc2 for changeset 44d412... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/5a780315cbfe changeset: 2056:5a780315cbfe user: Stephan Bosch date: Mon May 11 21:02:10 2015 +0200 description: Added tag 0.4.8.rc2 for changeset 44d41235776d diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 44d41235776d -r 5a780315cbfe .hgtags --- a/.hgtags Mon May 11 21:02:00 2015 +0200 +++ b/.hgtags Mon May 11 21:02:10 2015 +0200 @@ -28,3 +28,4 @@ 87b2d3e43a447f000b06ce1ee1ab73456c7357b3 0.4.7.rc3 f2323a90c202c4e9708a0c87e81a1b06d267a5c5 0.4.7 fcc97e953584269e5fa140363804f4016c33ef1c 0.4.8.rc1 +44d41235776de692f1f4ef9843ee6f3901fa3138 0.4.8.rc2 From pigeonhole at rename-it.nl Mon May 11 19:02:22 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 May 2015 21:02:22 +0200 Subject: dovecot-2.2-pigeonhole: Released v0.4.8.rc2 for Dovecot v2.2.17.... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/44d41235776d changeset: 2055:44d41235776d user: Stephan Bosch date: Mon May 11 21:02:00 2015 +0200 description: Released v0.4.8.rc2 for Dovecot v2.2.17.rc1. diffstat: configure.ac | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (9 lines): diff -r c00f51f3c91a -r 44d41235776d configure.ac --- a/configure.ac Mon May 11 10:03:02 2015 +0200 +++ b/configure.ac Mon May 11 21:02:00 2015 +0200 @@ -1,4 +1,4 @@ -AC_INIT([Pigeonhole], [0.4.8.rc1], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) +AC_INIT([Pigeonhole], [0.4.8.rc2], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_SRCDIR([src]) AC_CONFIG_MACRO_DIR([m4]) From pigeonhole at rename-it.nl Mon May 11 19:03:33 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 May 2015 21:03:33 +0200 Subject: dovecot-2.2-pigeonhole: Added signature for changeset 44d41235776d Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/00b74ac5af5b changeset: 2057:00b74ac5af5b user: Stephan Bosch date: Mon May 11 21:04:39 2015 +0200 description: Added signature for changeset 44d41235776d diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 5a780315cbfe -r 00b74ac5af5b .hgsigs --- a/.hgsigs Mon May 11 21:02:10 2015 +0200 +++ b/.hgsigs Mon May 11 21:04:39 2015 +0200 @@ -22,3 +22,4 @@ 87b2d3e43a447f000b06ce1ee1ab73456c7357b3 0 iQEcBAABAgAGBQJVB4QIAAoJEATWKx49+7T0MKcH/0Wg6xazPO9Q3v8rYZe4PFSlWoXv4P8ixy275pouHyLWKDltg2zVPq3ieDNLkOfNiQvBClWppQa5LPzgTN/9x5MmIYU/IpdXCKvl8i9G3qzRaGUL1oMtp3JocI8JyKx3R61V8u4VyJ+qBg0kLCs85ilNdo1ia9uFCYbZh7QOnM5FHJAsGzDPEriZyrcf3D/CTGDedYfoELEJXpSzW19Ov1qPIq501mBfrxXSd/MTUFkm8DExYW7i/YTBEsJNn58y1tsdC6FxuP8bZJRQlS/4PRR5euACxKpEV38Ig7b22a6P/MoZphCvZkIMBIxIB9RIEIljRyXSKESvssgUoA/v1Q8= f2323a90c202c4e9708a0c87e81a1b06d267a5c5 0 iQEcBAABAgAGBQJVC01gAAoJEATWKx49+7T0eNQH/iHUSapqKCaPoZByc1UHpp/FS7IEzbXUWTa71uTHN7z+/Gr6vKm4/xl0k10N6XhpY5aJImLt31dd8jSDg6hqhVwDlqIImJzCjf+flG4dZu5MIL2vibjEn7InYvMli5EcafQ08EkTWGm7IPOcqfnZheevU64piqaUr1YfNaR0g/cQcffvT9qXNovlqFQe+hDB2/wUj6zKiHx0rqUzAOh4dAcrfq1jCpXhpGChjezxyiGzFpLPopbEHQU9M9Fr6VFgY+v/mDVp6hj8DpE7Q84BxPvxJKi0il+h0/lliDZUA+XtZsVjckih5WBDC5tLgNRKujMzkAc2UNEU/1vJxkPkldI= fcc97e953584269e5fa140363804f4016c33ef1c 0 iQEcBAABAgAGBQJVT9pVAAoJEATWKx49+7T0/14H/3iejYaCOqm5ftfQDMlebUhbMa9LGEMwOPRoBxnzGOBoJGEQ7MS/aqk8ARM1i016jFtDa0oVWkTDyWZhDwQHS6yA2HkXPxizKHf9+9Kc2Lxjew4GIyTGSNmkaJb34XUKGhKx4YMfDXQiFsc0G/vVSO8UoZzvqho6g7bUmuWeuCrXmEW+M+bUG4KMCF0abYba7iasYKaV+1fSnbAnh5kRQqwfUfs/ugdU4f9VZxmnGdJymf540KdoqEzePMaxhEp6OPhhr/eVGabfWhV+RJiRI7atWQl7I6uoEkvziObuUKx3rl0xVoSVcKeCC4APsJdQQT/HDoPmWfyaQCV+WORmz6U= +44d41235776de692f1f4ef9843ee6f3901fa3138 0 iQEcBAABAgAGBQJVUP1CAAoJEATWKx49+7T0+GMH/3QKZewiy+jeeHmb8wYExW6eiAEprb7lq2U8UjN5iOuID7OnDbkZ0/jr/oU0zsxpch/lacpAa0cIvgvujhGuXzeR2XNO3MWyiOPe4t0WcFaoP4Hl+EV7faZipCdPx7mGbxFYSEx9rz4Kp0MBnG/I+yE2+I1AbBsHqQMC1meAA3llu1n/Wf9BOhyzUP4/AL9CSNUtO/6tUZriXQOrRZpwkcyc7mrH6Trpyy0YXkEVWAZ8jdusiVuLmwMzEHUaF4SXc2daHhRRRiwp3wCeIkyVvqm8BbxFGZ7ZMM7fKnhj3LGmSLfeloHHmCC3HU6SfzvNmv8Fcieah/thpl/EOdKiWeI= From dovecot at dovecot.org Mon May 11 19:39:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 19:39:17 +0000 Subject: dovecot-2.2: lib-fts: Replaced word-boundary/break-data.sh with ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e80969ea8684 changeset: 18641:e80969ea8684 user: Timo Sirainen date: Mon May 11 22:37:21 2015 +0300 description: lib-fts: Replaced word-boundary/break-data.sh with more portable awk scripts Patch by Michael Grimm. diffstat: src/lib-fts/Makefile.am | 12 ++-- src/lib-fts/word-boundary-data.awk | 103 +++++++++++++++++++++++++++++++++++++ src/lib-fts/word-boundary-data.sh | 99 ----------------------------------- src/lib-fts/word-break-data.awk | 102 ++++++++++++++++++++++++++++++++++++ src/lib-fts/word-break-data.sh | 77 --------------------------- 5 files changed, 211 insertions(+), 182 deletions(-) diffs (truncated from 427 to 300 lines): diff -r 3725c601dbaf -r e80969ea8684 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Mon May 11 21:55:42 2015 +0300 +++ b/src/lib-fts/Makefile.am Mon May 11 22:37:21 2015 +0300 @@ -22,20 +22,20 @@ udhr_fra.txt \ PropList.txt \ WordBreakProperty.txt \ - word-boundary-data.sh \ + word-boundary-data.awk \ word-boundary-data.c \ - word-break-data.sh \ + word-break-data.awk \ word-break-data.c WordBreakProperty.txt: test -f WordBreakProperty.txt || wget http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakProperty.txt -$(srcdir)/word-boundary-data.c: word-boundary-data.sh WordBreakProperty.txt - $(srcdir)/word-boundary-data.sh < WordBreakProperty.txt > $@ +$(srcdir)/word-boundary-data.c: word-boundary-data.awk WordBreakProperty.txt + $(AWK) -f $(srcdir)/word-boundary-data.awk < WordBreakProperty.txt > $@ PropList.txt: test -f PropList.txt || wget http://www.unicode.org/Public/UNIDATA/PropList.txt -$(srcdir)/word-break-data.c: word-break-data.sh PropList.txt - $(srcdir)/word-break-data.sh < PropList.txt > $@ +$(srcdir)/word-break-data.c: word-break-data.awk PropList.txt + $(AWK) -f $(srcdir)/word-break-data.awk < PropList.txt > $@ if BUILD_FTS_STEMMER diff -r 3725c601dbaf -r e80969ea8684 src/lib-fts/word-boundary-data.awk --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/word-boundary-data.awk Mon May 11 22:37:21 2015 +0300 @@ -0,0 +1,103 @@ +#!/usr/bin/awk -f + +# +# converts strings to hex numbers (gawk's strtonum function) +# adopted from http://www.gnu.org/software/gawk/manual/html_node/Strtonum-Function.html#Strtonum-Function +# +function mystrtonum(str) { + # adopted from http://www.gnu.org/software/gawk/manual/html_node/Strtonum-Function.html#Strtonum-Function + if (str ~ /^0[xX][[:xdigit:]]+$/) { + str = substr(str, 3) # lop off leading 0x + n = length(str) + ret = 0 + for (i = 1; i <= n; i++) { + c = substr(str, i, 1) + c = tolower(c) + # index() returns 0 if c not in string, + # includes c == "0" + k = index("123456789abcdef", c) + ret = ret * 16 + k + } + } else { + ret = "NOT-A-HEX-NUMBER" + } + return ret +} + +# +# expand number ranges (from..to) to sequences of numbers (emulate seq function) +# +function add_hexrange (start, end) { + from = mystrtonum("0x"start) + to = mystrtonum("0x"end) + for ( i=from; i<=to; i++ ) + temp[i] = i + result = temp[from] + for ( i=from+1; i<=to; i++ ) + result = result " " temp[i] + return result +} + +# +# initialization stuff (define categories of intrest in input file) +# +BEGIN { + FS = " " + ncategories = split("CR LF Newline Extend Regional_Indicator Format Katakana Hebrew_Letter ALetter \ + Single_Quote Double_Quote MidNumLet MidLetter MidNum Numeric ExtendNumLet", array_names) +} + +# +# evaluate every line in input read from +# +{ + # skip comments and empty lines + if ( $0 !~ /^#/ && NF != 0 ) { + # cycle over array_names and do the math + for (category in array_names) { + # identify categories of interest (attention: relies on leading '; ' and trailing ' #' anchors, + # might be suited regex preferable!) + if ( $0 ~ "; "array_names[category]" #" ) { + # distinguish beetween single numbers and number ranges (from..to) + if ( $1 ~ /\.\./ ) { + split($1, bounderies, "\.") + array[category] = array[category] " " add_hexrange(bounderies[1], bounderies[3]) + } else { + array[category] = array[category] " " mystrtonum("0x"$1) + } + } + } + } +} + +# +# format output to +# +END { + print "/* This file is automatically generated by word-boundary-data.awk from WordBreakProperty.txt */" + for (category=1; category<=ncategories; category++) { + n = split(array[category], integers) + print "static const uint32_t "array_names[category]"[]= {" + if (n == 1) { + # split puts '0' into integers if arraysize equals to 1, thus: + printf("\t0x%05X", array[category]) + } else { + for ( i=1; i<=n; i++) { + if ( i == 1 ) { + printf("\t0x%05X, ", integers[i]) + } else if ( (i-1)%8 == 0 ) { + if ( i != n ) { + printf("\n\t0x%05X, ", integers[i]) + } else { + printf("\n\t0x%05X", integers[i]) + } + } else if ( i != n ) { + printf("0x%05X, ", integers[i]) + } else { + printf("0x%05X", integers[i]) + } + } + } + print "\n};" + } +} diff -r 3725c601dbaf -r e80969ea8684 src/lib-fts/word-boundary-data.sh --- a/src/lib-fts/word-boundary-data.sh Mon May 11 21:55:42 2015 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,99 +0,0 @@ -#!/bin/bash -# TODO: Should perhaps be written in perl/python/awk -# FIXME: The runtime is a bit long. - -#Array names match category names in data file. -declare -a CR -declare -a LF -declare -a Newline -declare -a Extend -declare -a Regional_Indicator -declare -a Format -declare -a Katakana -declare -a Hebrew_Letter -declare -a ALetter -declare -a Single_Quote -declare -a Double_Quote -declare -a MidNumLet -declare -a MidLetter -declare -a MidNum -declare -a Numeric -declare -a ExtendNumLet - -WIDTH=5 - -add_hexrange () { - - array_name="$1" - from="$2" - to="$3" - - eval "$array_name+=($(seq -s ' ' "0x$from" "0x$to"))" -} - -print_c_array () { - - array_name="$1" - eval "array=("\${$array_name[@]}")" - array_length=${#array[@]} - i=1 - - printf "static const uint32_t %s[]= {\n\t" "$array_name" - - for val in "${array[@]}" ; do - printf "0x%0${WIDTH}X" "$val" - if [ $i -lt $array_length ]; then - echo -n ", " - if [ $(($i%8)) -eq 0 ]; then - echo -ne "\n\t" - fi - i=$((i+1)) - else - break - fi - done - - echo -ne "\n};\n" -} -#read everything except comments. -while read -s -a line; do - [ -z "${line[0]}" ] && continue #ignore empty lines - - case "${line[0]}" in \#*) continue ;; esac #ignore comments - - value="${line[0]}" - category="${line[2]}" - - case "$value" in - *..*) - start=`echo "$value" | cut -d . -f 1` - end=`echo "$value" | cut -d . -f 3` - add_hexrange "$category" "$start" "$end" - ;; - *) - value=`printf "%05X" $((16#$value))` - eval "$category+=(0x\$value)" - ;; - esac; - -done - -printf "/* This file is automatically generated by %s from WordBreakProperty.txt */\n" "$0" - -print_c_array CR -print_c_array LF -print_c_array Newline -print_c_array Extend -print_c_array Regional_Indicator -print_c_array Format -print_c_array Katakana -print_c_array Hebrew_Letter -print_c_array ALetter -print_c_array Single_Quote -print_c_array Double_Quote -print_c_array MidNumLet -print_c_array MidLetter -print_c_array MidNum -print_c_array Numeric -print_c_array ExtendNumLet - diff -r 3725c601dbaf -r e80969ea8684 src/lib-fts/word-break-data.awk --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/word-break-data.awk Mon May 11 22:37:21 2015 +0300 @@ -0,0 +1,102 @@ +#!/usr/bin/awk -f + +# +# converts strings to hex numbers (gawk's strtonum function) +# adopted from http://www.gnu.org/software/gawk/manual/html_node/Strtonum-Function.html#Strtonum-Function +# +function mystrtonum(str) { + # adopted from http://www.gnu.org/software/gawk/manual/html_node/Strtonum-Function.html#Strtonum-Function + if (str ~ /^0[xX][[:xdigit:]]+$/) { + str = substr(str, 3) # lop off leading 0x + n = length(str) + ret = 0 + for (i = 1; i <= n; i++) { + c = substr(str, i, 1) + c = tolower(c) + # index() returns 0 if c not in string, + # includes c == "0" + k = index("123456789abcdef", c) + ret = ret * 16 + k + } + } else { + ret = "NOT-A-HEX-NUMBER" + } + return ret +} + +# +# expand number ranges (from..to) to sequences of numbers (emulate seq function) +# +function add_hexrange (start, end) { + from = mystrtonum("0x"start) + to = mystrtonum("0x"end) + for ( i=from; i<=to; i++ ) + temp[i] = i + result = temp[from] + for ( i=from+1; i<=to; i++ ) + result = result " " temp[i] + return result +} + +# +# initialization stuff (define categories of intrest in input file) +# +BEGIN { + FS = " " + ncategories = split("White_Space Dash Terminal_Punctuation STerm Pattern_White_Space", array_names) +} + +# +# evaluate every line in input read from +# +{ + # skip comments and empty lines + if ( $0 !~ /^#/ && NF != 0 ) { + # cycle over array_names and do the math + for (category in array_names) { From dovecot at dovecot.org Mon May 11 19:40:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 11 May 2015 19:40:35 +0000 Subject: dovecot-2.2: Makefile: Fixed build concurrency issues with lib-fts Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7d52d6595f5e changeset: 18642:7d52d6595f5e user: Timo Sirainen date: Mon May 11 22:38:38 2015 +0300 description: Makefile: Fixed build concurrency issues with lib-fts diffstat: src/lib-fts/Makefile.am | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (18 lines): diff -r e80969ea8684 -r 7d52d6595f5e src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Mon May 11 22:37:21 2015 +0300 +++ b/src/lib-fts/Makefile.am Mon May 11 22:38:38 2015 +0300 @@ -30,12 +30,12 @@ WordBreakProperty.txt: test -f WordBreakProperty.txt || wget http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakProperty.txt $(srcdir)/word-boundary-data.c: word-boundary-data.awk WordBreakProperty.txt - $(AWK) -f $(srcdir)/word-boundary-data.awk < WordBreakProperty.txt > $@ + $(AWK) -f $(srcdir)/word-boundary-data.awk < WordBreakProperty.txt > $@.tmp && mv $@.tmp $@ PropList.txt: test -f PropList.txt || wget http://www.unicode.org/Public/UNIDATA/PropList.txt $(srcdir)/word-break-data.c: word-break-data.awk PropList.txt - $(AWK) -f $(srcdir)/word-break-data.awk < PropList.txt > $@ + $(AWK) -f $(srcdir)/word-break-data.awk < PropList.txt > $@.tmp && mv $@.tmp $@ if BUILD_FTS_STEMMER From pigeonhole at rename-it.nl Mon May 11 21:05:34 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 May 2015 23:05:34 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: storage: Fixed handling of de... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/59d59552247c changeset: 2058:59d59552247c user: Stephan Bosch date: Mon May 11 23:05:24 2015 +0200 description: lib-sieve: storage: Fixed handling of defaul script in sieve_storage_active_script_open(). It only worked when sieve_default_name was set. diffstat: src/lib-sieve/sieve-storage.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (13 lines): diff -r 00b74ac5af5b -r 59d59552247c src/lib-sieve/sieve-storage.c --- a/src/lib-sieve/sieve-storage.c Mon May 11 21:04:39 2015 +0200 +++ b/src/lib-sieve/sieve-storage.c Mon May 11 23:05:24 2015 +0200 @@ -870,8 +870,7 @@ if ( script != NULL || (storage->flags & SIEVE_STORAGE_FLAG_SYNCHRONIZING) != 0 || - storage->default_location == NULL || - storage->default_name == NULL ) { + storage->default_location == NULL) { if ( error_r != NULL ) *error_r = storage->error_code; return script; From dovecot at dovecot.org Tue May 12 09:22:58 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 09:22:58 +0000 Subject: dovecot-2.2: lib-fts: Reverted e80969ea8684 which replaced .sh s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2cfb80f7785e changeset: 18643:2cfb80f7785e user: Timo Sirainen date: Tue May 12 12:20:56 2015 +0300 description: lib-fts: Reverted e80969ea8684 which replaced .sh scripts with awk Bugs in older awk versions (used at least by Debian squeeze & wheezy) caused awk to crash while processing the script. diffstat: src/lib-fts/Makefile.am | 12 ++-- src/lib-fts/word-boundary-data.awk | 103 ------------------------------------- src/lib-fts/word-boundary-data.sh | 99 +++++++++++++++++++++++++++++++++++ src/lib-fts/word-break-data.awk | 102 ------------------------------------ src/lib-fts/word-break-data.sh | 77 +++++++++++++++++++++++++++ 5 files changed, 182 insertions(+), 211 deletions(-) diffs (truncated from 427 to 300 lines): diff -r 7d52d6595f5e -r 2cfb80f7785e src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Mon May 11 22:38:38 2015 +0300 +++ b/src/lib-fts/Makefile.am Tue May 12 12:20:56 2015 +0300 @@ -22,20 +22,20 @@ udhr_fra.txt \ PropList.txt \ WordBreakProperty.txt \ - word-boundary-data.awk \ + word-boundary-data.sh \ word-boundary-data.c \ - word-break-data.awk \ + word-break-data.sh \ word-break-data.c WordBreakProperty.txt: test -f WordBreakProperty.txt || wget http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakProperty.txt -$(srcdir)/word-boundary-data.c: word-boundary-data.awk WordBreakProperty.txt - $(AWK) -f $(srcdir)/word-boundary-data.awk < WordBreakProperty.txt > $@.tmp && mv $@.tmp $@ +$(srcdir)/word-boundary-data.c: word-boundary-data.sh WordBreakProperty.txt + $(srcdir)/word-boundary-data.sh < WordBreakProperty.txt > $@.tmp && mv $@.tmp $@ PropList.txt: test -f PropList.txt || wget http://www.unicode.org/Public/UNIDATA/PropList.txt -$(srcdir)/word-break-data.c: word-break-data.awk PropList.txt - $(AWK) -f $(srcdir)/word-break-data.awk < PropList.txt > $@.tmp && mv $@.tmp $@ +$(srcdir)/word-break-data.c: word-break-data.sh PropList.txt + $(srcdir)/word-break-data.sh < PropList.txt > $@.tmp && mv $@.tmp $@ if BUILD_FTS_STEMMER diff -r 7d52d6595f5e -r 2cfb80f7785e src/lib-fts/word-boundary-data.awk --- a/src/lib-fts/word-boundary-data.awk Mon May 11 22:38:38 2015 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,103 +0,0 @@ -#!/usr/bin/awk -f - -# -# converts strings to hex numbers (gawk's strtonum function) -# adopted from http://www.gnu.org/software/gawk/manual/html_node/Strtonum-Function.html#Strtonum-Function -# -function mystrtonum(str) { - # adopted from http://www.gnu.org/software/gawk/manual/html_node/Strtonum-Function.html#Strtonum-Function - if (str ~ /^0[xX][[:xdigit:]]+$/) { - str = substr(str, 3) # lop off leading 0x - n = length(str) - ret = 0 - for (i = 1; i <= n; i++) { - c = substr(str, i, 1) - c = tolower(c) - # index() returns 0 if c not in string, - # includes c == "0" - k = index("123456789abcdef", c) - ret = ret * 16 + k - } - } else { - ret = "NOT-A-HEX-NUMBER" - } - return ret -} - -# -# expand number ranges (from..to) to sequences of numbers (emulate seq function) -# -function add_hexrange (start, end) { - from = mystrtonum("0x"start) - to = mystrtonum("0x"end) - for ( i=from; i<=to; i++ ) - temp[i] = i - result = temp[from] - for ( i=from+1; i<=to; i++ ) - result = result " " temp[i] - return result -} - -# -# initialization stuff (define categories of intrest in input file) -# -BEGIN { - FS = " " - ncategories = split("CR LF Newline Extend Regional_Indicator Format Katakana Hebrew_Letter ALetter \ - Single_Quote Double_Quote MidNumLet MidLetter MidNum Numeric ExtendNumLet", array_names) -} - -# -# evaluate every line in input read from -# -{ - # skip comments and empty lines - if ( $0 !~ /^#/ && NF != 0 ) { - # cycle over array_names and do the math - for (category in array_names) { - # identify categories of interest (attention: relies on leading '; ' and trailing ' #' anchors, - # might be suited regex preferable!) - if ( $0 ~ "; "array_names[category]" #" ) { - # distinguish beetween single numbers and number ranges (from..to) - if ( $1 ~ /\.\./ ) { - split($1, bounderies, "\.") - array[category] = array[category] " " add_hexrange(bounderies[1], bounderies[3]) - } else { - array[category] = array[category] " " mystrtonum("0x"$1) - } - } - } - } -} - -# -# format output to -# -END { - print "/* This file is automatically generated by word-boundary-data.awk from WordBreakProperty.txt */" - for (category=1; category<=ncategories; category++) { - n = split(array[category], integers) - print "static const uint32_t "array_names[category]"[]= {" - if (n == 1) { - # split puts '0' into integers if arraysize equals to 1, thus: - printf("\t0x%05X", array[category]) - } else { - for ( i=1; i<=n; i++) { - if ( i == 1 ) { - printf("\t0x%05X, ", integers[i]) - } else if ( (i-1)%8 == 0 ) { - if ( i != n ) { - printf("\n\t0x%05X, ", integers[i]) - } else { - printf("\n\t0x%05X", integers[i]) - } - } else if ( i != n ) { - printf("0x%05X, ", integers[i]) - } else { - printf("0x%05X", integers[i]) - } - } - } - print "\n};" - } -} diff -r 7d52d6595f5e -r 2cfb80f7785e src/lib-fts/word-boundary-data.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/word-boundary-data.sh Tue May 12 12:20:56 2015 +0300 @@ -0,0 +1,99 @@ +#!/bin/bash +# TODO: Should perhaps be written in perl/python/awk +# FIXME: The runtime is a bit long. + +#Array names match category names in data file. +declare -a CR +declare -a LF +declare -a Newline +declare -a Extend +declare -a Regional_Indicator +declare -a Format +declare -a Katakana +declare -a Hebrew_Letter +declare -a ALetter +declare -a Single_Quote +declare -a Double_Quote +declare -a MidNumLet +declare -a MidLetter +declare -a MidNum +declare -a Numeric +declare -a ExtendNumLet + +WIDTH=5 + +add_hexrange () { + + array_name="$1" + from="$2" + to="$3" + + eval "$array_name+=($(seq -s ' ' "0x$from" "0x$to"))" +} + +print_c_array () { + + array_name="$1" + eval "array=("\${$array_name[@]}")" + array_length=${#array[@]} + i=1 + + printf "static const uint32_t %s[]= {\n\t" "$array_name" + + for val in "${array[@]}" ; do + printf "0x%0${WIDTH}X" "$val" + if [ $i -lt $array_length ]; then + echo -n ", " + if [ $(($i%8)) -eq 0 ]; then + echo -ne "\n\t" + fi + i=$((i+1)) + else + break + fi + done + + echo -ne "\n};\n" +} +#read everything except comments. +while read -s -a line; do + [ -z "${line[0]}" ] && continue #ignore empty lines + + case "${line[0]}" in \#*) continue ;; esac #ignore comments + + value="${line[0]}" + category="${line[2]}" + + case "$value" in + *..*) + start=`echo "$value" | cut -d . -f 1` + end=`echo "$value" | cut -d . -f 3` + add_hexrange "$category" "$start" "$end" + ;; + *) + value=`printf "%05X" $((16#$value))` + eval "$category+=(0x\$value)" + ;; + esac; + +done + +printf "/* This file is automatically generated by %s from WordBreakProperty.txt */\n" "$0" + +print_c_array CR +print_c_array LF +print_c_array Newline +print_c_array Extend +print_c_array Regional_Indicator +print_c_array Format +print_c_array Katakana +print_c_array Hebrew_Letter +print_c_array ALetter +print_c_array Single_Quote +print_c_array Double_Quote +print_c_array MidNumLet +print_c_array MidLetter +print_c_array MidNum +print_c_array Numeric +print_c_array ExtendNumLet + diff -r 7d52d6595f5e -r 2cfb80f7785e src/lib-fts/word-break-data.awk --- a/src/lib-fts/word-break-data.awk Mon May 11 22:38:38 2015 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,102 +0,0 @@ -#!/usr/bin/awk -f - -# -# converts strings to hex numbers (gawk's strtonum function) -# adopted from http://www.gnu.org/software/gawk/manual/html_node/Strtonum-Function.html#Strtonum-Function -# -function mystrtonum(str) { - # adopted from http://www.gnu.org/software/gawk/manual/html_node/Strtonum-Function.html#Strtonum-Function - if (str ~ /^0[xX][[:xdigit:]]+$/) { - str = substr(str, 3) # lop off leading 0x - n = length(str) - ret = 0 - for (i = 1; i <= n; i++) { - c = substr(str, i, 1) - c = tolower(c) - # index() returns 0 if c not in string, - # includes c == "0" - k = index("123456789abcdef", c) - ret = ret * 16 + k - } - } else { - ret = "NOT-A-HEX-NUMBER" - } - return ret -} - -# -# expand number ranges (from..to) to sequences of numbers (emulate seq function) -# -function add_hexrange (start, end) { - from = mystrtonum("0x"start) - to = mystrtonum("0x"end) - for ( i=from; i<=to; i++ ) - temp[i] = i - result = temp[from] - for ( i=from+1; i<=to; i++ ) - result = result " " temp[i] - return result -} - -# -# initialization stuff (define categories of intrest in input file) -# -BEGIN { - FS = " " - ncategories = split("White_Space Dash Terminal_Punctuation STerm Pattern_White_Space", array_names) -} - -# -# evaluate every line in input read from -# -{ - # skip comments and empty lines - if ( $0 !~ /^#/ && NF != 0 ) { - # cycle over array_names and do the math - for (category in array_names) { From dovecot at dovecot.org Tue May 12 09:32:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 09:32:37 +0000 Subject: dovecot-2.2: lib-fts: .sh scripts weren't executable - changed t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e991baeb8bb7 changeset: 18644:e991baeb8bb7 user: Timo Sirainen date: Tue May 12 12:30:40 2015 +0300 description: lib-fts: .sh scripts weren't executable - changed them to be run via bash directly. Better to avoid relying on the executable bit. diffstat: src/lib-fts/Makefile.am | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 2cfb80f7785e -r e991baeb8bb7 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Tue May 12 12:20:56 2015 +0300 +++ b/src/lib-fts/Makefile.am Tue May 12 12:30:40 2015 +0300 @@ -30,12 +30,12 @@ WordBreakProperty.txt: test -f WordBreakProperty.txt || wget http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakProperty.txt $(srcdir)/word-boundary-data.c: word-boundary-data.sh WordBreakProperty.txt - $(srcdir)/word-boundary-data.sh < WordBreakProperty.txt > $@.tmp && mv $@.tmp $@ + bash $(srcdir)/word-boundary-data.sh < WordBreakProperty.txt > $@.tmp && mv $@.tmp $@ PropList.txt: test -f PropList.txt || wget http://www.unicode.org/Public/UNIDATA/PropList.txt $(srcdir)/word-break-data.c: word-break-data.sh PropList.txt - $(srcdir)/word-break-data.sh < PropList.txt > $@.tmp && mv $@.tmp $@ + bash $(srcdir)/word-break-data.sh < PropList.txt > $@.tmp && mv $@.tmp $@ if BUILD_FTS_STEMMER From dovecot at dovecot.org Tue May 12 09:47:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 09:47:31 +0000 Subject: dovecot-2.2: fs-compress: Added NOPLUGIN_LDFLAGS to Makefile for... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0cbb125046a5 changeset: 18645:0cbb125046a5 user: Timo Sirainen date: Tue May 12 12:45:34 2015 +0300 description: fs-compress: Added NOPLUGIN_LDFLAGS to Makefile for helping with linking diffstat: src/plugins/fs-compress/Makefile.am | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r e991baeb8bb7 -r 0cbb125046a5 src/plugins/fs-compress/Makefile.am --- a/src/plugins/fs-compress/Makefile.am Tue May 12 12:30:40 2015 +0300 +++ b/src/plugins/fs-compress/Makefile.am Tue May 12 12:45:34 2015 +0300 @@ -7,6 +7,7 @@ -I$(top_srcdir)/src/lib-compression \ -I$(top_srcdir)/src/lib-fs +NOPLUGIN_LDFLAGS = libfs_compress_la_SOURCES = fs-compress.c libfs_compress_la_LIBADD = ../../lib-compression/libdovecot-compression.la libfs_compress_la_DEPENDENCIES = ../../lib-compression/libdovecot-compression.la From dovecot at dovecot.org Tue May 12 13:15:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 13:15:23 +0000 Subject: dovecot-2.2: lib-fts: autogenerate C arrays using perl Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/58d7234a6658 changeset: 18646:58d7234a6658 user: Phil Carmody date: Tue May 12 16:12:29 2015 +0300 description: lib-fts: autogenerate C arrays using perl The sh script had bashisms, the awk script crashed mawk, so let's try perl... Signed-off-by: Phil Carmody diffstat: src/lib-fts/Makefile.am | 11 +-- src/lib-fts/word-boundary-data.sh | 99 --------------------------------------- src/lib-fts/word-break-data.sh | 77 ------------------------------ src/lib-fts/word-properties.pl | 34 +++++++++++++ 4 files changed, 39 insertions(+), 182 deletions(-) diffs (252 lines): diff -r 0cbb125046a5 -r 58d7234a6658 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Tue May 12 12:45:34 2015 +0300 +++ b/src/lib-fts/Makefile.am Tue May 12 16:12:29 2015 +0300 @@ -21,21 +21,20 @@ EXTRA_DIST = \ udhr_fra.txt \ PropList.txt \ + word-properties.pl \ WordBreakProperty.txt \ - word-boundary-data.sh \ word-boundary-data.c \ - word-break-data.sh \ word-break-data.c WordBreakProperty.txt: test -f WordBreakProperty.txt || wget http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakProperty.txt -$(srcdir)/word-boundary-data.c: word-boundary-data.sh WordBreakProperty.txt - bash $(srcdir)/word-boundary-data.sh < WordBreakProperty.txt > $@.tmp && mv $@.tmp $@ +$(srcdir)/word-boundary-data.c: word-properties.pl PropList.txt + perl word-properties.pl boundaries WordBreakProperty.txt > $@.tmp && mv $@.tmp $@ PropList.txt: test -f PropList.txt || wget http://www.unicode.org/Public/UNIDATA/PropList.txt -$(srcdir)/word-break-data.c: word-break-data.sh PropList.txt - bash $(srcdir)/word-break-data.sh < PropList.txt > $@.tmp && mv $@.tmp $@ +$(srcdir)/word-break-data.c: word-properties.pl PropList.txt + perl word-properties.pl breaks PropList.txt > $@.tmp && mv $@.tmp $@ if BUILD_FTS_STEMMER diff -r 0cbb125046a5 -r 58d7234a6658 src/lib-fts/word-boundary-data.sh --- a/src/lib-fts/word-boundary-data.sh Tue May 12 12:45:34 2015 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,99 +0,0 @@ -#!/bin/bash -# TODO: Should perhaps be written in perl/python/awk -# FIXME: The runtime is a bit long. - -#Array names match category names in data file. -declare -a CR -declare -a LF -declare -a Newline -declare -a Extend -declare -a Regional_Indicator -declare -a Format -declare -a Katakana -declare -a Hebrew_Letter -declare -a ALetter -declare -a Single_Quote -declare -a Double_Quote -declare -a MidNumLet -declare -a MidLetter -declare -a MidNum -declare -a Numeric -declare -a ExtendNumLet - -WIDTH=5 - -add_hexrange () { - - array_name="$1" - from="$2" - to="$3" - - eval "$array_name+=($(seq -s ' ' "0x$from" "0x$to"))" -} - -print_c_array () { - - array_name="$1" - eval "array=("\${$array_name[@]}")" - array_length=${#array[@]} - i=1 - - printf "static const uint32_t %s[]= {\n\t" "$array_name" - - for val in "${array[@]}" ; do - printf "0x%0${WIDTH}X" "$val" - if [ $i -lt $array_length ]; then - echo -n ", " - if [ $(($i%8)) -eq 0 ]; then - echo -ne "\n\t" - fi - i=$((i+1)) - else - break - fi - done - - echo -ne "\n};\n" -} -#read everything except comments. -while read -s -a line; do - [ -z "${line[0]}" ] && continue #ignore empty lines - - case "${line[0]}" in \#*) continue ;; esac #ignore comments - - value="${line[0]}" - category="${line[2]}" - - case "$value" in - *..*) - start=`echo "$value" | cut -d . -f 1` - end=`echo "$value" | cut -d . -f 3` - add_hexrange "$category" "$start" "$end" - ;; - *) - value=`printf "%05X" $((16#$value))` - eval "$category+=(0x\$value)" - ;; - esac; - -done - -printf "/* This file is automatically generated by %s from WordBreakProperty.txt */\n" "$0" - -print_c_array CR -print_c_array LF -print_c_array Newline -print_c_array Extend -print_c_array Regional_Indicator -print_c_array Format -print_c_array Katakana -print_c_array Hebrew_Letter -print_c_array ALetter -print_c_array Single_Quote -print_c_array Double_Quote -print_c_array MidNumLet -print_c_array MidLetter -print_c_array MidNum -print_c_array Numeric -print_c_array ExtendNumLet - diff -r 0cbb125046a5 -r 58d7234a6658 src/lib-fts/word-break-data.sh --- a/src/lib-fts/word-break-data.sh Tue May 12 12:45:34 2015 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,77 +0,0 @@ -#!/bin/bash - -#Array names match category names in data file. -array_names="White_Space Dash Terminal_Punctuation STerm Pattern_White_Space" -declare -a White_Space -declare -a Dash -declare -a Terminal_Punctuation -declare -a STerm -declare -a Pattern_White_Space -#TODO include Pattern_Syntax? - -WIDTH=5 - -add_hexrange () { - - array_name="$1" - from="$2" - to="$3" - - eval "$array_name+=($(seq -s ' ' "0x$from" "0x$to"))" -} - -print_c_array () { - - array_name="$1" - eval "array=("\${$array_name[@]}")" - array_length=${#array[@]} - i=1 - - printf "static const uint32_t %s[]= {\n\t" "$array_name" - - for val in "${array[@]}" ; do - printf "0x%0${WIDTH}X" "$val" - if [ $i -lt $array_length ]; then - echo -n ", " - if [ $(($i%8)) -eq 0 ]; then - echo -ne "\n\t" - fi - i=$((i+1)) - else - break - fi - done - - echo -ne "\n};\n" -} -#read everything except comments. -while read -s -a line; do - [ -z "${line[0]}" ] && continue #ignore empty lines - - case "${line[0]}" in \#*) continue ;; esac #ignore comments - - value="${line[0]}" - category="${line[2]}" - - case "$array_names" in - *"$category"*) - case "$value" in - *..*) - start=`echo "$value" | cut -d . -f 1` - end=`echo "$value" | cut -d . -f 3` - add_hexrange "$category" "$start" "$end" - ;; - *) - value=`printf "%05X" $((16#$value))` - eval "$category+=(0x\$value)" - ;; - esac - ;; - esac -done -printf "/* This file is automatically generated by %s from PropList.txt */\n" "$0" - -for name in $array_names; do - print_c_array "$name" -done - diff -r 0cbb125046a5 -r 58d7234a6658 src/lib-fts/word-properties.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/word-properties.pl Tue May 12 16:12:29 2015 +0300 @@ -0,0 +1,34 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +my @categories; +my $which = shift(@ARGV); +if ($which eq 'boundaries') { + @categories = qw(CR LF Newline Extend Regional_Indicator Format Katakana Hebrew_Letter ALetter + Single_Quote Double_Quote MidNumLet MidLetter MidNum Numeric ExtendNumLet); +} elsif ($which eq 'breaks') { + @categories = qw(White_Space Dash Terminal_Punctuation STerm Pattern_White_Space); +} else { + die "specify 'boundaries' or 'breaks'"; +} + +my $catregexp=join('|', @categories); +my %catlists = map { $_ => []; } (@categories); + +while(<>) { + next if (m/^#/ or m/^\s*$/); + push(@{$catlists{$3}}, defined($2) ? (hex($1)..hex($2)) : hex($1)) + if (m/([[:xdigit:]]+)(?:\.\.([[:xdigit:]]+))?\s+; ($catregexp) #/) +} + +print "/* This file is automatically generated by word-properties.pl from $ARGV */\n"; +foreach(@categories) { + my $arref=$catlists{$_}; + print "static const uint32_t ${_}[]= {\n"; + while(scalar(@$arref)) { + print("\t", join(", ", map { sprintf("0x%05X", $_); } splice(@$arref, 0, 8))); + print(scalar(@$arref) ? ", \n" : "\n"); + } + print("};\n"); +} From dovecot at dovecot.org Tue May 12 13:46:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 13:46:43 +0000 Subject: dovecot-2.2: lib-fts: ICU normalizer code cleanup. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d09d2ea2c31a changeset: 18647:d09d2ea2c31a user: Teemu Huovila date: Tue May 12 16:44:45 2015 +0300 description: lib-fts: ICU normalizer code cleanup. Fold some long lines. Rename the internal struct to be more aligned with the other filters. diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 33 ++++++++++++++++++--------------- 1 files changed, 18 insertions(+), 15 deletions(-) diffs (96 lines): diff -r 58d7234a6658 -r d09d2ea2c31a src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Tue May 12 16:12:29 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Tue May 12 16:44:45 2015 +0300 @@ -15,7 +15,7 @@ #include #include -struct fts_filter_normalizer { +struct fts_filter_normalizer_icu { struct fts_filter filter; pool_t pool; const char *transliterator_id; @@ -116,8 +116,8 @@ static void fts_filter_normalizer_icu_destroy(struct fts_filter *filter) { - struct fts_filter_normalizer *np = - (struct fts_filter_normalizer *)filter; + struct fts_filter_normalizer_icu *np = + (struct fts_filter_normalizer_icu *)filter; if (np->transliterator != NULL) utrans_close(np->transliterator); @@ -130,7 +130,7 @@ struct fts_filter **filter_r, const char **error_r) { - struct fts_filter_normalizer *np; + struct fts_filter_normalizer_icu *np; pool_t pp; unsigned int i; const char *id = "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC"; @@ -146,9 +146,9 @@ } } - pp = pool_alloconly_create(MEMPOOL_GROWING"fts_filter_normalizer", - sizeof(struct fts_filter_normalizer)); - np = p_new(pp, struct fts_filter_normalizer, 1); + pp = pool_alloconly_create(MEMPOOL_GROWING"fts_filter_normalizer_icu", + sizeof(struct fts_filter_normalizer_icu)); + np = p_new(pp, struct fts_filter_normalizer_icu, 1); np->pool = pp; np->filter = *fts_filter_normalizer_icu; np->transliterator_id = p_strdup(pp, id); @@ -156,8 +156,9 @@ return 0; } -static int fts_filter_normalizer_icu_create_trans(struct fts_filter_normalizer *np, - const char **error_r) +static int +fts_filter_normalizer_icu_create_trans(struct fts_filter_normalizer_icu *np, + const char **error_r) { UErrorCode err = U_ZERO_ERROR; UParseError perr; @@ -168,8 +169,8 @@ make_uchar(np->transliterator_id, &id_uchar, &id_len_uchar); - np->transliterator = utrans_openU(id_uchar, u_strlen(id_uchar), UTRANS_FORWARD, - NULL, 0, &perr, &err); + np->transliterator = utrans_openU(id_uchar, u_strlen(id_uchar), + UTRANS_FORWARD, NULL, 0, &perr, &err); if (U_FAILURE(err)) { string_t *str = t_str_new(128); @@ -177,7 +178,8 @@ np->transliterator_id, u_errorName(err)); if (perr.line >= 1) { /* we have only one line in our ID */ - str_printfa(str, " (parse error on offset %u)", perr.offset); + str_printfa(str, " (parse error on offset %u)", + perr.offset); } *error_r = str_c(str); return -1; @@ -189,8 +191,8 @@ fts_filter_normalizer_icu_filter(struct fts_filter *filter, const char **token, const char **error_r) { - struct fts_filter_normalizer *np = - (struct fts_filter_normalizer *)filter; + struct fts_filter_normalizer_icu *np = + (struct fts_filter_normalizer_icu *)filter; UErrorCode err = U_ZERO_ERROR; UChar *utext = NULL; int32_t utext_cap = 0; @@ -274,4 +276,5 @@ .v = &normalizer_filter_vfuncs }; -const struct fts_filter *fts_filter_normalizer_icu = &fts_filter_normalizer_icu_real; +const struct fts_filter *fts_filter_normalizer_icu = + &fts_filter_normalizer_icu_real; From dovecot at dovecot.org Tue May 12 15:18:51 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 15:18:51 +0000 Subject: dovecot-2.2: lib-sql: Don't crash in Cassandra if connection to ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d71fa3ae930f changeset: 18648:d71fa3ae930f user: Timo Sirainen date: Tue May 12 18:16:54 2015 +0300 description: lib-sql: Don't crash in Cassandra if connection to it failed. diffstat: src/lib-sql/driver-cassandra.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (18 lines): diff -r d09d2ea2c31a -r d71fa3ae930f src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Tue May 12 16:44:45 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Tue May 12 18:16:54 2015 +0300 @@ -530,9 +530,11 @@ { if (db->orig_ioloop == NULL) return; - io_loop_set_current(db->orig_ioloop); - db->io_pipe = io_loop_move_io(&db->io_pipe); - io_loop_set_current(db->ioloop); + if (db->io_pipe != NULL) { + io_loop_set_current(db->orig_ioloop); + db->io_pipe = io_loop_move_io(&db->io_pipe); + io_loop_set_current(db->ioloop); + } io_loop_destroy(&db->ioloop); } From dovecot at dovecot.org Tue May 12 16:16:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 16:16:11 +0000 Subject: dovecot-2.2: lib-dict: Allow registering builtin dict drivers mu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ac259cd62fbc changeset: 18649:ac259cd62fbc user: Timo Sirainen date: Tue May 12 19:12:11 2015 +0300 description: lib-dict: Allow registering builtin dict drivers multiple times. diffstat: src/lib-dict/dict-register.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (25 lines): diff -r d71fa3ae930f -r ac259cd62fbc src/lib-dict/dict-register.c --- a/src/lib-dict/dict-register.c Tue May 12 18:16:54 2015 +0300 +++ b/src/lib-dict/dict-register.c Tue May 12 19:12:11 2015 +0300 @@ -3,8 +3,12 @@ #include "lib.h" #include "dict-private.h" +static int refcount = 0; + void dict_drivers_register_builtin(void) { + if (refcount++ > 0) + return; dict_driver_register(&dict_driver_client); dict_driver_register(&dict_driver_file); dict_driver_register(&dict_driver_fs); @@ -15,6 +19,8 @@ void dict_drivers_unregister_builtin(void) { + if (--refcount > 0) + return; dict_driver_unregister(&dict_driver_client); dict_driver_unregister(&dict_driver_file); dict_driver_unregister(&dict_driver_fs); From dovecot at dovecot.org Tue May 12 16:16:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 16:16:16 +0000 Subject: dovecot-2.2: doveadm: Register builtin dict drivers always at init Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2460008d50d1 changeset: 18650:2460008d50d1 user: Timo Sirainen date: Tue May 12 19:12:34 2015 +0300 description: doveadm: Register builtin dict drivers always at init diffstat: src/doveadm/doveadm.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (27 lines): diff -r ac259cd62fbc -r 2460008d50d1 src/doveadm/doveadm.c --- a/src/doveadm/doveadm.c Tue May 12 19:12:11 2015 +0300 +++ b/src/doveadm/doveadm.c Tue May 12 19:12:34 2015 +0300 @@ -5,6 +5,7 @@ #include "str.h" #include "env-util.h" #include "execv-const.h" +#include "dict.h" #include "master-service-private.h" #include "master-service-settings.h" #include "settings-parser.h" @@ -317,6 +318,7 @@ quick_init = FALSE; doveadm_dump_init(); doveadm_mail_init(); + dict_drivers_register_builtin(); doveadm_load_modules(); if (cmd_name == NULL) { @@ -356,6 +358,7 @@ doveadm_mail_deinit(); doveadm_dump_deinit(); doveadm_unload_modules(); + dict_drivers_unregister_builtin(); doveadm_print_deinit(); } doveadm_cmds_deinit(); From dovecot at dovecot.org Tue May 12 16:16:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 16:16:17 +0000 Subject: dovecot-2.2: lib: connection API was unnecessarily delaying clie... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1fac17a2bc53 changeset: 18651:1fac17a2bc53 user: Timo Sirainen date: Tue May 12 19:13:31 2015 +0300 description: lib: connection API was unnecessarily delaying client_connected() calls for UNIX sockets. diffstat: src/lib/connection.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 2460008d50d1 -r 1fac17a2bc53 src/lib/connection.c --- a/src/lib/connection.c Tue May 12 19:12:34 2015 +0300 +++ b/src/lib/connection.c Tue May 12 19:13:31 2015 +0300 @@ -262,7 +262,7 @@ return -1; conn->fd_in = conn->fd_out = fd; - if (conn->port != 0 || conn->list->v.client_connected != NULL) { + if (conn->port != 0) { conn->io = io_add(conn->fd_out, IO_WRITE, connection_socket_connected, conn); if (set->client_connect_timeout_msecs != 0) { From dovecot at dovecot.org Tue May 12 16:16:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 16:16:24 +0000 Subject: dovecot-2.2: cassandra: Leave consistency to default if it's not... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f0273ba240ae changeset: 18652:f0273ba240ae user: Timo Sirainen date: Tue May 12 19:14:13 2015 +0300 description: cassandra: Leave consistency to default if it's not specified in config. diffstat: src/lib-sql/driver-cassandra.c | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diffs (30 lines): diff -r 1fac17a2bc53 -r f0273ba240ae src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Tue May 12 19:13:31 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Tue May 12 19:14:13 2015 +0300 @@ -44,6 +44,8 @@ struct sql_result *sync_result; char *error; + + unsigned int set_consistency:1; }; struct cassandra_result { @@ -322,6 +324,7 @@ } else if (strcmp(key, "consistency") == 0) { if (consistency_parse(value, &db->consistency) < 0) i_fatal("cassandra: Unknown consistency: %s", value); + db->set_consistency = TRUE; } else { i_fatal("cassandra: Unknown connect string: %s", key); } @@ -468,7 +471,8 @@ result->query = i_strdup(query); result->row_pool = pool_alloconly_create("cassandra result", 512); result->statement = cass_statement_new(query, 0); - cass_statement_set_consistency(result->statement, db->consistency); + if (db->set_consistency) + cass_statement_set_consistency(result->statement, db->consistency); future = cass_session_execute(db->session, result->statement); driver_cassandra_set_callback(future, db, query_callback, result); } From dovecot at dovecot.org Tue May 12 16:45:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 16:45:46 +0000 Subject: dovecot-2.2: dict-sql: Fixed non-recursive iteration. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fce3e3eef9f0 changeset: 18653:fce3e3eef9f0 user: Timo Sirainen date: Tue May 12 19:43:25 2015 +0300 description: dict-sql: Fixed non-recursive iteration. If path has e.g. "a/$var/b/$var2" and we're iterating "a/", we shouldn't return anything under "a/*/b/". diffstat: src/lib-dict/dict-sql.c | 19 +++++++++++++------ 1 files changed, 13 insertions(+), 6 deletions(-) diffs (55 lines): diff -r f0273ba240ae -r fce3e3eef9f0 src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Tue May 12 19:14:13 2015 +0300 +++ b/src/lib-dict/dict-sql.c Tue May 12 19:43:25 2015 +0300 @@ -110,7 +110,7 @@ static bool dict_sql_map_match(const struct dict_sql_map *map, const char *path, ARRAY_TYPE(const_string) *values, unsigned int *pat_len_r, - unsigned int *path_len_r, bool partial_ok) + unsigned int *path_len_r, bool partial_ok, bool recurse) { const char *path_start = path; const char *pat, *field, *p; @@ -174,8 +174,14 @@ else if (!partial_ok) return FALSE; else { - /* partial matches must end with '/' */ - return pat == map->pattern || pat[-1] == '/'; + /* partial matches must end with '/'. */ + if (pat != map->pattern && pat[-1] != '/') + return FALSE; + /* if we're not recursing, there should be only one $variable + left. */ + if (recurse) + return TRUE; + return pat[0] == '$' && strchr(pat, '/') == NULL; } } @@ -192,7 +198,7 @@ /* start matching from the previously successful match */ idx = (dict->prev_map_match_idx + i) % count; if (dict_sql_map_match(&maps[idx], path, values, - &len, &len, FALSE)) { + &len, &len, FALSE, FALSE)) { dict->prev_map_match_idx = idx; return &maps[idx]; } @@ -314,13 +320,14 @@ struct sql_dict *dict = (struct sql_dict *)ctx->ctx.dict; const struct dict_sql_map *maps; unsigned int i, count, pat_len, path_len; + bool recurse = (ctx->flags & DICT_ITERATE_FLAG_RECURSE) != 0; t_array_init(values, dict->set->max_field_count); maps = array_get(&dict->set->maps, &count); for (i = ctx->next_map_idx; i < count; i++) { if (dict_sql_map_match(&maps[i], ctx->paths[ctx->path_idx], - values, &pat_len, &path_len, TRUE) && - ((ctx->flags & DICT_ITERATE_FLAG_RECURSE) != 0 || + values, &pat_len, &path_len, TRUE, recurse) && + (recurse || array_count(values)+1 >= array_count(&maps[i].sql_fields))) { ctx->key_prefix_len = path_len; ctx->pattern_prefix_len = pat_len; From dovecot at dovecot.org Tue May 12 16:45:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 16:45:46 +0000 Subject: dovecot-2.2: doveadm dict iter: Added -1 parameter to enable DIC... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/04e169b3bfe6 changeset: 18654:04e169b3bfe6 user: Timo Sirainen date: Tue May 12 19:43:48 2015 +0300 description: doveadm dict iter: Added -1 parameter to enable DICT_ITERATE_FLAG_EXACT_KEY diffstat: src/doveadm/doveadm-dict.c | 20 +++++++++----------- 1 files changed, 9 insertions(+), 11 deletions(-) diffs (52 lines): diff -r fce3e3eef9f0 -r 04e169b3bfe6 src/doveadm/doveadm-dict.c --- a/src/doveadm/doveadm-dict.c Tue May 12 19:43:25 2015 +0300 +++ b/src/doveadm/doveadm-dict.c Tue May 12 19:43:48 2015 +0300 @@ -12,21 +12,20 @@ static struct dict * cmd_dict_init_full(int *argc, char **argv[], int own_arg_count, int key_arg_idx, - doveadm_command_t *cmd, bool *recurse) + doveadm_command_t *cmd, enum dict_iterate_flags *iter_flags) { - const char *getopt_args = recurse == NULL ? "u:" : "Ru:"; + const char *getopt_args = iter_flags == NULL ? "u:" : "1Ru:"; struct dict *dict; const char *error, *username = ""; int c; - if (recurse != NULL) - *recurse = FALSE; - while ((c = getopt(*argc, *argv, getopt_args)) > 0) { switch (c) { + case '1': + *iter_flags |= DICT_ITERATE_FLAG_EXACT_KEY; + break; case 'R': - i_assert(recurse != NULL); - *recurse = TRUE; + *iter_flags |= DICT_ITERATE_FLAG_RECURSE; break; case 'u': username = optarg; @@ -155,17 +154,16 @@ { struct dict *dict; struct dict_iterate_context *iter; + enum dict_iterate_flags iter_flags = 0; const char *key, *value; - bool recurse; - dict = cmd_dict_init_full(&argc, &argv, 1, 0, cmd_dict_iter, &recurse); + dict = cmd_dict_init_full(&argc, &argv, 1, 0, cmd_dict_iter, &iter_flags); doveadm_print_init(DOVEADM_PRINT_TYPE_TAB); doveadm_print_header_simple("key"); doveadm_print_header_simple("value"); - iter = dict_iterate_init(dict, argv[0], - recurse ? DICT_ITERATE_FLAG_RECURSE : 0); + iter = dict_iterate_init(dict, argv[0], iter_flags); while (dict_iterate(iter, &key, &value)) { doveadm_print(key); doveadm_print(value); From dovecot at dovecot.org Tue May 12 18:54:05 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 18:54:05 +0000 Subject: dovecot-2.2: doveadm fs: Improved error message logging. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c4bef3533ecd changeset: 18655:c4bef3533ecd user: Timo Sirainen date: Tue May 12 21:52:07 2015 +0300 description: doveadm fs: Improved error message logging. diffstat: src/doveadm/doveadm-fs.c | 14 +++++++++----- 1 files changed, 9 insertions(+), 5 deletions(-) diffs (31 lines): diff -r 04e169b3bfe6 -r c4bef3533ecd src/doveadm/doveadm-fs.c --- a/src/doveadm/doveadm-fs.c Tue May 12 19:43:48 2015 +0300 +++ b/src/doveadm/doveadm-fs.c Tue May 12 21:52:07 2015 +0300 @@ -70,7 +70,8 @@ i_error("%s doesn't exist", fs_file_path(file)); doveadm_exit_code = DOVEADM_EX_NOTFOUND; } else if (input->stream_errno != 0) { - i_error("read(%s) failed: %m", fs_file_path(file)); + i_error("read(%s) failed: %s", fs_file_path(file), + fs_file_last_error(file)); doveadm_exit_code = EX_TEMPFAIL; } i_stream_unref(&input); @@ -126,10 +127,13 @@ output = fs_write_stream(file); input = i_stream_create_file(src_path, IO_BLOCK_SIZE); if ((ret = o_stream_send_istream(output, input)) < 0) { - if (output->stream_errno != 0) - i_error("write(%s) failed: %m", dest_path); - else - i_error("read(%s) failed: %m", src_path); + if (output->stream_errno != 0) { + i_error("write(%s) failed: %s", dest_path, + o_stream_get_error(output)); + } else { + i_error("read(%s) failed: %s", src_path, + i_stream_get_error(input)); + } doveadm_exit_code = EX_TEMPFAIL; } i_stream_destroy(&input); From pigeonhole at rename-it.nl Tue May 12 18:53:57 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 12 May 2015 20:53:57 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: storage: Added parameter to s... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/9308f3f2e4d4 changeset: 2059:9308f3f2e4d4 user: Stephan Bosch date: Tue May 12 20:53:46 2015 +0200 description: lib-sieve: storage: Added parameter to sieve_script_delete() to allow ignoring the active status of the script. Normally, the active script cannot be deleted. This change is necessary to handle doveadm sieve delete -a correctly when the provided script is the default. diffstat: src/lib-sieve/sieve-script.c | 23 +++++++++++++------ src/lib-sieve/sieve-script.h | 6 +++- src/lib-sieve/sieve-storage.c | 2 +- src/managesieve/cmd-deletescript.c | 2 +- src/plugins/doveadm-sieve/doveadm-sieve-cmd-delete.c | 8 +----- src/plugins/doveadm-sieve/doveadm-sieve-sync.c | 2 +- 6 files changed, 25 insertions(+), 18 deletions(-) diffs (126 lines): diff -r 59d59552247c -r 9308f3f2e4d4 src/lib-sieve/sieve-script.c --- a/src/lib-sieve/sieve-script.c Mon May 11 23:05:24 2015 +0200 +++ b/src/lib-sieve/sieve-script.c Tue May 12 20:53:46 2015 +0200 @@ -592,7 +592,7 @@ } else if ( sieve_script_activate(newscript, (time_t)-1) < 0 ) { /* Failed to activate; roll back */ ret = -1; - (void)sieve_script_delete(newscript); + (void)sieve_script_delete(newscript, TRUE); sieve_script_unref(&newscript); } @@ -611,20 +611,25 @@ return ret; } -int sieve_script_delete(struct sieve_script *script) +int sieve_script_delete(struct sieve_script *script, + bool ignore_active) { struct sieve_storage *storage = script->storage; + bool is_active = FALSE; int ret = 0; i_assert( script->open ); // FIXME: auto-open? /* Is the requested script active? */ if ( sieve_script_is_active(script) > 0 ) { - sieve_script_set_error(script, SIEVE_ERROR_ACTIVE, - "Cannot delete the active Sieve script."); - if (storage->default_for != NULL) - sieve_storage_copy_error(storage->default_for, storage); - return -1; + is_active = TRUE; + if ( !ignore_active ) { + sieve_script_set_error(script, SIEVE_ERROR_ACTIVE, + "Cannot delete the active Sieve script."); + if (storage->default_for != NULL) + sieve_storage_copy_error(storage->default_for, storage); + return -1; + } } /* Trying to delete the default script? */ @@ -635,6 +640,10 @@ i_assert( (script->storage->flags & SIEVE_STORAGE_FLAG_READWRITE) != 0 ); + /* Deactivate it explicity */ + if ( ignore_active && is_active ) + (void)sieve_storage_deactivate(storage, (time_t)-1); + i_assert( script->v.delete != NULL ); ret = script->v.delete(script); diff -r 59d59552247c -r 9308f3f2e4d4 src/lib-sieve/sieve-script.h --- a/src/lib-sieve/sieve-script.h Mon May 11 23:05:24 2015 +0200 +++ b/src/lib-sieve/sieve-script.h Tue May 12 20:53:46 2015 +0200 @@ -102,8 +102,10 @@ int sieve_script_rename (struct sieve_script *script, const char *newname); int sieve_script_is_active(struct sieve_script *script); -int sieve_script_activate(struct sieve_script *script, time_t mtime); -int sieve_script_delete(struct sieve_script *script); +int sieve_script_activate + (struct sieve_script *script, time_t mtime); +int sieve_script_delete + (struct sieve_script *script, bool ignore_active); /* * Properties diff -r 59d59552247c -r 9308f3f2e4d4 src/lib-sieve/sieve-storage.c --- a/src/lib-sieve/sieve-storage.c Mon May 11 23:05:24 2015 +0200 +++ b/src/lib-sieve/sieve-storage.c Tue May 12 20:53:46 2015 +0200 @@ -1140,7 +1140,7 @@ } else if ( sieve_script_activate(script, (time_t)-1) < 0 ) { /* Failed to activate; roll back */ ret = -1; - (void)sieve_script_delete(script); + (void)sieve_script_delete(script, TRUE); sieve_script_unref(&script); } diff -r 59d59552247c -r 9308f3f2e4d4 src/managesieve/cmd-deletescript.c --- a/src/managesieve/cmd-deletescript.c Mon May 11 23:05:24 2015 +0200 +++ b/src/managesieve/cmd-deletescript.c Tue May 12 20:53:46 2015 +0200 @@ -28,7 +28,7 @@ return TRUE; } - if ( sieve_script_delete(script) < 0 ) { + if ( sieve_script_delete(script, FALSE) < 0 ) { client_send_storage_error(client, storage); } else { client_send_ok(client, "Deletescript completed."); diff -r 59d59552247c -r 9308f3f2e4d4 src/plugins/doveadm-sieve/doveadm-sieve-cmd-delete.c --- a/src/plugins/doveadm-sieve/doveadm-sieve-cmd-delete.c Mon May 11 23:05:24 2015 +0200 +++ b/src/plugins/doveadm-sieve/doveadm-sieve-cmd-delete.c Tue May 12 20:53:46 2015 +0200 @@ -39,13 +39,9 @@ if (script == NULL) { sret = -1; } else { - if (sieve_script_delete(script) < 0) { + if (sieve_script_delete(script, ctx->ignore_active) < 0) { (void)sieve_storage_get_last_error(storage, &error); - if (!ctx->ignore_active || error != SIEVE_ERROR_ACTIVE || - sieve_storage_deactivate(storage, (time_t)-1) < 0 || - sieve_script_delete(script) < 0) { - sret = -1; - } + sret = -1; } sieve_script_unref(&script); } diff -r 59d59552247c -r 9308f3f2e4d4 src/plugins/doveadm-sieve/doveadm-sieve-sync.c --- a/src/plugins/doveadm-sieve/doveadm-sieve-sync.c Mon May 11 23:05:24 2015 +0200 +++ b/src/plugins/doveadm-sieve/doveadm-sieve-sync.c Tue May 12 20:53:46 2015 +0200 @@ -118,7 +118,7 @@ if (script == NULL) { ret = -1; } else { - ret = sieve_script_delete(script); + ret = sieve_script_delete(script, TRUE); sieve_script_unref(&script); } From dovecot at dovecot.org Tue May 12 19:00:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 19:00:10 +0000 Subject: dovecot-2.2: lib: connection API: Added delayed_unix_client_conn... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6f167f2e550d changeset: 18656:6f167f2e550d user: Timo Sirainen date: Tue May 12 21:58:03 2015 +0300 description: lib: connection API: Added delayed_unix_client_connected_callback setting. 092a51d80bad commit changed this functionality first to fix lib-http code, but it broke other code. 1fac17a2bc53 reversed the original behavior. This change allows either behavior optionally. diffstat: src/lib/connection.c | 3 ++- src/lib/connection.h | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diffs (40 lines): diff -r c4bef3533ecd -r 6f167f2e550d src/lib/connection.c --- a/src/lib/connection.c Tue May 12 21:52:07 2015 +0300 +++ b/src/lib/connection.c Tue May 12 21:58:03 2015 +0300 @@ -262,7 +262,8 @@ return -1; conn->fd_in = conn->fd_out = fd; - if (conn->port != 0) { + if (conn->port != 0 || + conn->list->set.delayed_unix_client_connected_callback) { conn->io = io_add(conn->fd_out, IO_WRITE, connection_socket_connected, conn); if (set->client_connect_timeout_msecs != 0) { diff -r c4bef3533ecd -r 6f167f2e550d src/lib/connection.h --- a/src/lib/connection.h Tue May 12 21:52:07 2015 +0300 +++ b/src/lib/connection.h Tue May 12 21:58:03 2015 +0300 @@ -27,8 +27,9 @@ struct connection_vfuncs { void (*destroy)(struct connection *conn); - /* For UNIX socket clients this gets called immediately with - success=TRUE, for IP connections it gets called later: + /* For UNIX socket clients this gets called immediately (unless + delayed_unix_client_connected_callback=TRUE) with success=TRUE, + for IP connections it gets called later: If connect() fails, sets success=FALSE and errno. Streams aren't initialized in that situation either. destroy() is called after @@ -57,6 +58,11 @@ bool client; bool dont_send_version; + /* Don't call client_connected() immediately on + connection_client_connect() with UNIX sockets. This is mainly + to make the functionality identical with inet sockets, which may + simplify the calling code. */ + bool delayed_unix_client_connected_callback; }; struct connection { From pigeonhole at rename-it.nl Tue May 12 20:51:24 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 12 May 2015 22:51:24 +0200 Subject: dovecot-2.2-pigeonhole: lib-smtp: storage: Fixed small problem i... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/3fab77ef9302 changeset: 2060:3fab77ef9302 user: Stephan Bosch date: Tue May 12 22:51:12 2015 +0200 description: lib-smtp: storage: Fixed small problem in handling of default script. When the default was first overriden, then deactivated and finally overriden again, it would erroneously implicitly re-activate. diffstat: src/lib-sieve/sieve-storage.c | 44 +++++++++++++++++++++++++++++++++++++----- 1 files changed, 38 insertions(+), 6 deletions(-) diffs (89 lines): diff -r 9308f3f2e4d4 -r 3fab77ef9302 src/lib-sieve/sieve-storage.c --- a/src/lib-sieve/sieve-storage.c Tue May 12 20:53:46 2015 +0200 +++ b/src/lib-sieve/sieve-storage.c Tue May 12 22:51:12 2015 +0200 @@ -673,11 +673,10 @@ * Script access */ -struct sieve_script *sieve_storage_get_script +static struct sieve_script *sieve_storage_get_script_direct (struct sieve_storage *storage, const char *name, enum sieve_error *error_r) { - struct sieve_instance *svinst = storage->svinst; struct sieve_script *script; if ( error_r != NULL ) @@ -697,7 +696,18 @@ i_assert(storage->v.get_script != NULL); script = storage->v.get_script(storage, name); + return script; +} +struct sieve_script *sieve_storage_get_script +(struct sieve_storage *storage, const char *name, + enum sieve_error *error_r) +{ + struct sieve_instance *svinst = storage->svinst; + struct sieve_script *script; + + script = sieve_storage_get_script_direct + (storage, name, error_r); if ( script == NULL ) { /* Error */ if ( storage->error_code == SIEVE_ERROR_NOT_FOUND && @@ -768,6 +778,26 @@ return script; } +static int sieve_storage_check_script_direct +(struct sieve_storage *storage, const char *name, + enum sieve_error *error_r) ATTR_NULL(3) +{ + struct sieve_script *script; + enum sieve_error error; + int ret; + + if ( error_r == NULL ) + error_r = &error; + + script = sieve_storage_get_script_direct(storage, name, error_r); + if ( script == NULL ) + return ( *error_r == SIEVE_ERROR_NOT_FOUND ? 0 : -1 ); + + ret = sieve_script_open(script, error_r); + return ( ret >= 0 ? 1 : + ( *error_r == SIEVE_ERROR_NOT_FOUND ? 0 : -1 ) ); +} + int sieve_storage_check_script (struct sieve_storage *storage, const char *name, enum sieve_error *error_r) @@ -775,12 +805,12 @@ struct sieve_script *script; enum sieve_error error; - if (error_r == NULL) + if ( error_r == NULL ) error_r = &error; script = sieve_storage_open_script(storage, name, error_r); - if (script == NULL) - return ( *error_r == SIEVE_ERROR_NOT_FOUND ? 0 : -1); + if ( script == NULL ) + return ( *error_r == SIEVE_ERROR_NOT_FOUND ? 0 : -1 ); sieve_script_unref(&script); return 1; @@ -1117,7 +1147,9 @@ storage->default_location != NULL && (storage->flags & SIEVE_STORAGE_FLAG_SYNCHRONIZING) == 0 && strcmp(sctx->scriptname, storage->default_name) == 0 && - sieve_storage_save_will_activate(sctx) ) + sieve_storage_save_will_activate(sctx) && + sieve_storage_check_script_direct + (storage, storage->default_name, NULL) <= 0 ) default_activate = TRUE; scriptname = t_strdup(sctx->scriptname); From dovecot at dovecot.org Tue May 12 21:28:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 21:28:15 +0000 Subject: dovecot-2.2: Released v2.2.17.rc2. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2c1ce55de520 changeset: 18657:2c1ce55de520 user: Timo Sirainen date: Tue May 12 23:06:30 2015 +0300 description: Released v2.2.17.rc2. diffstat: NEWS | 2 ++ configure.ac | 2 +- 2 files changed, 3 insertions(+), 1 deletions(-) diffs (24 lines): diff -r 6f167f2e550d -r 2c1ce55de520 NEWS --- a/NEWS Tue May 12 21:58:03 2015 +0300 +++ b/NEWS Tue May 12 23:06:30 2015 +0300 @@ -32,6 +32,8 @@ + doveadm: Added -F parameter to read a list of users from the given file and run the command for all the users. This is similar to -A parameter reading the list of users from userdb lookup. + + Implemented initial Cassandra CQL support as lib-sql backend. It's + only usable as dict backend currently. - auth: If auth_master_user_separator was set, auth process could be crashed by trying to log in with empty master username. - imap-login, pop3-login: Fixed crash on handshake failures with new diff -r 6f167f2e550d -r 2c1ce55de520 configure.ac --- a/configure.ac Tue May 12 21:58:03 2015 +0300 +++ b/configure.ac Tue May 12 23:06:30 2015 +0300 @@ -2,7 +2,7 @@ # Be sure to update ABI version also if anything changes that might require # recompiling plugins. Most importantly that means if any structs are changed. -AC_INIT([Dovecot],[2.2.17.rc1],[dovecot at dovecot.org]) +AC_INIT([Dovecot],[2.2.17.rc2],[dovecot at dovecot.org]) AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv17($PACKAGE_VERSION)", [Dovecot ABI version]) AC_CONFIG_SRCDIR([src]) From dovecot at dovecot.org Tue May 12 21:28:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 21:28:15 +0000 Subject: dovecot-2.2: Added tag 2.2.17.rc2 for changeset 2c1ce55de520 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b93731b6bc98 changeset: 18658:b93731b6bc98 user: Timo Sirainen date: Tue May 12 23:06:30 2015 +0300 description: Added tag 2.2.17.rc2 for changeset 2c1ce55de520 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 2c1ce55de520 -r b93731b6bc98 .hgtags --- a/.hgtags Tue May 12 23:06:30 2015 +0300 +++ b/.hgtags Tue May 12 23:06:30 2015 +0300 @@ -124,3 +124,4 @@ a3c27cec411261b3ff2947795b3b8ff96a5320a1 2.2.16.rc1 0f8f0d8ee60745f6ce7ff379da7f2660aec7c4d4 2.2.16 da685736985a17c2b6b1ff24c4f762a4009ce598 2.2.17.rc1 +2c1ce55de520484431a63041602829d21195b4e9 2.2.17.rc2 From dovecot at dovecot.org Tue May 12 21:28:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 21:28:15 +0000 Subject: dovecot-2.2: Added signature for changeset 2c1ce55de520 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/71d88e63b183 changeset: 18659:71d88e63b183 user: Timo Sirainen date: Tue May 12 23:06:33 2015 +0300 description: Added signature for changeset 2c1ce55de520 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r b93731b6bc98 -r 71d88e63b183 .hgsigs --- a/.hgsigs Tue May 12 23:06:30 2015 +0300 +++ b/.hgsigs Tue May 12 23:06:33 2015 +0300 @@ -87,3 +87,4 @@ a3c27cec411261b3ff2947795b3b8ff96a5320a1 0 iEYEABECAAYFAlT5zBYACgkQyUhSUUBVisnQ4gCdFb/4T+WZYCAxRKjfup+xIiP26bkAn2CbIlsJBU1f3WnAJX9KsAmhphlY 0f8f0d8ee60745f6ce7ff379da7f2660aec7c4d4 0 iEYEABECAAYFAlUBs5IACgkQyUhSUUBVisn/yACcCUC3xDOCPFAgTKd72uyDrEUv7wcAn1tNcf+AyIUmM9ksDeB2fuZZZJFT da685736985a17c2b6b1ff24c4f762a4009ce598 0 iEYEABECAAYFAlVORBAACgkQyUhSUUBViskBnQCfRmUjoFty2pUJrrrdIvkEgOCiVEUAnRK0OYM4z5bxs9xcBlXZn8n2qyFx +2c1ce55de520484431a63041602829d21195b4e9 0 iEYEABECAAYFAlVSXUYACgkQyUhSUUBVislsrwCdEtWDWrQPsJua75RL83lpkeh7DagAoKW+534P0iziiX/nD6QzNmBTHODe From dovecot at dovecot.org Tue May 12 21:35:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 12 May 2015 21:35:32 +0000 Subject: dovecot-2.2: Make static analyzer happier. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e899171adc14 changeset: 18660:e899171adc14 user: Timo Sirainen date: Wed May 13 00:33:18 2015 +0300 description: Make static analyzer happier. diffstat: src/doveadm/doveadm-dict.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (15 lines): diff -r 71d88e63b183 -r e899171adc14 src/doveadm/doveadm-dict.c --- a/src/doveadm/doveadm-dict.c Tue May 12 23:06:33 2015 +0300 +++ b/src/doveadm/doveadm-dict.c Wed May 13 00:33:18 2015 +0300 @@ -22,9 +22,11 @@ while ((c = getopt(*argc, *argv, getopt_args)) > 0) { switch (c) { case '1': + i_assert(iter_flags != NULL); *iter_flags |= DICT_ITERATE_FLAG_EXACT_KEY; break; case 'R': + i_assert(iter_flags != NULL); *iter_flags |= DICT_ITERATE_FLAG_RECURSE; break; case 'u': From dovecot at dovecot.org Wed May 13 01:13:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 01:13:34 +0000 Subject: dovecot-2.2: director: Improved debug logging about connecting t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f43adca71e15 changeset: 18661:f43adca71e15 user: Timo Sirainen date: Wed May 13 04:07:46 2015 +0300 description: director: Improved debug logging about connecting to another director. diffstat: src/director/director.c | 18 ++++++++++++++++-- 1 files changed, 16 insertions(+), 2 deletions(-) diffs (28 lines): diff -r e899171adc14 -r f43adca71e15 src/director/director.c --- a/src/director/director.c Wed May 13 00:33:18 2015 +0300 +++ b/src/director/director.c Wed May 13 04:07:46 2015 +0300 @@ -109,8 +109,22 @@ if (director_has_outgoing_connection(dir, host)) return 0; - dir_debug("Connecting to %s:%u", - net_ip2addr(&host->ip), host->port); + if (director_debug) { + string_t *str = t_str_new(128); + + str_printfa(str, "Connecting to %s:%u (as %s", + net_ip2addr(&host->ip), host->port, + net_ip2addr(&dir->self_ip)); + if (host->last_network_failure > 0) { + str_printfa(str, ", last network failure %ds ago", + (int)(ioloop_time - host->last_network_failure)); + } + if (host->last_protocol_failure > 0) { + str_printfa(str, ", last protocol failure %ds ago", + (int)(ioloop_time - host->last_protocol_failure)); + } + dir_debug("%s", str_c(str)); + } port = dir->test_port != 0 ? dir->test_port : host->port; fd = net_connect_ip(&host->ip, port, &dir->self_ip); if (fd == -1) { From dovecot at dovecot.org Wed May 13 01:13:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 01:13:34 +0000 Subject: dovecot-2.2: director: If we disconnect a director, pass the rea... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/27baf04f2b18 changeset: 18662:27baf04f2b18 user: Timo Sirainen date: Wed May 13 04:10:18 2015 +0300 description: director: If we disconnect a director, pass the reason all the way to deinit's debug logging. diffstat: src/director/director-connection.c | 42 +++++++++++++++++++++++-------------- 1 files changed, 26 insertions(+), 16 deletions(-) diffs (132 lines): diff -r f43adca71e15 -r 27baf04f2b18 src/director/director-connection.c --- a/src/director/director-connection.c Wed May 13 04:07:46 2015 +0300 +++ b/src/director/director-connection.c Wed May 13 04:10:18 2015 +0300 @@ -131,11 +131,13 @@ unsigned int done_pending:1; }; -static void director_connection_disconnected(struct director_connection **conn); +static void director_connection_disconnected(struct director_connection **conn, + const char *reason); static void director_connection_reconnect(struct director_connection **conn, const char *reason); static void -director_connection_log_disconnect(struct director_connection *conn, int err); +director_connection_log_disconnect(struct director_connection *conn, int err, + const char *errstr); static int director_connection_send_done(struct director_connection *conn); static void ATTR_FORMAT(2, 3) @@ -169,7 +171,7 @@ i_error("director(%s): Handshaking DONE timed out (%u secs)", conn->name, secs); } - director_connection_disconnected(&conn); + director_connection_disconnected(&conn, "Handshake timeout"); } static void @@ -187,7 +189,7 @@ static void director_connection_wait_timeout(struct director_connection *conn) { - director_connection_log_disconnect(conn, ETIMEDOUT); + director_connection_log_disconnect(conn, ETIMEDOUT, ""); director_connection_deinit(&conn, "Timeout waiting for disconnect after CONNECT"); } @@ -1453,7 +1455,8 @@ } static void -director_connection_log_disconnect(struct director_connection *conn, int err) +director_connection_log_disconnect(struct director_connection *conn, int err, + const char *errstr) { unsigned int secs = ioloop_time - conn->created; string_t *str = t_str_new(128); @@ -1471,7 +1474,10 @@ str_append(str, "Connection closed"); if (err != 0 && err != EPIPE) { errno = err; - str_printfa(str, ": %m"); + if (errstr[0] == '\0') + str_printfa(str, ": %m"); + else + str_printfa(str, ": %s", errstr); } str_printfa(str, " (connected %u secs, " @@ -1496,8 +1502,9 @@ return; case -1: /* disconnected */ - director_connection_log_disconnect(conn, conn->input->stream_errno); - director_connection_disconnected(&conn); + director_connection_log_disconnect(conn, conn->input->stream_errno, + i_stream_get_error(conn->input)); + director_connection_disconnected(&conn, i_stream_get_error(conn->input)); return; case -2: /* buffer full */ @@ -1637,10 +1644,12 @@ o_stream_cork(conn->output); ret = director_connection_send_users(conn); o_stream_uncork(conn->output); - if (ret < 0) - director_connection_disconnected(&conn); - else + if (ret < 0) { + director_connection_disconnected(&conn, + o_stream_get_error(conn->output)); + } else { o_stream_set_flush_pending(conn->output, TRUE); + } return ret; } return o_stream_flush(conn->output); @@ -1698,7 +1707,7 @@ if ((err = net_geterror(conn->fd)) != 0) { i_error("director(%s): connect() failed: %s", conn->name, strerror(err)); - director_connection_disconnected(&conn); + director_connection_disconnected(&conn, strerror(err)); return; } conn->connected = TRUE; @@ -1810,7 +1819,8 @@ } } -void director_connection_disconnected(struct director_connection **_conn) +void director_connection_disconnected(struct director_connection **_conn, + const char *reason) { struct director_connection *conn = *_conn; struct director *dir = conn->dir; @@ -1822,7 +1832,7 @@ conn->host->last_network_failure = ioloop_time; } - director_connection_deinit(_conn, ""); + director_connection_deinit(_conn, reason); if (dir->right == NULL) director_connect(dir); } @@ -1868,14 +1878,14 @@ director_connection_ping_idle_timeout(struct director_connection *conn) { i_error("director(%s): Ping timed out, disconnecting", conn->name); - director_connection_disconnected(&conn); + director_connection_disconnected(&conn, "Ping timeout"); } static void director_connection_pong_timeout(struct director_connection *conn) { i_error("director(%s): PONG reply not received although other " "input keeps coming, disconnecting", conn->name); - director_connection_disconnected(&conn); + director_connection_disconnected(&conn, "Pong timeout"); } void director_connection_ping(struct director_connection *conn) From dovecot at dovecot.org Wed May 13 01:13:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 01:13:35 +0000 Subject: dovecot-2.2: director: Fixed crash if director sent invalid data... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1c268a7cc74a changeset: 18663:1c268a7cc74a user: Timo Sirainen date: Wed May 13 04:10:53 2015 +0300 description: director: Fixed crash if director sent invalid data too early. diffstat: src/director/director-connection.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 27baf04f2b18 -r 1c268a7cc74a src/director/director-connection.c --- a/src/director/director-connection.c Wed May 13 04:10:18 2015 +0300 +++ b/src/director/director-connection.c Wed May 13 04:10:53 2015 +0300 @@ -150,7 +150,8 @@ conn->cur_cmd, t_strdup_vprintf(fmt, args), conn->cur_line); va_end(args); - conn->host->last_protocol_failure = ioloop_time; + if (conn->host != NULL) + conn->host->last_protocol_failure = ioloop_time; } static void From dovecot at dovecot.org Wed May 13 01:13:45 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 01:13:45 +0000 Subject: dovecot-2.2: director: Include useful statistics in process title. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/502755a1af5f changeset: 18664:502755a1af5f user: Timo Sirainen date: Wed May 13 04:11:34 2015 +0300 description: director: Include useful statistics in process title. diffstat: src/director/director-connection.c | 7 +++++++ src/director/director-request.c | 2 ++ src/director/director.h | 3 +++ src/director/main.c | 35 +++++++++++++++++++++++++++++++++-- src/director/user-directory.c | 5 +++++ src/director/user-directory.h | 2 ++ 6 files changed, 52 insertions(+), 2 deletions(-) diffs (165 lines): diff -r 1c268a7cc74a -r 502755a1af5f src/director/director-connection.c --- a/src/director/director-connection.c Wed May 13 04:10:53 2015 +0300 +++ b/src/director/director-connection.c Wed May 13 04:11:34 2015 +0300 @@ -1496,6 +1496,7 @@ { struct director *dir = conn->dir; char *line; + uoff_t prev_offset; bool ret; switch (i_stream_read(conn->input)) { @@ -1524,7 +1525,11 @@ } director_sync_freeze(dir); + prev_offset = conn->input->v_offset; while ((line = i_stream_next_line(conn->input)) != NULL) { + dir->ring_traffic_input += conn->input->v_offset - prev_offset; + prev_offset = conn->input->v_offset; + T_BEGIN { ret = director_connection_handle_line(conn, line); } T_END; @@ -1872,6 +1877,8 @@ "disconnecting", conn->name); } o_stream_close(conn->output); + } else { + conn->dir->ring_traffic_output += len; } } diff -r 1c268a7cc74a -r 502755a1af5f src/director/director-request.c --- a/src/director/director-request.c Wed May 13 04:10:53 2015 +0300 +++ b/src/director/director-request.c Wed May 13 04:11:34 2015 +0300 @@ -128,6 +128,8 @@ unsigned int username_hash = user_directory_get_username_hash(dir->users, username); + dir->num_requests++; + request = i_new(struct director_request, 1); request->dir = dir; request->create_time = ioloop_time; diff -r 1c268a7cc74a -r 502755a1af5f src/director/director.h --- a/src/director/director.h Wed May 13 04:10:53 2015 +0300 +++ b/src/director/director.h Wed May 13 04:11:34 2015 +0300 @@ -79,6 +79,9 @@ time_t ring_first_alone; + uint64_t num_requests; + uint64_t ring_traffic_input, ring_traffic_output; + /* director ring handshaking is complete. director can start serving clients. */ unsigned int ring_handshaked:1; diff -r 1c268a7cc74a -r 502755a1af5f src/director/main.c --- a/src/director/main.c Wed May 13 04:10:53 2015 +0300 +++ b/src/director/main.c Wed May 13 04:11:34 2015 +0300 @@ -3,7 +3,9 @@ #include "lib.h" #include "ioloop.h" #include "array.h" +#include "str.h" #include "restrict-access.h" +#include "process-title.h" #include "master-interface.h" #include "master-service.h" #include "master-service-settings.h" @@ -11,6 +13,7 @@ #include "doveadm-connection.h" #include "login-connection.h" #include "notify-connection.h" +#include "user-directory.h" #include "director.h" #include "director-host.h" #include "director-connection.h" @@ -25,6 +28,28 @@ static struct director *director; static struct notify_connection *notify_conn; +static struct timeout *to_proctitle_refresh; + +static void director_refresh_proctitle_timeout(void *context ATTR_UNUSED) +{ + static uint64_t prev_requests = 0, prev_input = 0, prev_output; + string_t *str; + + str = t_str_new(64); + str_printfa(str, "[%u users", user_directory_count(director->users)); + str_printfa(str, ", %lu req/s", + (unsigned long)(director->num_requests - prev_requests)); + str_printfa(str, ", %llu+%llu kB/s", + (unsigned long long)(director->ring_traffic_input - prev_input)/1024, + (unsigned long long)(director->ring_traffic_output - prev_output)/1024); + str_append_c(str, ']'); + + prev_requests = director->num_requests; + prev_input = director->ring_traffic_input; + prev_output = director->ring_traffic_output; + + process_title_set(str_c(str)); +} static int director_client_connected(int fd, const struct ip_addr *ip) { @@ -145,6 +170,11 @@ struct ip_addr listen_ip; unsigned int listen_port; + if (master_service_settings_get(master_service)->verbose_proctitle) { + to_proctitle_refresh = + timeout_add(1000, director_refresh_proctitle_timeout, + (void *)NULL); + } set = master_service_settings_get_others(master_service)[0]; listen_port = find_inet_listener_port(&listen_ip, set); @@ -168,6 +198,8 @@ static void main_deinit(void) { + if (to_proctitle_refresh != NULL) + timeout_remove(&to_proctitle_refresh); if (notify_conn != NULL) notify_connection_deinit(¬ify_conn); director_deinit(&director); @@ -183,8 +215,7 @@ NULL }; const enum master_service_flags service_flags = - MASTER_SERVICE_FLAG_NO_IDLE_DIE | - MASTER_SERVICE_FLAG_UPDATE_PROCTITLE; + MASTER_SERVICE_FLAG_NO_IDLE_DIE; unsigned int test_port = 0; const char *error; bool debug = FALSE; diff -r 1c268a7cc74a -r 502755a1af5f src/director/user-directory.c --- a/src/director/user-directory.c Wed May 13 04:10:53 2015 +0300 +++ b/src/director/user-directory.c Wed May 13 04:11:34 2015 +0300 @@ -91,6 +91,11 @@ user_free(dir, dir->head); } +unsigned int user_directory_count(struct user_directory *dir) +{ + return hash_table_count(dir->hash); +} + struct user *user_directory_lookup(struct user_directory *dir, unsigned int username_hash) { diff -r 1c268a7cc74a -r 502755a1af5f src/director/user-directory.h --- a/src/director/user-directory.h Wed May 13 04:10:53 2015 +0300 +++ b/src/director/user-directory.h Wed May 13 04:11:34 2015 +0300 @@ -51,6 +51,8 @@ user_directory_init(unsigned int timeout_secs, const char *username_hash_fmt); void user_directory_deinit(struct user_directory **dir); +/* Returns the number of users currently in directory. */ +unsigned int user_directory_count(struct user_directory *dir); /* Look up username from directory. Returns NULL if not found. */ struct user *user_directory_lookup(struct user_directory *dir, unsigned int username_hash); From dovecot at dovecot.org Wed May 13 02:27:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 02:27:35 +0000 Subject: dovecot-2.2: director: Don't send DIRECTOR command infinitely in... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b7aed6290e7e changeset: 18665:b7aed6290e7e user: Timo Sirainen date: Wed May 13 05:22:22 2015 +0300 description: director: Don't send DIRECTOR command infinitely in loop if that director got disconnected. diffstat: src/director/director-connection.c | 14 ++++++++++++-- 1 files changed, 12 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 502755a1af5f -r b7aed6290e7e src/director/director-connection.c --- a/src/director/director-connection.c Wed May 13 04:11:34 2015 +0300 +++ b/src/director/director-connection.c Wed May 13 05:22:22 2015 +0300 @@ -670,8 +670,18 @@ itself. some hosts may see this twice, but that's the only way to guarantee that it gets seen by everyone. reseting the host multiple times may cause us to handle its commands multiple times, but the - commands can handle that. */ - director_notify_ring_added(host, director_connection_get_host(conn)); + commands can handle that. however, we need to also handle a + situation where the added director never comes back - we don't want + to send the director information in a loop forever. */ + if (conn->dir->right != NULL && + director_host_cmp_to_self(host, conn->dir->right->host, + conn->dir->self_host) > 0) { + dir_debug("Received DIRECTOR update for a host where we should be connected to. " + "Not forwarding it since it's probably crashed."); + } else { + director_notify_ring_added(host, + director_connection_get_host(conn)); + } return TRUE; } From dovecot at dovecot.org Wed May 13 02:27:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 02:27:36 +0000 Subject: dovecot-2.2: director: Added a new DIRECTOR-LOOKUP command that ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e178413a905d changeset: 18666:e178413a905d user: Timo Sirainen date: Wed May 13 05:25:31 2015 +0300 description: director: Added a new DIRECTOR-LOOKUP command that auth connections can use. The parameters are the same as what auth lookup would receive from auth process. So the idea is that a proxy could do an auth lookup, then forward the reply to director, which would return back the updated reply with the host field added. diffstat: src/director/auth-connection.c | 8 +--- src/director/auth-connection.h | 5 +- src/director/login-connection.c | 68 ++++++++++++++++++++++++++++++++++------ 3 files changed, 63 insertions(+), 18 deletions(-) diffs (161 lines): diff -r b7aed6290e7e -r e178413a905d src/director/auth-connection.c --- a/src/director/auth-connection.c Wed May 13 05:22:22 2015 +0300 +++ b/src/director/auth-connection.c Wed May 13 05:25:31 2015 +0300 @@ -121,12 +121,10 @@ conn->callback(NULL, conn->context); } -void auth_connection_send(struct auth_connection *conn, - const void *data, size_t size) +struct ostream *auth_connection_send(struct auth_connection *conn) { - i_assert(conn->fd != -1); - - o_stream_nsend(conn->output, data, size); + i_assert(conn->output != NULL); + return conn->output; } void auth_connections_deinit(void) diff -r b7aed6290e7e -r e178413a905d src/director/auth-connection.h --- a/src/director/auth-connection.h Wed May 13 05:22:22 2015 +0300 +++ b/src/director/auth-connection.h Wed May 13 05:25:31 2015 +0300 @@ -13,9 +13,8 @@ /* Start connecting. Returns 0 if ok, -1 if connect failed. */ int auth_connection_connect(struct auth_connection *conn); -/* Send data to auth connection. */ -void auth_connection_send(struct auth_connection *conn, - const void *data, size_t size); +/* Get auth connection's output stream. */ +struct ostream *auth_connection_send(struct auth_connection *conn); void auth_connections_deinit(void); diff -r b7aed6290e7e -r e178413a905d src/director/login-connection.c --- a/src/director/login-connection.c Wed May 13 05:22:22 2015 +0300 +++ b/src/director/login-connection.c Wed May 13 05:25:31 2015 +0300 @@ -3,8 +3,10 @@ #include "lib.h" #include "ioloop.h" #include "net.h" +#include "istream.h" #include "ostream.h" #include "llist.h" +#include "str.h" #include "master-service.h" #include "director.h" #include "director-request.h" @@ -20,12 +22,14 @@ int fd; struct io *io; + struct istream *input; struct ostream *output; struct auth_connection *auth; struct director *dir; unsigned int destroyed:1; unsigned int userdb:1; + unsigned int input_newline:1; }; struct login_host_request { @@ -40,25 +44,66 @@ static struct login_connection *login_connections; +static void auth_input_line(const char *line, void *context); static void login_connection_unref(struct login_connection **_conn); +static void +login_connection_director_lookup(struct login_connection *conn, + const unsigned char *data, size_t size) +{ + T_BEGIN { + string_t *line = t_str_new(128); + str_append(line, "OK\t"); + str_append_n(line, data, size); + auth_input_line(str_c(line), conn); + } T_END; +} + static void login_connection_input(struct login_connection *conn) { - unsigned char buf[4096]; - ssize_t ret; + const unsigned char *data, *p; + size_t size; + struct ostream *auth_output; - ret = read(conn->fd, buf, sizeof(buf)); - if (ret <= 0) { - if (ret < 0) { - if (errno == EAGAIN) - return; - if (errno != ECONNRESET) - i_error("read(login connection) failed: %m"); + auth_output = auth_connection_send(conn->auth); + switch (i_stream_read(conn->input)) { + case -2: + data = i_stream_get_data(conn->input, &size); + o_stream_nsend(auth_output, data, size); + i_stream_skip(conn->input, size); + conn->input_newline = FALSE; + return; + case -1: + if (conn->input->stream_errno != 0 && + conn->input->stream_errno != ECONNRESET) { + i_error("read(login connection) failed: %s", + i_stream_get_error(conn->input)); } login_connection_deinit(&conn); return; + case 0: + return; + default: + break; } - auth_connection_send(conn->auth, buf, ret); + + o_stream_cork(auth_output); + data = i_stream_get_data(conn->input, &size); + while ((p = memchr(data, '\n', size)) != NULL) { + size_t linelen = p-data; + + if (!conn->input_newline || linelen <= 16 || + memcmp(data, "DIRECTOR-LOOKUP\t", 16) != 0) { + /* forward data to auth process */ + o_stream_nsend(auth_output, data, linelen+1); + conn->input_newline = TRUE; + } else { + login_connection_director_lookup(conn, data+16, linelen-16); + } + i_stream_skip(conn->input, linelen+1); + data = i_stream_get_data(conn->input, &size); + } + o_stream_uncork(auth_output); } static void @@ -215,10 +260,12 @@ conn->fd = fd; conn->auth = auth; conn->dir = dir; + conn->input = i_stream_create_fd(conn->fd, IO_BLOCK_SIZE, FALSE); conn->output = o_stream_create_fd(conn->fd, (size_t)-1, FALSE); o_stream_set_no_error_handling(conn->output, TRUE); conn->io = io_add(conn->fd, IO_READ, login_connection_input, conn); conn->userdb = userdb; + conn->input_newline = TRUE; auth_connection_set_callback(conn->auth, auth_input_line, conn); DLLIST_PREPEND(&login_connections, conn); @@ -237,6 +284,7 @@ DLLIST_REMOVE(&login_connections, conn); io_remove(&conn->io); + i_stream_destroy(&conn->input); o_stream_destroy(&conn->output); if (close(conn->fd) < 0) i_error("close(login connection) failed: %m"); From dovecot at dovecot.org Wed May 13 09:26:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 09:26:11 +0000 Subject: dovecot-2.2: lib-sql: sqlite and cassandra libs were mixed up in... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/69dcc2c8cd9d changeset: 18667:69dcc2c8cd9d user: Timo Sirainen date: Wed May 13 05:34:16 2015 +0300 description: lib-sql: sqlite and cassandra libs were mixed up in the Makefile diffstat: src/lib-sql/Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r e178413a905d -r 69dcc2c8cd9d src/lib-sql/Makefile.am --- a/src/lib-sql/Makefile.am Wed May 13 05:25:31 2015 +0300 +++ b/src/lib-sql/Makefile.am Wed May 13 05:34:16 2015 +0300 @@ -19,7 +19,7 @@ SQL_DRIVER_PLUGINS += sqlite endif if BUILD_CASSANDRA -SQLITE_LIB = libdriver_cassandra.la +CASSANDRA_LIB = libdriver_cassandra.la SQL_DRIVER_PLUGINS += cassandra endif From dovecot at dovecot.org Wed May 13 09:26:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 09:26:11 +0000 Subject: dovecot-2.2: Added quota-clone plugin. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f5749f22276b changeset: 18668:f5749f22276b user: Timo Sirainen date: Wed May 13 12:24:02 2015 +0300 description: Added quota-clone plugin. diffstat: NEWS | 1 + configure.ac | 1 + src/plugins/Makefile.am | 1 + src/plugins/quota-clone/Makefile.am | 19 ++ src/plugins/quota-clone/quota-clone-plugin.c | 197 +++++++++++++++++++++++++++ src/plugins/quota-clone/quota-clone-plugin.h | 7 + 6 files changed, 226 insertions(+), 0 deletions(-) diffs (268 lines): diff -r 69dcc2c8cd9d -r f5749f22276b NEWS --- a/NEWS Wed May 13 05:34:16 2015 +0300 +++ b/NEWS Wed May 13 12:24:02 2015 +0300 @@ -34,6 +34,7 @@ -A parameter reading the list of users from userdb lookup. + Implemented initial Cassandra CQL support as lib-sql backend. It's only usable as dict backend currently. + + Added quota-clone plugin to copy current quota usage to a dict. - auth: If auth_master_user_separator was set, auth process could be crashed by trying to log in with empty master username. - imap-login, pop3-login: Fixed crash on handshake failures with new diff -r 69dcc2c8cd9d -r f5749f22276b configure.ac --- a/configure.ac Wed May 13 05:34:16 2015 +0300 +++ b/configure.ac Wed May 13 12:24:02 2015 +0300 @@ -2991,6 +2991,7 @@ src/plugins/notify/Makefile src/plugins/pop3-migration/Makefile src/plugins/quota/Makefile +src/plugins/quota-clone/Makefile src/plugins/imap-quota/Makefile src/plugins/replication/Makefile src/plugins/snarf/Makefile diff -r 69dcc2c8cd9d -r f5749f22276b src/plugins/Makefile.am --- a/src/plugins/Makefile.am Wed May 13 05:34:16 2015 +0300 +++ b/src/plugins/Makefile.am Wed May 13 12:24:02 2015 +0300 @@ -25,6 +25,7 @@ mail-log \ mailbox-alias \ quota \ + quota-clone \ imap-quota \ pop3-migration \ replication \ diff -r 69dcc2c8cd9d -r f5749f22276b src/plugins/quota-clone/Makefile.am --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/quota-clone/Makefile.am Wed May 13 12:24:02 2015 +0300 @@ -0,0 +1,19 @@ +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-dict \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/plugins/quota + +NOPLUGIN_LDFLAGS = +lib20_quota_clone_plugin_la_LDFLAGS = -module -avoid-version + +module_LTLIBRARIES = \ + lib20_quota_clone_plugin.la + +lib20_quota_clone_plugin_la_SOURCES = \ + quota-clone-plugin.c + +noinst_HEADERS = \ + quota-clone-plugin.h diff -r 69dcc2c8cd9d -r f5749f22276b src/plugins/quota-clone/quota-clone-plugin.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/quota-clone/quota-clone-plugin.c Wed May 13 12:24:02 2015 +0300 @@ -0,0 +1,197 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "module-context.h" +#include "dict.h" +#include "mail-storage-private.h" +#include "quota.h" +#include "quota-clone-plugin.h" + +#define DICT_QUOTA_CLONE_PATH DICT_PATH_PRIVATE"quota/" +#define DICT_QUOTA_CLONE_BYTES_PATH DICT_QUOTA_CLONE_PATH"storage" +#define DICT_QUOTA_CLONE_COUNT_PATH DICT_QUOTA_CLONE_PATH"messages" + +#define QUOTA_CLONE_USER_CONTEXT(obj) \ + MODULE_CONTEXT(obj, quota_clone_user_module) +#define QUOTA_CLONE_CONTEXT(obj) \ + MODULE_CONTEXT(obj, quota_clone_storage_module) + +static MODULE_CONTEXT_DEFINE_INIT(quota_clone_user_module, + &mail_user_module_register); +static MODULE_CONTEXT_DEFINE_INIT(quota_clone_storage_module, + &mail_storage_module_register); + +struct quota_clone_user { + union mail_user_module_context module_ctx; + struct dict *dict; +}; + +struct quota_clone_mailbox { + union mailbox_module_context module_ctx; + bool quota_changed; +}; + +static void quota_clone_flush(struct mailbox *box) +{ + struct quota_clone_mailbox *qbox = QUOTA_CLONE_CONTEXT(box); + struct quota_clone_user *quser = + QUOTA_CLONE_USER_CONTEXT(box->storage->user); + struct dict_transaction_context *trans; + struct quota_root_iter *iter; + struct quota_root *root; + uint64_t value, limit; + int ret; + + /* we'll clone the first quota root */ + iter = quota_root_iter_init(box); + root = quota_root_iter_next(iter); + if (root == NULL) { + /* no quota roots defined for this mailbox - ignore */ + qbox->quota_changed = FALSE; + return; + } + quota_root_iter_deinit(&iter); + + trans = dict_transaction_begin(quser->dict); + /* update bytes */ + ret = quota_get_resource(root, "", QUOTA_NAME_STORAGE_BYTES, + &value, &limit); + if (ret < 0) + i_error("quota_clone_plugin: Failed to lookup current quota bytes"); + else { + dict_set(trans, DICT_QUOTA_CLONE_BYTES_PATH, + t_strdup_printf("%llu", (unsigned long long)value)); + } + /* update messages */ + ret = quota_get_resource(root, "", QUOTA_NAME_MESSAGES, + &value, &limit); + if (ret < 0) + i_error("quota_clone_plugin: Failed to lookup current quota count"); + else { + dict_set(trans, DICT_QUOTA_CLONE_COUNT_PATH, + t_strdup_printf("%llu", (unsigned long long)value)); + } + if (dict_transaction_commit(&trans) < 0) + i_error("quota_clone_plugin: Failed to commit dict update"); + else + qbox->quota_changed = FALSE; +} + +static int quota_clone_save_finish(struct mail_save_context *ctx) +{ + struct quota_clone_mailbox *qbox = + QUOTA_CLONE_CONTEXT(ctx->transaction->box); + + qbox->quota_changed = TRUE; + return qbox->module_ctx.super.save_finish(ctx); +} + +static int +quota_clone_copy(struct mail_save_context *ctx, struct mail *mail) +{ + struct quota_clone_mailbox *qbox = + QUOTA_CLONE_CONTEXT(ctx->transaction->box); + + qbox->quota_changed = TRUE; + return qbox->module_ctx.super.copy(ctx, mail); +} + +static void +quota_clone_mailbox_sync_notify(struct mailbox *box, uint32_t uid, + enum mailbox_sync_type sync_type) +{ + struct quota_clone_mailbox *qbox = QUOTA_CLONE_CONTEXT(box); + + if (qbox->module_ctx.super.sync_notify != NULL) + qbox->module_ctx.super.sync_notify(box, uid, sync_type); + + if (sync_type == MAILBOX_SYNC_TYPE_EXPUNGE) + qbox->quota_changed = TRUE; +} + +static void quota_clone_mailbox_close(struct mailbox *box) +{ + struct quota_clone_mailbox *qbox = QUOTA_CLONE_CONTEXT(box); + + qbox->module_ctx.super.close(box); + if (qbox->quota_changed) + quota_clone_flush(box); +} + +static void quota_clone_mailbox_allocated(struct mailbox *box) +{ + struct quota_clone_user *quser = + QUOTA_CLONE_USER_CONTEXT(box->storage->user); + struct mailbox_vfuncs *v = box->vlast; + struct quota_clone_mailbox *qbox; + + if (quser == NULL) + return; + + qbox = p_new(box->pool, struct quota_clone_mailbox, 1); + qbox->module_ctx.super = *v; + box->vlast = &qbox->module_ctx.super; + + v->save_finish = quota_clone_save_finish; + v->copy = quota_clone_copy; + v->sync_notify = quota_clone_mailbox_sync_notify; + v->close = quota_clone_mailbox_close; + MODULE_CONTEXT_SET(box, quota_clone_storage_module, qbox); +} + +static void quota_clone_mail_user_deinit(struct mail_user *user) +{ + struct quota_clone_user *quser = QUOTA_CLONE_USER_CONTEXT(user); + + dict_deinit(&quser->dict); + quser->module_ctx.super.deinit(user); +} + +static void quota_clone_mail_user_created(struct mail_user *user) +{ + struct quota_clone_user *quser; + struct mail_user_vfuncs *v = user->vlast; + struct dict_settings dict_set; + struct dict *dict; + const char *uri, *error; + + uri = mail_user_plugin_getenv(user, "quota_clone_dict"); + if (uri == NULL || uri[0] == '\0') { + i_error("The quota_clone_dict setting is missing from configuration"); + return; + } + + memset(&dict_set, 0, sizeof(dict_set)); + dict_set.username = user->username; + dict_set.base_dir = user->set->base_dir; + (void)mail_user_get_home(user, &dict_set.home_dir); + if (dict_init_full(uri, &dict_set, &dict, &error) < 0) { + i_error("quota_clone_dict: Failed to initialize '%s': %s", + uri, error); + return; + } + + quser = p_new(user->pool, struct quota_clone_user, 1); + quser->module_ctx.super = *v; + user->vlast = &quser->module_ctx.super; + v->deinit = quota_clone_mail_user_deinit; + quser->dict = dict; + MODULE_CONTEXT_SET(user, quota_clone_user_module, quser); +} + +static struct mail_storage_hooks quota_clone_mail_storage_hooks = { + .mailbox_allocated = quota_clone_mailbox_allocated, + .mail_user_created = quota_clone_mail_user_created +}; + +void quota_clone_plugin_init(struct module *module ATTR_UNUSED) +{ + mail_storage_hooks_add(module, "a_clone_mail_storage_hooks); +} + +void quota_clone_plugin_deinit(void) +{ + mail_storage_hooks_remove("a_clone_mail_storage_hooks); +} + +const char *quota_clone_plugin_dependencies[] = { "quota", NULL }; diff -r 69dcc2c8cd9d -r f5749f22276b src/plugins/quota-clone/quota-clone-plugin.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/quota-clone/quota-clone-plugin.h Wed May 13 12:24:02 2015 +0300 @@ -0,0 +1,7 @@ +#ifndef QUOTA_CLONE_PLUGIN_H +#define QUOTA_CLONE_PLUGIN_H + +void quota_clone_plugin_init(struct module *module); +void quota_clone_plugin_deinit(void); + +#endif From dovecot at dovecot.org Wed May 13 12:35:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 12:35:28 +0000 Subject: dovecot-2.2: director: Reverted previous e178413a905d commit aft... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/67fdd6f962f5 changeset: 18669:67fdd6f962f5 user: Timo Sirainen date: Wed May 13 13:15:57 2015 +0300 description: director: Reverted previous e178413a905d commit after all - do it a bit differently. diffstat: src/director/auth-connection.c | 8 +++- src/director/auth-connection.h | 5 +- src/director/login-connection.c | 68 ++++++---------------------------------- 3 files changed, 18 insertions(+), 63 deletions(-) diffs (161 lines): diff -r f5749f22276b -r 67fdd6f962f5 src/director/auth-connection.c --- a/src/director/auth-connection.c Wed May 13 12:24:02 2015 +0300 +++ b/src/director/auth-connection.c Wed May 13 13:15:57 2015 +0300 @@ -121,10 +121,12 @@ conn->callback(NULL, conn->context); } -struct ostream *auth_connection_send(struct auth_connection *conn) +void auth_connection_send(struct auth_connection *conn, + const void *data, size_t size) { - i_assert(conn->output != NULL); - return conn->output; + i_assert(conn->fd != -1); + + o_stream_nsend(conn->output, data, size); } void auth_connections_deinit(void) diff -r f5749f22276b -r 67fdd6f962f5 src/director/auth-connection.h --- a/src/director/auth-connection.h Wed May 13 12:24:02 2015 +0300 +++ b/src/director/auth-connection.h Wed May 13 13:15:57 2015 +0300 @@ -13,8 +13,9 @@ /* Start connecting. Returns 0 if ok, -1 if connect failed. */ int auth_connection_connect(struct auth_connection *conn); -/* Get auth connection's output stream. */ -struct ostream *auth_connection_send(struct auth_connection *conn); +/* Send data to auth connection. */ +void auth_connection_send(struct auth_connection *conn, + const void *data, size_t size); void auth_connections_deinit(void); diff -r f5749f22276b -r 67fdd6f962f5 src/director/login-connection.c --- a/src/director/login-connection.c Wed May 13 12:24:02 2015 +0300 +++ b/src/director/login-connection.c Wed May 13 13:15:57 2015 +0300 @@ -3,10 +3,8 @@ #include "lib.h" #include "ioloop.h" #include "net.h" -#include "istream.h" #include "ostream.h" #include "llist.h" -#include "str.h" #include "master-service.h" #include "director.h" #include "director-request.h" @@ -22,14 +20,12 @@ int fd; struct io *io; - struct istream *input; struct ostream *output; struct auth_connection *auth; struct director *dir; unsigned int destroyed:1; unsigned int userdb:1; - unsigned int input_newline:1; }; struct login_host_request { @@ -44,66 +40,25 @@ static struct login_connection *login_connections; -static void auth_input_line(const char *line, void *context); static void login_connection_unref(struct login_connection **_conn); -static void -login_connection_director_lookup(struct login_connection *conn, - const unsigned char *data, size_t size) -{ - T_BEGIN { - string_t *line = t_str_new(128); - str_append(line, "OK\t"); - str_append_n(line, data, size); - auth_input_line(str_c(line), conn); - } T_END; -} - static void login_connection_input(struct login_connection *conn) { - const unsigned char *data, *p; - size_t size; - struct ostream *auth_output; + unsigned char buf[4096]; + ssize_t ret; - auth_output = auth_connection_send(conn->auth); - switch (i_stream_read(conn->input)) { - case -2: - data = i_stream_get_data(conn->input, &size); - o_stream_nsend(auth_output, data, size); - i_stream_skip(conn->input, size); - conn->input_newline = FALSE; - return; - case -1: - if (conn->input->stream_errno != 0 && - conn->input->stream_errno != ECONNRESET) { - i_error("read(login connection) failed: %s", - i_stream_get_error(conn->input)); + ret = read(conn->fd, buf, sizeof(buf)); + if (ret <= 0) { + if (ret < 0) { + if (errno == EAGAIN) + return; + if (errno != ECONNRESET) + i_error("read(login connection) failed: %m"); } login_connection_deinit(&conn); return; - case 0: - return; - default: - break; } - - o_stream_cork(auth_output); - data = i_stream_get_data(conn->input, &size); - while ((p = memchr(data, '\n', size)) != NULL) { - size_t linelen = p-data; - - if (!conn->input_newline || linelen <= 16 || - memcmp(data, "DIRECTOR-LOOKUP\t", 16) != 0) { - /* forward data to auth process */ - o_stream_nsend(auth_output, data, linelen+1); - conn->input_newline = TRUE; - } else { - login_connection_director_lookup(conn, data+16, linelen-16); - } - i_stream_skip(conn->input, linelen+1); - data = i_stream_get_data(conn->input, &size); - } - o_stream_uncork(auth_output); + auth_connection_send(conn->auth, buf, ret); } static void @@ -260,12 +215,10 @@ conn->fd = fd; conn->auth = auth; conn->dir = dir; - conn->input = i_stream_create_fd(conn->fd, IO_BLOCK_SIZE, FALSE); conn->output = o_stream_create_fd(conn->fd, (size_t)-1, FALSE); o_stream_set_no_error_handling(conn->output, TRUE); conn->io = io_add(conn->fd, IO_READ, login_connection_input, conn); conn->userdb = userdb; - conn->input_newline = TRUE; auth_connection_set_callback(conn->auth, auth_input_line, conn); DLLIST_PREPEND(&login_connections, conn); @@ -284,7 +237,6 @@ DLLIST_REMOVE(&login_connections, conn); io_remove(&conn->io); - i_stream_destroy(&conn->input); o_stream_destroy(&conn->output); if (close(conn->fd) < 0) i_error("close(login connection) failed: %m"); From dovecot at dovecot.org Wed May 13 12:35:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 12:35:33 +0000 Subject: dovecot-2.2: director: Minor code cleanup - allow access to auth... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d54dc360cd3c changeset: 18670:d54dc360cd3c user: Timo Sirainen date: Wed May 13 13:57:24 2015 +0300 description: director: Minor code cleanup - allow access to auth connection's ostream directly. diffstat: src/director/auth-connection.c | 8 +++----- src/director/auth-connection.h | 5 ++--- src/director/login-connection.c | 4 +++- 3 files changed, 8 insertions(+), 9 deletions(-) diffs (55 lines): diff -r 67fdd6f962f5 -r d54dc360cd3c src/director/auth-connection.c --- a/src/director/auth-connection.c Wed May 13 13:15:57 2015 +0300 +++ b/src/director/auth-connection.c Wed May 13 13:57:24 2015 +0300 @@ -121,12 +121,10 @@ conn->callback(NULL, conn->context); } -void auth_connection_send(struct auth_connection *conn, - const void *data, size_t size) +struct ostream *auth_connection_get_output(struct auth_connection *conn) { - i_assert(conn->fd != -1); - - o_stream_nsend(conn->output, data, size); + i_assert(conn->output != NULL); + return conn->output; } void auth_connections_deinit(void) diff -r 67fdd6f962f5 -r d54dc360cd3c src/director/auth-connection.h --- a/src/director/auth-connection.h Wed May 13 13:15:57 2015 +0300 +++ b/src/director/auth-connection.h Wed May 13 13:57:24 2015 +0300 @@ -13,9 +13,8 @@ /* Start connecting. Returns 0 if ok, -1 if connect failed. */ int auth_connection_connect(struct auth_connection *conn); -/* Send data to auth connection. */ -void auth_connection_send(struct auth_connection *conn, - const void *data, size_t size); +/* Get auth connection's output stream. */ +struct ostream *auth_connection_get_output(struct auth_connection *conn); void auth_connections_deinit(void); diff -r 67fdd6f962f5 -r d54dc360cd3c src/director/login-connection.c --- a/src/director/login-connection.c Wed May 13 13:15:57 2015 +0300 +++ b/src/director/login-connection.c Wed May 13 13:57:24 2015 +0300 @@ -44,6 +44,7 @@ static void login_connection_input(struct login_connection *conn) { + struct ostream *output; unsigned char buf[4096]; ssize_t ret; @@ -58,7 +59,8 @@ login_connection_deinit(&conn); return; } - auth_connection_send(conn->auth, buf, ret); + output = auth_connection_get_output(conn->auth); + o_stream_nsend(output, buf, ret); } static void From dovecot at dovecot.org Wed May 13 12:35:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 12:35:33 +0000 Subject: dovecot-2.2: lib-master: Added master_service_get_socket_name() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5614bc437643 changeset: 18671:5614bc437643 user: Timo Sirainen date: Wed May 13 15:32:54 2015 +0300 description: lib-master: Added master_service_get_socket_name() diffstat: src/lib-master/master-service.c | 12 ++++++++++++ src/lib-master/master-service.h | 3 +++ 2 files changed, 15 insertions(+), 0 deletions(-) diffs (35 lines): diff -r d54dc360cd3c -r 5614bc437643 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Wed May 13 13:57:24 2015 +0300 +++ b/src/lib-master/master-service.c Wed May 13 15:32:54 2015 +0300 @@ -541,6 +541,18 @@ return service->socket_count; } +const char *master_service_get_socket_name(struct master_service *service, + int listen_fd) +{ + unsigned int i; + + i_assert(listen_fd >= MASTER_LISTEN_FD_FIRST); + + i = listen_fd - MASTER_LISTEN_FD_FIRST; + return i < service->listener_names_count ? + service->listener_names[i] : ""; +} + void master_service_set_avail_overflow_callback(struct master_service *service, void (*callback)(void)) { diff -r d54dc360cd3c -r 5614bc437643 src/lib-master/master-service.h --- a/src/lib-master/master-service.h Wed May 13 13:57:24 2015 +0300 +++ b/src/lib-master/master-service.h Wed May 13 15:32:54 2015 +0300 @@ -125,6 +125,9 @@ unsigned int master_service_get_service_count(struct master_service *service); /* Return the number of listener sockets. */ unsigned int master_service_get_socket_count(struct master_service *service); +/* Returns the name of the listener socket, or "" if none is specified. */ +const char *master_service_get_socket_name(struct master_service *service, + int listen_fd); /* Returns configuration file path. */ const char *master_service_get_config_path(struct master_service *service); From dovecot at dovecot.org Wed May 13 12:35:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 12:35:39 +0000 Subject: dovecot-2.2: director: Cleanup for director socket type configur... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fea09ab164dc changeset: 18672:fea09ab164dc user: Timo Sirainen date: Wed May 13 15:33:27 2015 +0300 description: director: Cleanup for director socket type configuration. It's now possible to use any type of a socket for inet listeners by specifying the name for the listener. The available types are: auth (default), userdb, ring (= director<->director connection), admin/doveadm. This change should be backwards compatible with previous configuration. This setting also deprecates director_doveadm_port setting. diffstat: src/director/main.c | 167 ++++++++++++++++++++++++++++++++++++--------------- 1 files changed, 116 insertions(+), 51 deletions(-) diffs (219 lines): diff -r 5614bc437643 -r fea09ab164dc src/director/main.c --- a/src/director/main.c Wed May 13 15:32:54 2015 +0300 +++ b/src/director/main.c Wed May 13 15:33:27 2015 +0300 @@ -26,9 +26,18 @@ #define AUTH_SOCKET_PATH "auth-login" #define AUTH_USERDB_SOCKET_PATH "auth-userdb" +enum director_socket_type { + DIRECTOR_SOCKET_TYPE_UNKNOWN = 0, + DIRECTOR_SOCKET_TYPE_AUTH, + DIRECTOR_SOCKET_TYPE_USERDB, + DIRECTOR_SOCKET_TYPE_RING, + DIRECTOR_SOCKET_TYPE_DOVEADM +}; + static struct director *director; static struct notify_connection *notify_conn; static struct timeout *to_proctitle_refresh; +static ARRAY(enum director_socket_type) listener_socket_types; static void director_refresh_proctitle_timeout(void *context ATTR_UNUSED) { @@ -51,6 +60,84 @@ process_title_set(str_c(str)); } +static enum director_socket_type +director_socket_type_get_from_name(const char *path) +{ + const char *name, *suffix; + + name = strrchr(path, '/'); + if (name == NULL) + name = path; + else + name++; + + suffix = strrchr(name, '-'); + if (suffix == NULL) + suffix = name; + else + suffix++; + + if (strcmp(suffix, "auth") == 0) + return DIRECTOR_SOCKET_TYPE_AUTH; + else if (strcmp(suffix, "userdb") == 0) + return DIRECTOR_SOCKET_TYPE_USERDB; + else if (strcmp(suffix, "ring") == 0) + return DIRECTOR_SOCKET_TYPE_RING; + else if (strcmp(suffix, "admin") == 0 || + strcmp(suffix, "doveadm") == 0) + return DIRECTOR_SOCKET_TYPE_DOVEADM; + else + return DIRECTOR_SOCKET_TYPE_UNKNOWN; +} + +static enum director_socket_type +listener_get_socket_type_fallback(const struct director_settings *set, + int listen_fd) +{ + unsigned int local_port; + + if (net_getsockname(listen_fd, NULL, &local_port) == 0) { + /* TCP/IP connection */ + if (local_port == set->director_doveadm_port) + return DIRECTOR_SOCKET_TYPE_DOVEADM; + else + return DIRECTOR_SOCKET_TYPE_RING; + } + return DIRECTOR_SOCKET_TYPE_AUTH; +} + +static void listener_sockets_init(const struct director_settings *set, + struct ip_addr *listen_ip_r, + unsigned int *listen_port_r) +{ + const char *name; + unsigned int i, socket_count, port; + struct ip_addr ip; + enum director_socket_type type; + + *listen_port_r = 0; + + i_array_init(&listener_socket_types, 8); + socket_count = master_service_get_socket_count(master_service); + for (i = 0; i < socket_count; i++) { + int listen_fd = MASTER_LISTEN_FD_FIRST + i; + + name = master_service_get_socket_name(master_service, listen_fd); + type = director_socket_type_get_from_name(name); + if (type == DIRECTOR_SOCKET_TYPE_UNKNOWN) { + /* mainly for backwards compatibility */ + type = listener_get_socket_type_fallback(set, listen_fd); + } + if (type == DIRECTOR_SOCKET_TYPE_RING && *listen_port_r == 0 && + net_getsockname(listen_fd, &ip, &port) == 0 && port > 0) { + i_warning("listen port = %d", port); + *listen_ip_r = ip; + *listen_port_r = port; + } + array_idx_set(&listener_socket_types, listen_fd, &type); + } +} + static int director_client_connected(int fd, const struct ip_addr *ip) { struct director_host *host; @@ -70,8 +157,7 @@ { struct auth_connection *auth; const char *socket_path; - struct ip_addr ip; - unsigned int local_port, len; + const enum director_socket_type *typep; bool userdb; if (conn->fifo) { @@ -84,58 +170,36 @@ return; } - if (net_getpeername(conn->fd, &ip, NULL) == 0 && - net_getsockname(conn->fd, NULL, &local_port) == 0 && - (IPADDR_IS_V4(&ip) || IPADDR_IS_V6(&ip))) { - /* TCP/IP connection */ - if (local_port == director->set->director_doveadm_port) { + typep = array_idx(&listener_socket_types, conn->listen_fd); + switch (*typep) { + case DIRECTOR_SOCKET_TYPE_UNKNOWN: + i_unreached(); + case DIRECTOR_SOCKET_TYPE_AUTH: + case DIRECTOR_SOCKET_TYPE_USERDB: + /* a) userdb connection, probably for lmtp proxy + b) login connection + Both of them are handled exactly the same, except for which + auth socket they connect to. */ + userdb = *typep == DIRECTOR_SOCKET_TYPE_USERDB; + socket_path = userdb ? AUTH_USERDB_SOCKET_PATH : + AUTH_SOCKET_PATH; + auth = auth_connection_init(socket_path); + if (auth_connection_connect(auth) < 0) { + auth_connection_deinit(&auth); + break; + } + master_service_client_connection_accept(conn); + (void)login_connection_init(director, conn->fd, auth, userdb); + break; + case DIRECTOR_SOCKET_TYPE_RING: + if (director_client_connected(conn->fd, &conn->remote_ip) == 0) master_service_client_connection_accept(conn); - (void)doveadm_connection_init(director, conn->fd); - } else { - if (director_client_connected(conn->fd, &ip) == 0) - master_service_client_connection_accept(conn); - } - return; - } - - len = strlen(conn->name); - if (len > 6 && strcmp(conn->name + len - 6, "-admin") == 0) { - /* doveadm connection */ + break; + case DIRECTOR_SOCKET_TYPE_DOVEADM: master_service_client_connection_accept(conn); (void)doveadm_connection_init(director, conn->fd); - return; + break; } - - /* a) userdb connection, probably for lmtp proxy - b) login connection - Both of them are handled exactly the same, except for which - auth socket they connect to. */ - userdb = len > 7 && strcmp(conn->name + len - 7, "-userdb") == 0; - socket_path = userdb ? AUTH_USERDB_SOCKET_PATH : AUTH_SOCKET_PATH; - auth = auth_connection_init(socket_path); - if (auth_connection_connect(auth) == 0) { - master_service_client_connection_accept(conn); - (void)login_connection_init(director, conn->fd, auth, userdb); - } else { - auth_connection_deinit(&auth); - } -} - -static unsigned int -find_inet_listener_port(struct ip_addr *ip_r, - const struct director_settings *set) -{ - unsigned int i, socket_count, port; - - socket_count = master_service_get_socket_count(master_service); - for (i = 0; i < socket_count; i++) { - int fd = MASTER_LISTEN_FD_FIRST + i; - - if (net_getsockname(fd, ip_r, &port) == 0 && port > 0 && - port != set->director_doveadm_port) - return port; - } - return 0; } static void director_state_changed(struct director *dir) @@ -177,7 +241,7 @@ } set = master_service_settings_get_others(master_service)[0]; - listen_port = find_inet_listener_port(&listen_ip, set); + listener_sockets_init(set, &listen_ip, &listen_port); if (listen_port == 0 && *set->director_servers != '\0') { i_fatal("No inet_listeners defined for director service " "(for standalone keep director_servers empty)"); @@ -206,6 +270,7 @@ doveadm_connections_deinit(); login_connections_deinit(); auth_connections_deinit(); + array_free(&listener_socket_types); } int main(int argc, char *argv[]) From dovecot at dovecot.org Wed May 13 12:49:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 12:49:16 +0000 Subject: dovecot-2.2: lib-master: If executing doveconf, use master_servi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f8f0a782213b changeset: 18673:f8f0a782213b user: Timo Sirainen date: Wed May 13 15:47:14 2015 +0300 description: lib-master: If executing doveconf, use master_service_settings_input.service for filter Instead of hardcoding to the master_service->name, which may be different. This fixes reading protocol sieve {} settings when if managesieve was reading settings via doveconf (= executed from command line). diffstat: src/lib-master/master-service-settings.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r fea09ab164dc -r f8f0a782213b src/lib-master/master-service-settings.c --- a/src/lib-master/master-service-settings.c Wed May 13 15:33:27 2015 +0300 +++ b/src/lib-master/master-service-settings.c Wed May 13 15:47:14 2015 +0300 @@ -119,8 +119,10 @@ argv_max_count = 11 + (service->argc + 1) + 1; conf_argv = t_new(const char *, argv_max_count); conf_argv[i++] = DOVECOT_CONFIG_BIN_PATH; - conf_argv[i++] = "-f"; - conf_argv[i++] = t_strconcat("service=", service->name, NULL); + if (input->service != NULL) { + conf_argv[i++] = "-f"; + conf_argv[i++] = t_strconcat("service=", input->service, NULL); + } conf_argv[i++] = "-c"; conf_argv[i++] = service->config_path; if (input->module != NULL) { From dovecot at dovecot.org Wed May 13 13:01:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 13:01:46 +0000 Subject: dovecot-2.2: director: Removed accidentally committed debug log ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c97d10467b62 changeset: 18674:c97d10467b62 user: Timo Sirainen date: Wed May 13 15:59:48 2015 +0300 description: director: Removed accidentally committed debug log message diffstat: src/director/main.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diffs (11 lines): diff -r f8f0a782213b -r c97d10467b62 src/director/main.c --- a/src/director/main.c Wed May 13 15:47:14 2015 +0300 +++ b/src/director/main.c Wed May 13 15:59:48 2015 +0300 @@ -130,7 +130,6 @@ } if (type == DIRECTOR_SOCKET_TYPE_RING && *listen_port_r == 0 && net_getsockname(listen_fd, &ip, &port) == 0 && port > 0) { - i_warning("listen port = %d", port); *listen_ip_r = ip; *listen_port_r = port; } From dovecot at dovecot.org Wed May 13 13:24:26 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 13:24:26 +0000 Subject: dovecot-2.2: director: Added "authreply" socket type. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/52cdf321fa07 changeset: 18675:52cdf321fa07 user: Timo Sirainen date: Wed May 13 16:22:27 2015 +0300 description: director: Added "authreply" socket type. This allows defining a socket, which receives auth replies. Director then adds the "host" field to it if it's missing and returns back the original string. The idea is that eventually a director ring could be running independently from Dovecot proxies. diffstat: src/director/login-connection.c | 71 +++++++++++++++++++++++++++++++++++----- src/director/login-connection.h | 9 ++++- src/director/main.c | 12 ++++++- 3 files changed, 81 insertions(+), 11 deletions(-) diffs (208 lines): diff -r c97d10467b62 -r 52cdf321fa07 src/director/login-connection.c --- a/src/director/login-connection.c Wed May 13 15:59:48 2015 +0300 +++ b/src/director/login-connection.c Wed May 13 16:22:27 2015 +0300 @@ -3,6 +3,7 @@ #include "lib.h" #include "ioloop.h" #include "net.h" +#include "istream.h" #include "ostream.h" #include "llist.h" #include "master-service.h" @@ -13,19 +14,24 @@ #include +#define AUTHREPLY_PROTOCOL_MAJOR_VERSION 1 +#define AUTHREPLY_PROTOCOL_MINOR_VERSION 0 + struct login_connection { struct login_connection *prev, *next; int refcount; + enum login_connection_type type; int fd; struct io *io; + struct istream *input; struct ostream *output; struct auth_connection *auth; struct director *dir; + unsigned int handshaked:1; unsigned int destroyed:1; - unsigned int userdb:1; }; struct login_host_request { @@ -40,6 +46,7 @@ static struct login_connection *login_connections; +static void auth_input_line(const char *line, void *context); static void login_connection_unref(struct login_connection **_conn); static void login_connection_input(struct login_connection *conn) @@ -63,6 +70,33 @@ o_stream_nsend(output, buf, ret); } +static void login_connection_authreply_input(struct login_connection *conn) +{ + const char *line; + + while ((line = i_stream_read_next_line(conn->input)) != NULL) T_BEGIN { + if (!conn->handshaked) { + if (!version_string_verify(line, "director-authreply-client", + AUTHREPLY_PROTOCOL_MAJOR_VERSION)) { + i_error("authreply client sent invalid handshake: %s", line); + login_connection_deinit(&conn); + return; + } + conn->handshaked = TRUE; + } else { + auth_input_line(line, conn); + } + } T_END; + if (conn->input->eof) { + if (conn->input->stream_errno != 0 && + conn->input->stream_errno != ECONNRESET) { + i_error("read(authreply connection) failed: %s", + i_stream_get_error(conn->input)); + } + login_connection_deinit(&conn); + } +} + static void login_connection_send_line(struct login_connection *conn, const char *line) { @@ -138,9 +172,11 @@ login_connection_deinit(&conn); return; } - if (!conn->userdb && strncmp(line, "OK\t", 3) == 0) + if (conn->type != LOGIN_CONNECTION_TYPE_USERDB && + strncmp(line, "OK\t", 3) == 0) line_params = line + 3; - else if (conn->userdb && strncmp(line, "PASS\t", 5) == 0) + else if (conn->type == LOGIN_CONNECTION_TYPE_USERDB && + strncmp(line, "PASS\t", 5) == 0) line_params = line + 5; else { login_connection_send_line(conn, line); @@ -208,21 +244,35 @@ struct login_connection * login_connection_init(struct director *dir, int fd, - struct auth_connection *auth, bool userdb) + struct auth_connection *auth, + enum login_connection_type type) { struct login_connection *conn; conn = i_new(struct login_connection, 1); conn->refcount = 1; conn->fd = fd; - conn->auth = auth; conn->dir = dir; conn->output = o_stream_create_fd(conn->fd, (size_t)-1, FALSE); o_stream_set_no_error_handling(conn->output, TRUE); - conn->io = io_add(conn->fd, IO_READ, login_connection_input, conn); - conn->userdb = userdb; + if (type != LOGIN_CONNECTION_TYPE_AUTHREPLY) { + i_assert(auth != NULL); + conn->auth = auth; + conn->io = io_add(conn->fd, IO_READ, + login_connection_input, conn); + auth_connection_set_callback(conn->auth, auth_input_line, conn); + } else { + i_assert(auth == NULL); + conn->input = i_stream_create_fd(conn->fd, IO_BLOCK_SIZE, FALSE); + conn->io = io_add(conn->fd, IO_READ, + login_connection_authreply_input, conn); + o_stream_nsend_str(conn->output, t_strdup_printf( + "VERSION\tdirector-authreply-server\t%d\t%d\n", + AUTHREPLY_PROTOCOL_MAJOR_VERSION, + AUTHREPLY_PROTOCOL_MINOR_VERSION)); + } + conn->type = type; - auth_connection_set_callback(conn->auth, auth_input_line, conn); DLLIST_PREPEND(&login_connections, conn); return conn; } @@ -239,12 +289,15 @@ DLLIST_REMOVE(&login_connections, conn); io_remove(&conn->io); + if (conn->input != NULL) + i_stream_destroy(&conn->input); o_stream_destroy(&conn->output); if (close(conn->fd) < 0) i_error("close(login connection) failed: %m"); conn->fd = -1; - auth_connection_deinit(&conn->auth); + if (conn->auth != NULL) + auth_connection_deinit(&conn->auth); login_connection_unref(&conn); master_service_client_connection_destroyed(master_service); diff -r c97d10467b62 -r 52cdf321fa07 src/director/login-connection.h --- a/src/director/login-connection.h Wed May 13 15:59:48 2015 +0300 +++ b/src/director/login-connection.h Wed May 13 16:22:27 2015 +0300 @@ -3,9 +3,16 @@ struct director; +enum login_connection_type { + LOGIN_CONNECTION_TYPE_AUTH, + LOGIN_CONNECTION_TYPE_USERDB, + LOGIN_CONNECTION_TYPE_AUTHREPLY +}; + struct login_connection * login_connection_init(struct director *dir, int fd, - struct auth_connection *auth, bool userdb); + struct auth_connection *auth, + enum login_connection_type type); void login_connection_deinit(struct login_connection **conn); void login_connections_deinit(void); diff -r c97d10467b62 -r 52cdf321fa07 src/director/main.c --- a/src/director/main.c Wed May 13 15:59:48 2015 +0300 +++ b/src/director/main.c Wed May 13 16:22:27 2015 +0300 @@ -30,6 +30,7 @@ DIRECTOR_SOCKET_TYPE_UNKNOWN = 0, DIRECTOR_SOCKET_TYPE_AUTH, DIRECTOR_SOCKET_TYPE_USERDB, + DIRECTOR_SOCKET_TYPE_AUTHREPLY, DIRECTOR_SOCKET_TYPE_RING, DIRECTOR_SOCKET_TYPE_DOVEADM }; @@ -81,6 +82,8 @@ return DIRECTOR_SOCKET_TYPE_AUTH; else if (strcmp(suffix, "userdb") == 0) return DIRECTOR_SOCKET_TYPE_USERDB; + else if (strcmp(suffix, "authreply") == 0) + return DIRECTOR_SOCKET_TYPE_AUTHREPLY; else if (strcmp(suffix, "ring") == 0) return DIRECTOR_SOCKET_TYPE_RING; else if (strcmp(suffix, "admin") == 0 || @@ -188,7 +191,14 @@ break; } master_service_client_connection_accept(conn); - (void)login_connection_init(director, conn->fd, auth, userdb); + (void)login_connection_init(director, conn->fd, auth, + userdb ? LOGIN_CONNECTION_TYPE_USERDB : + LOGIN_CONNECTION_TYPE_AUTH); + break; + case DIRECTOR_SOCKET_TYPE_AUTHREPLY: + master_service_client_connection_accept(conn); + (void)login_connection_init(director, conn->fd, NULL, + LOGIN_CONNECTION_TYPE_AUTHREPLY); break; case DIRECTOR_SOCKET_TYPE_RING: if (director_client_connected(conn->fd, &conn->remote_ip) == 0) From dovecot at dovecot.org Wed May 13 13:48:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 13:48:14 +0000 Subject: dovecot-2.2: fts-solr: Don't update "last indexed UID" if we cou... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5483aeb1ce9b changeset: 18676:5483aeb1ce9b user: Timo Sirainen date: Wed May 13 16:46:14 2015 +0300 description: fts-solr: Don't update "last indexed UID" if we couldn't actually add the mails to Solr. diffstat: src/plugins/fts-solr/fts-backend-solr.c | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-) diffs (17 lines): diff -r 52cdf321fa07 -r 5483aeb1ce9b src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Wed May 13 16:22:27 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Wed May 13 16:46:14 2015 +0300 @@ -401,7 +401,12 @@ const char *box_guid; if (ctx->prev_uid != 0) { - fts_index_set_last_uid(ctx->cur_box, ctx->prev_uid); + /* flush solr between mailboxes, so we don't wrongly update + last_uid before we know it has succeeded */ + if (fts_backed_solr_build_commit(ctx) < 0) + _ctx->failed = TRUE; + else if (!_ctx->failed) + fts_index_set_last_uid(ctx->cur_box, ctx->prev_uid); ctx->prev_uid = 0; } From dovecot at dovecot.org Wed May 13 13:56:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 13:56:28 +0000 Subject: dovecot-2.2: fts-solr: Crashfixes Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e6e04177374d changeset: 18677:e6e04177374d user: Timo Sirainen date: Wed May 13 16:54:27 2015 +0300 description: fts-solr: Crashfixes diffstat: src/plugins/fts-solr/fts-backend-solr-old.c | 2 +- src/plugins/fts-solr/fts-backend-solr.c | 2 +- src/plugins/fts-solr/solr-connection.c | 10 +++++++--- src/plugins/fts-solr/solr-connection.h | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diffs (64 lines): diff -r 5483aeb1ce9b -r e6e04177374d src/plugins/fts-solr/fts-backend-solr-old.c --- a/src/plugins/fts-solr/fts-backend-solr-old.c Wed May 13 16:46:14 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr-old.c Wed May 13 16:54:27 2015 +0300 @@ -433,7 +433,7 @@ solr_connection_post_more(ctx->post, str_data(ctx->cmd), str_len(ctx->cmd)); - return solr_connection_post_end(ctx->post); + return solr_connection_post_end(&ctx->post); } static int diff -r 5483aeb1ce9b -r e6e04177374d src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Wed May 13 16:46:14 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Wed May 13 16:54:27 2015 +0300 @@ -339,7 +339,7 @@ solr_connection_post_more(ctx->post, str_data(ctx->cmd), str_len(ctx->cmd)); - return solr_connection_post_end(ctx->post); + return solr_connection_post_end(&ctx->post); } static void diff -r 5483aeb1ce9b -r e6e04177374d src/plugins/fts-solr/solr-connection.c --- a/src/plugins/fts-solr/solr-connection.c Wed May 13 16:46:14 2015 +0300 +++ b/src/plugins/fts-solr/solr-connection.c Wed May 13 16:54:27 2015 +0300 @@ -497,18 +497,22 @@ if (post->failed) return; - if (http_client_request_send_payload(&post->http_req, data, size) != 0 && - conn->request_status < 0) + if (conn->request_status == 0) + (void)http_client_request_send_payload(&post->http_req, data, size); + if (conn->request_status < 0) post->failed = TRUE; } -int solr_connection_post_end(struct solr_connection_post *post) +int solr_connection_post_end(struct solr_connection_post **_post) { + struct solr_connection_post *post = *_post; struct solr_connection *conn = post->conn; int ret = post->failed ? -1 : 0; i_assert(conn->posting); + *_post = NULL; + if (!post->failed) { if (http_client_request_finish_payload(&post->http_req) <= 0 || conn->request_status < 0) { diff -r 5483aeb1ce9b -r e6e04177374d src/plugins/fts-solr/solr-connection.h --- a/src/plugins/fts-solr/solr-connection.h Wed May 13 16:46:14 2015 +0300 +++ b/src/plugins/fts-solr/solr-connection.h Wed May 13 16:54:27 2015 +0300 @@ -25,6 +25,6 @@ solr_connection_post_begin(struct solr_connection *conn); void solr_connection_post_more(struct solr_connection_post *post, const unsigned char *data, size_t size); -int solr_connection_post_end(struct solr_connection_post *post); +int solr_connection_post_end(struct solr_connection_post **post); #endif From dovecot at dovecot.org Wed May 13 14:18:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 14:18:20 +0000 Subject: dovecot-2.2: fts-solr: Fixed memory leak at user deinit. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8973648dd361 changeset: 18678:8973648dd361 user: Timo Sirainen date: Wed May 13 17:16:21 2015 +0300 description: fts-solr: Fixed memory leak at user deinit. diffstat: src/plugins/fts-solr/fts-backend-solr-old.c | 1 + src/plugins/fts-solr/fts-backend-solr.c | 1 + src/plugins/fts-solr/solr-connection.c | 5 ++++- src/plugins/fts-solr/solr-connection.h | 2 +- 4 files changed, 7 insertions(+), 2 deletions(-) diffs (50 lines): diff -r e6e04177374d -r 8973648dd361 src/plugins/fts-solr/fts-backend-solr-old.c --- a/src/plugins/fts-solr/fts-backend-solr-old.c Wed May 13 16:54:27 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr-old.c Wed May 13 17:16:21 2015 +0300 @@ -249,6 +249,7 @@ { struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; + solr_connection_deinit(&backend->solr_conn); i_free(backend->id_namespace); i_free(backend->id_username); i_free(backend); diff -r e6e04177374d -r 8973648dd361 src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Wed May 13 16:54:27 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Wed May 13 17:16:21 2015 +0300 @@ -179,6 +179,7 @@ { struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; + solr_connection_deinit(&backend->solr_conn); i_free(backend); } diff -r e6e04177374d -r 8973648dd361 src/plugins/fts-solr/solr-connection.c --- a/src/plugins/fts-solr/solr-connection.c Wed May 13 16:54:27 2015 +0300 +++ b/src/plugins/fts-solr/solr-connection.c Wed May 13 17:16:21 2015 +0300 @@ -142,8 +142,11 @@ return 0; } -void solr_connection_deinit(struct solr_connection *conn) +void solr_connection_deinit(struct solr_connection **_conn) { + struct solr_connection *conn = *_conn; + + *_conn = NULL; XML_ParserFree(conn->xml_parser); i_free(conn->http_host); i_free(conn->http_base_url); diff -r e6e04177374d -r 8973648dd361 src/plugins/fts-solr/solr-connection.h --- a/src/plugins/fts-solr/solr-connection.h Wed May 13 16:54:27 2015 +0300 +++ b/src/plugins/fts-solr/solr-connection.h Wed May 13 17:16:21 2015 +0300 @@ -15,7 +15,7 @@ int solr_connection_init(const char *url, bool debug, struct solr_connection **conn_r, const char **error_r); -void solr_connection_deinit(struct solr_connection *conn); +void solr_connection_deinit(struct solr_connection **conn); int solr_connection_select(struct solr_connection *conn, const char *query, pool_t pool, struct solr_result ***box_results_r); From dovecot at dovecot.org Wed May 13 16:28:45 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 16:28:45 +0000 Subject: dovecot-2.2: mbox: Fixed corruption in some usage patterns. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b6ea460e7cc4 changeset: 18679:b6ea460e7cc4 user: Timo Sirainen date: Wed May 13 19:24:55 2015 +0300 description: mbox: Fixed corruption in some usage patterns. Something like: - first mail is being expunged - other mails are being rewritten and they get their space from the expunged mail - there's not enough space for the last mail to get space - we add more space - we'll now need to use up the space. We can't just decide to mark the mails dirty. diffstat: src/lib-storage/index/mbox/mbox-sync.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 8973648dd361 -r b6ea460e7cc4 src/lib-storage/index/mbox/mbox-sync.c --- a/src/lib-storage/index/mbox/mbox-sync.c Wed May 13 17:16:21 2015 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync.c Wed May 13 19:24:55 2015 +0300 @@ -689,8 +689,10 @@ } } else if (mail_ctx->need_rewrite) { mbox_sync_update_header(mail_ctx); - if (sync_ctx->delay_writes) { - /* mark it dirty and do it later */ + if (sync_ctx->delay_writes && sync_ctx->need_space_seq == 0) { + /* mark it dirty and do it later. we can't do this + if we're in the middle of rewriting acquiring more + space. */ mail_ctx->dirty = TRUE; return 0; } From dovecot at dovecot.org Wed May 13 16:28:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 16:28:46 +0000 Subject: dovecot-2.2: mbox: If we notice our internal state is wrong, avo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1171265c3834 changeset: 18680:1171265c3834 user: Timo Sirainen date: Wed May 13 19:26:45 2015 +0300 description: mbox: If we notice our internal state is wrong, avoid further corruption by returning error. diffstat: src/lib-storage/index/mbox/mbox-sync-rewrite.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (24 lines): diff -r b6ea460e7cc4 -r 1171265c3834 src/lib-storage/index/mbox/mbox-sync-rewrite.c --- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c Wed May 13 19:24:55 2015 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c Wed May 13 19:26:45 2015 +0300 @@ -429,6 +429,7 @@ " bytes, now needs %"PRIuSIZE_T" bytes", seq, mails[idx].uid, mails[idx].uid_broken, (uoff_t)-mails[idx].space, need_space); + return -1; } } @@ -595,8 +596,10 @@ mails[idx].from_offset += move_diff; } - i_assert(mails[idx].from_offset == start_offset); - i_assert(move_diff + (off_t)expunged_space >= 0); + if (ret == 0) { + i_assert(mails[idx].from_offset == start_offset); + i_assert(move_diff + (off_t)expunged_space >= 0); + } mbox_sync_file_updated(sync_ctx, FALSE); sync_ctx->prev_msg_uid = orig_prev_msg_uid; From dovecot at dovecot.org Wed May 13 16:38:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 16:38:28 +0000 Subject: dovecot-2.2: lib-lda: Don't crash if postmaster_address is given... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/25dc311a5faa changeset: 18681:25dc311a5faa user: Timo Sirainen date: Wed May 13 19:36:29 2015 +0300 description: lib-lda: Don't crash if postmaster_address is given as command line parameter. diffstat: src/lib-lda/lda-settings.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 1171265c3834 -r 25dc311a5faa src/lib-lda/lda-settings.c --- a/src/lib-lda/lda-settings.c Wed May 13 19:26:45 2015 +0300 +++ b/src/lib-lda/lda-settings.c Wed May 13 19:36:29 2015 +0300 @@ -77,8 +77,8 @@ if (*set->hostname == '\0') set->hostname = p_strdup(pool, my_hostdomain()); - i_assert(set->postmaster_address[0] == SETTING_STRVAR_UNEXPANDED[0]); - if (set->postmaster_address[1] == '\0') { + if (set->postmaster_address[0] == SETTING_STRVAR_UNEXPANDED[0] && + set->postmaster_address[1] == '\0') { /* check for valid looking fqdn in hostname */ if (strchr(set->hostname, '.') == NULL) { *error_r = "postmaster_address setting not given"; From pigeonhole at rename-it.nl Wed May 13 17:53:16 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 13 May 2015 19:53:16 +0200 Subject: dovecot-2.2-pigeonhole: managesieve: Storage quota was not alway... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/ce4709c0ee63 changeset: 2062:ce4709c0ee63 user: Stephan Bosch date: Wed May 13 19:52:51 2015 +0200 description: managesieve: Storage quota was not always enforced properly for scripts uploaded as quoted string. Nobody does that, but we support it. diffstat: src/managesieve/cmd-putscript.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r cf1513460334 -r ce4709c0ee63 src/managesieve/cmd-putscript.c --- a/src/managesieve/cmd-putscript.c Wed May 13 19:52:43 2015 +0200 +++ b/src/managesieve/cmd-putscript.c Wed May 13 19:52:51 2015 +0200 @@ -183,6 +183,11 @@ /* If quoted string, the size was not known until now */ if ( ctx->script_size == 0 ) { + if (sieve_script_get_size(script, &ctx->script_size) < 0) { + client_send_storage_error(client, ctx->storage); + cmd_putscript_finish(ctx); + return TRUE; + } /* Check quota; max size is already checked */ if ( ctx->scriptname != NULL && !managesieve_quota_check_all (client, ctx->scriptname, ctx->script_size) ) { From pigeonhole at rename-it.nl Wed May 13 17:53:16 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 13 May 2015 19:53:16 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: script: Forgot to assign stor... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/cf1513460334 changeset: 2061:cf1513460334 user: Stephan Bosch date: Wed May 13 19:52:43 2015 +0200 description: lib-sieve: script: Forgot to assign storage error when determining script size from stream. diffstat: src/lib-sieve/sieve-script.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diffs (19 lines): diff -r 3fab77ef9302 -r cf1513460334 src/lib-sieve/sieve-script.c --- a/src/lib-sieve/sieve-script.c Tue May 12 22:51:12 2015 +0200 +++ b/src/lib-sieve/sieve-script.c Wed May 13 19:52:43 2015 +0200 @@ -271,7 +271,14 @@ sieve_script_get_stream(script, &stream, NULL) < 0 ) return -1; - return i_stream_get_size(script->stream, TRUE, size_r); + if (i_stream_get_size(script->stream, TRUE, size_r) < 0) { + sieve_storage_set_critical(script->storage, + "i_stream_get_size(%s) failed: %s", + i_stream_get_name(script->stream), + i_stream_get_error(script->stream)); + return -1; + } + return 0; } bool sieve_script_is_open(const struct sieve_script *script) From pigeonhole at rename-it.nl Wed May 13 17:53:16 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 13 May 2015 19:53:16 +0200 Subject: dovecot-2.2-pigeonhole: managesieve: Managesieve used "managesie... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/555374414869 changeset: 2063:555374414869 user: Stephan Bosch date: Wed May 13 19:52:51 2015 +0200 description: managesieve: Managesieve used "managesieve" rather than "sieve" as login service name, which means that all managesieve-specific settings where ignored. diffstat: src/managesieve/main.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (23 lines): diff -r ce4709c0ee63 -r 555374414869 src/managesieve/main.c --- a/src/managesieve/main.c Wed May 13 19:52:51 2015 +0200 +++ b/src/managesieve/main.c Wed May 13 19:52:51 2015 +0200 @@ -158,7 +158,8 @@ buffer_t *input_buf; memset(&input, 0, sizeof(input)); - input.module = input.service = "managesieve"; + input.module = "managesieve"; + input.service = "sieve"; input.username = username != NULL ? username : getenv("USER"); if (input.username == NULL && IS_STANDALONE()) input.username = getlogin(); @@ -188,7 +189,8 @@ buffer_t input_buf; memset(&input, 0, sizeof(input)); - input.module = input.service = "managesieve"; + input.module = "managesieve"; + input.service = "sieve"; input.local_ip = client->auth_req.local_ip; input.remote_ip = client->auth_req.remote_ip; input.username = username; From pigeonhole at rename-it.nl Wed May 13 17:53:16 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 13 May 2015 19:53:16 +0200 Subject: dovecot-2.2-pigeonhole: managesieve: Implemented support for rep... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/ff3ab88205a6 changeset: 2064:ff3ab88205a6 user: Stephan Bosch date: Wed May 13 19:52:51 2015 +0200 description: managesieve: Implemented support for reporting command statistics at disconnect. diffstat: src/managesieve/cmd-deletescript.c | 1 + src/managesieve/cmd-getscript.c | 3 +++ src/managesieve/cmd-putscript.c | 8 ++++++++ src/managesieve/cmd-renamescript.c | 6 ++++-- src/managesieve/managesieve-client.c | 22 +++++++++++++++++++--- src/managesieve/managesieve-client.h | 9 +++++++++ 6 files changed, 44 insertions(+), 5 deletions(-) diffs (118 lines): diff -r 555374414869 -r ff3ab88205a6 src/managesieve/cmd-deletescript.c --- a/src/managesieve/cmd-deletescript.c Wed May 13 19:52:51 2015 +0200 +++ b/src/managesieve/cmd-deletescript.c Wed May 13 19:52:51 2015 +0200 @@ -31,6 +31,7 @@ if ( sieve_script_delete(script, FALSE) < 0 ) { client_send_storage_error(client, storage); } else { + client->deleted_count++; client_send_ok(client, "Deletescript completed."); } diff -r 555374414869 -r ff3ab88205a6 src/managesieve/cmd-getscript.c --- a/src/managesieve/cmd-getscript.c Wed May 13 19:52:51 2015 +0200 +++ b/src/managesieve/cmd-getscript.c Wed May 13 19:52:51 2015 +0200 @@ -40,6 +40,9 @@ return TRUE; } + client->get_count++; + client->get_bytes += ctx->script_size; + client_send_line(client, ""); client_send_ok(client, "Getscript completed."); return TRUE; diff -r 555374414869 -r ff3ab88205a6 src/managesieve/cmd-putscript.c --- a/src/managesieve/cmd-putscript.c Wed May 13 19:52:51 2015 +0200 +++ b/src/managesieve/cmd-putscript.c Wed May 13 19:52:51 2015 +0200 @@ -249,6 +249,14 @@ /* Report result to user */ if ( success ) { + if ( ctx->scriptname != NULL ) { + client->put_count++; + client->put_bytes += ctx->script_size; + } else { + client->check_count++; + client->check_bytes += ctx->script_size; + } + if ( sieve_get_warnings(ehandler) > 0 ) client_send_okresp(client, "WARNINGS", str_c(errors)); else { diff -r 555374414869 -r ff3ab88205a6 src/managesieve/cmd-renamescript.c --- a/src/managesieve/cmd-renamescript.c Wed May 13 19:52:51 2015 +0200 +++ b/src/managesieve/cmd-renamescript.c Wed May 13 19:52:51 2015 +0200 @@ -29,10 +29,12 @@ return TRUE; } - if (sieve_script_rename(script, newname) < 0) + if (sieve_script_rename(script, newname) < 0) { client_send_storage_error(client, storage); - else + } else { + client->renamed_count++; client_send_ok(client, "Renamescript completed."); + } sieve_script_unref(&script); return TRUE; diff -r 555374414869 -r ff3ab88205a6 src/managesieve/managesieve-client.c --- a/src/managesieve/managesieve-client.c Wed May 13 19:52:51 2015 +0200 +++ b/src/managesieve/managesieve-client.c Wed May 13 19:52:51 2015 +0200 @@ -175,6 +175,14 @@ static const char *client_stats(struct client *client) { static struct var_expand_table static_tab[] = { + { 't', NULL, "put_bytes" }, + { 'p', NULL, "put_count" }, + { 'b', NULL, "get_bytes" }, + { 'g', NULL, "get_count" }, + { 'v', NULL, "check_bytes" }, + { 'c', NULL, "check_count" }, + { 'd', NULL, "deleted_count" }, + { 'r', NULL, "renamed_count" }, { 'i', NULL, "input" }, { 'o', NULL, "output" }, { '\0', NULL, "session" }, @@ -186,9 +194,17 @@ tab = t_malloc(sizeof(static_tab)); memcpy(tab, static_tab, sizeof(static_tab)); - tab[0].value = dec2str(i_stream_get_absolute_offset(client->input)); - tab[1].value = dec2str(client->output->offset); - tab[2].value = client->session_id; + tab[0].value = dec2str(client->put_bytes); + tab[1].value = dec2str(client->put_count); + tab[2].value = dec2str(client->get_bytes); + tab[3].value = dec2str(client->get_count); + tab[4].value = dec2str(client->check_bytes); + tab[5].value = dec2str(client->check_count); + tab[6].value = dec2str(client->deleted_count); + tab[7].value = dec2str(client->renamed_count); + tab[8].value = dec2str(i_stream_get_absolute_offset(client->input)); + tab[9].value = dec2str(client->output->offset); + tab[10].value = client->session_id; str = t_str_new(128); var_expand(str, client->set->managesieve_logout_format, tab); diff -r 555374414869 -r ff3ab88205a6 src/managesieve/managesieve-client.h --- a/src/managesieve/managesieve-client.h Wed May 13 19:52:51 2015 +0200 +++ b/src/managesieve/managesieve-client.h Wed May 13 19:52:51 2015 +0200 @@ -57,6 +57,15 @@ struct managesieve_parser *parser; struct client_command_context cmd; + uoff_t put_bytes; + uoff_t get_bytes; + uoff_t check_bytes; + unsigned int put_count; + unsigned int get_count; + unsigned int check_count; + unsigned int deleted_count; + unsigned int renamed_count; + unsigned int disconnected:1; unsigned int destroyed:1; unsigned int command_pending:1; From dovecot at dovecot.org Wed May 13 18:37:54 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 18:37:54 +0000 Subject: dovecot-2.2: Released v2.2.17. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/166106aaefc5 changeset: 18682:166106aaefc5 user: Timo Sirainen date: Wed May 13 19:45:07 2015 +0300 description: Released v2.2.17. diffstat: NEWS | 5 ++--- configure.ac | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diffs (31 lines): diff -r 25dc311a5faa -r 166106aaefc5 NEWS --- a/NEWS Wed May 13 19:36:29 2015 +0300 +++ b/NEWS Wed May 13 19:45:07 2015 +0300 @@ -1,4 +1,4 @@ -v2.2.17 2015-05-xx Timo Sirainen +v2.2.17 2015-05-13 Timo Sirainen * Dovecot no longer checks or warns if a mountpoint is removed. This was causing more trouble than it was worth. Make sure that all the @@ -45,8 +45,7 @@ - imapc: Fixed \Muted label handling in gmail-migration. - imapc: Various bugfixes and improvements. - Trash plugin fixes by Alexei Gradinari - - mbox: Fixed crash/corruption in some situations when the first mail - was expunged. + - mbox: Fixed crash/corruption in some situations v2.2.16 2015-03-12 Timo Sirainen diff -r 25dc311a5faa -r 166106aaefc5 configure.ac --- a/configure.ac Wed May 13 19:36:29 2015 +0300 +++ b/configure.ac Wed May 13 19:45:07 2015 +0300 @@ -2,7 +2,7 @@ # Be sure to update ABI version also if anything changes that might require # recompiling plugins. Most importantly that means if any structs are changed. -AC_INIT([Dovecot],[2.2.17.rc2],[dovecot at dovecot.org]) +AC_INIT([Dovecot],[2.2.17],[dovecot at dovecot.org]) AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv17($PACKAGE_VERSION)", [Dovecot ABI version]) AC_CONFIG_SRCDIR([src]) From dovecot at dovecot.org Wed May 13 18:37:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 18:37:55 +0000 Subject: dovecot-2.2: Added tag 2.2.17 for changeset 166106aaefc5 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7e437ea0cc0f changeset: 18683:7e437ea0cc0f user: Timo Sirainen date: Wed May 13 19:45:07 2015 +0300 description: Added tag 2.2.17 for changeset 166106aaefc5 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 166106aaefc5 -r 7e437ea0cc0f .hgtags --- a/.hgtags Wed May 13 19:45:07 2015 +0300 +++ b/.hgtags Wed May 13 19:45:07 2015 +0300 @@ -125,3 +125,4 @@ 0f8f0d8ee60745f6ce7ff379da7f2660aec7c4d4 2.2.16 da685736985a17c2b6b1ff24c4f762a4009ce598 2.2.17.rc1 2c1ce55de520484431a63041602829d21195b4e9 2.2.17.rc2 +166106aaefc5e6cf3850dba9534ca90153931ad9 2.2.17 From dovecot at dovecot.org Wed May 13 18:37:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 18:37:55 +0000 Subject: dovecot-2.2: Added signature for changeset 166106aaefc5 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ce1aa9c639ab changeset: 18684:ce1aa9c639ab user: Timo Sirainen date: Wed May 13 19:45:11 2015 +0300 description: Added signature for changeset 166106aaefc5 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 7e437ea0cc0f -r ce1aa9c639ab .hgsigs --- a/.hgsigs Wed May 13 19:45:07 2015 +0300 +++ b/.hgsigs Wed May 13 19:45:11 2015 +0300 @@ -88,3 +88,4 @@ 0f8f0d8ee60745f6ce7ff379da7f2660aec7c4d4 0 iEYEABECAAYFAlUBs5IACgkQyUhSUUBVisn/yACcCUC3xDOCPFAgTKd72uyDrEUv7wcAn1tNcf+AyIUmM9ksDeB2fuZZZJFT da685736985a17c2b6b1ff24c4f762a4009ce598 0 iEYEABECAAYFAlVORBAACgkQyUhSUUBViskBnQCfRmUjoFty2pUJrrrdIvkEgOCiVEUAnRK0OYM4z5bxs9xcBlXZn8n2qyFx 2c1ce55de520484431a63041602829d21195b4e9 0 iEYEABECAAYFAlVSXUYACgkQyUhSUUBVislsrwCdEtWDWrQPsJua75RL83lpkeh7DagAoKW+534P0iziiX/nD6QzNmBTHODe +166106aaefc5e6cf3850dba9534ca90153931ad9 0 iEYEABECAAYFAlVTf5MACgkQyUhSUUBViskLHgCeMooopuB7egroMj8w78YarjjRieYAnjVSKcz0sXShlbYRsLRPOnyfq/pS From dovecot at dovecot.org Wed May 13 18:37:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 18:37:55 +0000 Subject: dovecot-2.2: lib-fts: Makefile compiling dependency fix Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/13787b077df4 changeset: 18685:13787b077df4 user: Timo Sirainen date: Wed May 13 20:47:35 2015 +0300 description: lib-fts: Makefile compiling dependency fix diffstat: src/lib-fts/Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r ce1aa9c639ab -r 13787b077df4 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Wed May 13 19:45:11 2015 +0300 +++ b/src/lib-fts/Makefile.am Wed May 13 20:47:35 2015 +0300 @@ -28,7 +28,7 @@ WordBreakProperty.txt: test -f WordBreakProperty.txt || wget http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakProperty.txt -$(srcdir)/word-boundary-data.c: word-properties.pl PropList.txt +$(srcdir)/word-boundary-data.c: word-properties.pl WordBreakProperty.txt perl word-properties.pl boundaries WordBreakProperty.txt > $@.tmp && mv $@.tmp $@ PropList.txt: From dovecot at dovecot.org Wed May 13 18:37:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 13 May 2015 18:37:56 +0000 Subject: dovecot-2.2: fts-lucene: Removed dead code Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/041b6b9921c5 changeset: 18686:041b6b9921c5 user: Timo Sirainen date: Wed May 13 21:35:56 2015 +0300 description: fts-lucene: Removed dead code Found by Coverity diffstat: src/plugins/fts-lucene/fts-backend-lucene.c | 23 +++++++++-------------- 1 files changed, 9 insertions(+), 14 deletions(-) diffs (37 lines): diff -r 13787b077df4 -r 041b6b9921c5 src/plugins/fts-lucene/fts-backend-lucene.c --- a/src/plugins/fts-lucene/fts-backend-lucene.c Wed May 13 20:47:35 2015 +0300 +++ b/src/plugins/fts-lucene/fts-backend-lucene.c Wed May 13 21:35:56 2015 +0300 @@ -96,24 +96,19 @@ backend->selected_box_generation == box->generation_sequence) return 0; - if (box != NULL) { - if (fts_lucene_get_mailbox_guid(box, guid) < 0) - return -1; - buffer_create_from_data(&buf, guid_hex, MAILBOX_GUID_HEX_LENGTH); - binary_to_hex_append(&buf, guid, GUID_128_SIZE); - for (i = 0; i < N_ELEMENTS(wguid_hex); i++) - wguid_hex[i] = guid_hex[i]; + if (fts_lucene_get_mailbox_guid(box, guid) < 0) + return -1; + buffer_create_from_data(&buf, guid_hex, MAILBOX_GUID_HEX_LENGTH); + binary_to_hex_append(&buf, guid, GUID_128_SIZE); + for (i = 0; i < N_ELEMENTS(wguid_hex); i++) + wguid_hex[i] = guid_hex[i]; - lucene_index_select_mailbox(backend->index, wguid_hex); - } else { - lucene_index_unselect_mailbox(backend->index); - memset(&guid, 0, sizeof(guid)); - } + lucene_index_select_mailbox(backend->index, wguid_hex); + backend->selected_box = box; memcpy(backend->selected_box_guid, guid, sizeof(backend->selected_box_guid)); - backend->selected_box_generation = - box == NULL ? 0 : box->generation_sequence; + backend->selected_box_generation = box->generation_sequence; return 0; } From pigeonhole at rename-it.nl Wed May 13 20:52:00 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 13 May 2015 22:52:00 +0200 Subject: dovecot-2.2-pigeonhole: Released v0.4.8.rc3 for Dovecot v2.2.17. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/7b154d69394f changeset: 2065:7b154d69394f user: Stephan Bosch date: Wed May 13 22:50:46 2015 +0200 description: Released v0.4.8.rc3 for Dovecot v2.2.17. diffstat: NEWS | 10 +++++++++- configure.ac | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diffs (35 lines): diff -r ff3ab88205a6 -r 7b154d69394f NEWS --- a/NEWS Wed May 13 19:52:51 2015 +0200 +++ b/NEWS Wed May 13 22:50:46 2015 +0200 @@ -8,13 +8,21 @@ them and store a private adjusted version. This could also be achieved by copying the default script into the user's script storage, but that way updates to the global sieve_default script would be ignored. + + ManageSieve: Implemented support for reporting command statistics at + disconnect. Statistics include the number of bytes and scripts uploaded/ + downloaded/checked and the number of scripts deleted/renamed. - Fixed problem in address test: erroneously decoded mime-encoded words in address headers. - extprograms plugin: Fixed failure occurring when connecting to script service without the need to read back the output from the external program. - Fixed bug in script storage path normalization occurring with relative symbolic links below root. - - Fixed and updated various parts of the documentation + - Fixed and updated various parts of the documentation + - ManageSieve: Used "managesieve" rather than "sieve" as login service name, + which means that all managesieve-specific settings where ignored. + - Managesieve: Storage quota was not always enforced properly for scripts + uploaded as quoted string. Nobody uses that, but it is allowed in the + specification and we support it, so it should work properly. v0.4.7 19-03-2015 Stephan Bosch diff -r ff3ab88205a6 -r 7b154d69394f configure.ac --- a/configure.ac Wed May 13 19:52:51 2015 +0200 +++ b/configure.ac Wed May 13 22:50:46 2015 +0200 @@ -1,4 +1,4 @@ -AC_INIT([Pigeonhole], [0.4.8.rc2], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) +AC_INIT([Pigeonhole], [0.4.8.rc3], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_SRCDIR([src]) AC_CONFIG_MACRO_DIR([m4]) From pigeonhole at rename-it.nl Wed May 13 20:52:00 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 13 May 2015 22:52:00 +0200 Subject: dovecot-2.2-pigeonhole: Added tag 0.4.8.rc3 for changeset 7b154d... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/68a97053e32d changeset: 2066:68a97053e32d user: Stephan Bosch date: Wed May 13 22:50:58 2015 +0200 description: Added tag 0.4.8.rc3 for changeset 7b154d69394f diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 7b154d69394f -r 68a97053e32d .hgtags --- a/.hgtags Wed May 13 22:50:46 2015 +0200 +++ b/.hgtags Wed May 13 22:50:58 2015 +0200 @@ -29,3 +29,4 @@ f2323a90c202c4e9708a0c87e81a1b06d267a5c5 0.4.7 fcc97e953584269e5fa140363804f4016c33ef1c 0.4.8.rc1 44d41235776de692f1f4ef9843ee6f3901fa3138 0.4.8.rc2 +7b154d69394fcdbbc0b066978d864e551b736ce9 0.4.8.rc3 From pigeonhole at rename-it.nl Wed May 13 20:53:35 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 13 May 2015 22:53:35 +0200 Subject: dovecot-2.2-pigeonhole: Added signature for changeset 7b154d69394f Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/74f2b17cd83d changeset: 2067:74f2b17cd83d user: Stephan Bosch date: Wed May 13 22:54:18 2015 +0200 description: Added signature for changeset 7b154d69394f diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 68a97053e32d -r 74f2b17cd83d .hgsigs --- a/.hgsigs Wed May 13 22:50:58 2015 +0200 +++ b/.hgsigs Wed May 13 22:54:18 2015 +0200 @@ -23,3 +23,4 @@ f2323a90c202c4e9708a0c87e81a1b06d267a5c5 0 iQEcBAABAgAGBQJVC01gAAoJEATWKx49+7T0eNQH/iHUSapqKCaPoZByc1UHpp/FS7IEzbXUWTa71uTHN7z+/Gr6vKm4/xl0k10N6XhpY5aJImLt31dd8jSDg6hqhVwDlqIImJzCjf+flG4dZu5MIL2vibjEn7InYvMli5EcafQ08EkTWGm7IPOcqfnZheevU64piqaUr1YfNaR0g/cQcffvT9qXNovlqFQe+hDB2/wUj6zKiHx0rqUzAOh4dAcrfq1jCpXhpGChjezxyiGzFpLPopbEHQU9M9Fr6VFgY+v/mDVp6hj8DpE7Q84BxPvxJKi0il+h0/lliDZUA+XtZsVjckih5WBDC5tLgNRKujMzkAc2UNEU/1vJxkPkldI= fcc97e953584269e5fa140363804f4016c33ef1c 0 iQEcBAABAgAGBQJVT9pVAAoJEATWKx49+7T0/14H/3iejYaCOqm5ftfQDMlebUhbMa9LGEMwOPRoBxnzGOBoJGEQ7MS/aqk8ARM1i016jFtDa0oVWkTDyWZhDwQHS6yA2HkXPxizKHf9+9Kc2Lxjew4GIyTGSNmkaJb34XUKGhKx4YMfDXQiFsc0G/vVSO8UoZzvqho6g7bUmuWeuCrXmEW+M+bUG4KMCF0abYba7iasYKaV+1fSnbAnh5kRQqwfUfs/ugdU4f9VZxmnGdJymf540KdoqEzePMaxhEp6OPhhr/eVGabfWhV+RJiRI7atWQl7I6uoEkvziObuUKx3rl0xVoSVcKeCC4APsJdQQT/HDoPmWfyaQCV+WORmz6U= 44d41235776de692f1f4ef9843ee6f3901fa3138 0 iQEcBAABAgAGBQJVUP1CAAoJEATWKx49+7T0+GMH/3QKZewiy+jeeHmb8wYExW6eiAEprb7lq2U8UjN5iOuID7OnDbkZ0/jr/oU0zsxpch/lacpAa0cIvgvujhGuXzeR2XNO3MWyiOPe4t0WcFaoP4Hl+EV7faZipCdPx7mGbxFYSEx9rz4Kp0MBnG/I+yE2+I1AbBsHqQMC1meAA3llu1n/Wf9BOhyzUP4/AL9CSNUtO/6tUZriXQOrRZpwkcyc7mrH6Trpyy0YXkEVWAZ8jdusiVuLmwMzEHUaF4SXc2daHhRRRiwp3wCeIkyVvqm8BbxFGZ7ZMM7fKnhj3LGmSLfeloHHmCC3HU6SfzvNmv8Fcieah/thpl/EOdKiWeI= +7b154d69394fcdbbc0b066978d864e551b736ce9 0 iQEcBAABAgAGBQJVU7n1AAoJEATWKx49+7T0sOYH/jL46oo70cwImQN5pyAHudSarM9Mtls8//vX7JmJ/b8OPftwwx4ExBNYmvHSv315CPjq6ntMA7GMZ1tAd9lHk+ZB2WFL/Y7MQZUGgowo6K/F36MW1Z6tGN2rtnooqglyamKUPqrLIurs5Hc9444VrGMcdeMecGn9mEpxULy1jsMmo0AVVsyf50PHbYRwSgq960onz93lKhTcaD7/Sc76FPbJenqstP9iPQBuXugVxv/p9gN31Xx/V8RlxmHIjeHljx33m76sTGQ/8ca2XKvBwft8dZU4lTsrSmcvt+0wxATurjtfJ+aqPlH5pPXmqydbsEe6F3fjZTblwexc4kFNDsQ= From pigeonhole at rename-it.nl Wed May 13 21:13:47 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 13 May 2015 23:13:47 +0200 Subject: dovecot-2.2-pigeonhole: Small adjustment to NEWS file. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/b491aa4139bc changeset: 2068:b491aa4139bc user: Stephan Bosch date: Wed May 13 23:13:42 2015 +0200 description: Small adjustment to NEWS file. diffstat: NEWS | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 74f2b17cd83d -r b491aa4139bc NEWS --- a/NEWS Wed May 13 22:54:18 2015 +0200 +++ b/NEWS Wed May 13 23:13:42 2015 +0200 @@ -6,8 +6,8 @@ + Implemented magic to make sieve_default script visible from ManageSieve under a configurable name. This way, users can see the default rules, edit them and store a private adjusted version. This could also be achieved by - copying the default script into the user's script storage, but that way - updates to the global sieve_default script would be ignored. + copying the default script into the user's script storage, but updates to + the global sieve_default script would be ignored that way. + ManageSieve: Implemented support for reporting command statistics at disconnect. Statistics include the number of bytes and scripts uploaded/ downloaded/checked and the number of scripts deleted/renamed. From dovecot at dovecot.org Thu May 14 08:28:01 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 14 May 2015 08:28:01 +0000 Subject: dovecot-2.2: director: UNIX auth sockets were wrongly detected a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cc6b6d7c1574 changeset: 18687:cc6b6d7c1574 user: Timo Sirainen date: Thu May 14 11:26:00 2015 +0300 description: director: UNIX auth sockets were wrongly detected as doveadm or ring sockets. A workaround would be to use login/director-auth socket name instead of login/director. diffstat: src/director/main.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 041b6b9921c5 -r cc6b6d7c1574 src/director/main.c --- a/src/director/main.c Wed May 13 21:35:56 2015 +0300 +++ b/src/director/main.c Thu May 14 11:26:00 2015 +0300 @@ -99,7 +99,8 @@ { unsigned int local_port; - if (net_getsockname(listen_fd, NULL, &local_port) == 0) { + if (net_getsockname(listen_fd, NULL, &local_port) == 0 && + local_port != 0) { /* TCP/IP connection */ if (local_port == set->director_doveadm_port) return DIRECTOR_SOCKET_TYPE_DOVEADM; From dovecot at dovecot.org Thu May 14 08:29:04 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 14 May 2015 08:29:04 +0000 Subject: dovecot-2.2: lib-lda: Added missing include file. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/05e14a49507f changeset: 18688:05e14a49507f user: Timo Sirainen date: Thu May 14 11:27:04 2015 +0300 description: lib-lda: Added missing include file. This fixes building Pigeonhole in some systems. diffstat: src/lib-lda/mail-deliver.h | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r cc6b6d7c1574 -r 05e14a49507f src/lib-lda/mail-deliver.h --- a/src/lib-lda/mail-deliver.h Thu May 14 11:26:00 2015 +0300 +++ b/src/lib-lda/mail-deliver.h Thu May 14 11:27:04 2015 +0300 @@ -5,6 +5,8 @@ #include "mail-types.h" #include "mail-error.h" +#include + /* How many seconds to wait for replies from SMTP before failing. Used for sending rejects, forward, etc. */ #define LDA_SUBMISSION_TIMEOUT_SECS 30 From dovecot at dovecot.org Thu May 14 10:36:02 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 14 May 2015 10:36:02 +0000 Subject: dovecot-2.2: pgsql: Include connect state string on connect fail... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a0dd9dad3b8c changeset: 18689:a0dd9dad3b8c user: Timo Sirainen date: Thu May 14 13:33:59 2015 +0300 description: pgsql: Include connect state string on connect failure errors. diffstat: src/lib-sql/driver-pgsql.c | 13 +++++++++---- 1 files changed, 9 insertions(+), 4 deletions(-) diffs (59 lines): diff -r 05e14a49507f -r a0dd9dad3b8c src/lib-sql/driver-pgsql.c --- a/src/lib-sql/driver-pgsql.c Thu May 14 11:27:04 2015 +0300 +++ b/src/lib-sql/driver-pgsql.c Thu May 14 13:33:59 2015 +0300 @@ -26,6 +26,7 @@ struct sql_result *sync_result; char *error; + const char *connect_state; unsigned int fatal_error:1; }; @@ -148,16 +149,18 @@ switch (ret) { case PGRES_POLLING_READING: + db->connect_state = "wait for input"; io_dir = IO_READ; break; case PGRES_POLLING_WRITING: + db->connect_state = "wait for output"; io_dir = IO_WRITE; break; case PGRES_POLLING_OK: break; case PGRES_POLLING_FAILED: - i_error("%s: Connect failed to database %s: %s", - pgsql_prefix(db), PQdb(db->pg), last_error(db)); + i_error("%s: Connect failed to database %s: %s (state: %s)", + pgsql_prefix(db), PQdb(db->pg), last_error(db), db->connect_state); driver_pgsql_close(db); return; } @@ -168,6 +171,7 @@ } if (io_dir == 0) { + db->connect_state = "connected"; if (db->to_connect != NULL) timeout_remove(&db->to_connect); driver_pgsql_set_state(db, SQL_DB_STATE_IDLE); @@ -183,8 +187,8 @@ { unsigned int secs = ioloop_time - db->api.last_connect_try; - i_error("%s: Connect failed: Timeout after %u seconds", - pgsql_prefix(db), secs); + i_error("%s: Connect failed: Timeout after %u seconds (state: %s)", + pgsql_prefix(db), secs, db->connect_state); driver_pgsql_close(db); } @@ -213,6 +217,7 @@ i_assert(db->to_connect == NULL); db->to_connect = timeout_add(SQL_CONNECT_TIMEOUT_SECS * 1000, driver_pgsql_connect_timeout, db); + db->connect_state = "connecting"; db->io = io_add(PQsocket(db->pg), IO_WRITE, connect_callback, db); db->io_dir = IO_WRITE; driver_pgsql_set_state(db, SQL_DB_STATE_CONNECTING); From dovecot at dovecot.org Thu May 14 10:40:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 14 May 2015 10:40:22 +0000 Subject: dovecot-2.2: configure: Use consistent quotation for AC_DEFINEs ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a2c3a0f671a6 changeset: 18690:a2c3a0f671a6 user: Matti Hamalainen date: Thu May 14 12:36:38 2015 +0300 description: configure: Use consistent quotation for AC_DEFINEs as per autoconf recommendations. Also fix one minor typo. diffstat: configure.ac | 277 +++++++++++++++++++++++++++++----------------------------- 1 files changed, 138 insertions(+), 139 deletions(-) diffs (truncated from 1078 to 300 lines): diff -r a0dd9dad3b8c -r a2c3a0f671a6 configure.ac --- a/configure.ac Thu May 14 13:33:59 2015 +0300 +++ b/configure.ac Thu May 14 12:36:38 2015 +0300 @@ -37,14 +37,14 @@ AC_ARG_ENABLE(devel-checks, AS_HELP_STRING([--enable-devel-checks], [Enable some extra expensive checks for developers]), if test x$enableval = xyes; then - AC_DEFINE(DEBUG,, Build with extra debugging checks) + AC_DEFINE(DEBUG,, [Build with extra debugging checks]) want_devel_checks=yes fi) AC_ARG_ENABLE(asserts, AS_HELP_STRING([--enable-asserts], [Enable asserts (default)]), if test x$enableval = xno; then - AC_DEFINE(DISABLE_ASSERTS,, Disable asserts) + AC_DEFINE(DISABLE_ASSERTS,, [Disable asserts]) fi) AC_ARG_WITH(shared-libs, @@ -267,7 +267,7 @@ ], []) if test "$systemdsystemunitdir" != ""; then AC_SUBST(systemdsystemunitdir) - AC_DEFINE(HAVE_SYSTEMD,, Define if you want to use systemd socket activation) + AC_DEFINE(HAVE_SYSTEMD,, [Define if you want to use systemd socket activation]) fi AM_CONDITIONAL(HAVE_SYSTEMD, test "$systemdsystemunitdir" != "") @@ -318,9 +318,9 @@ AC_PROG_LIBTOOL AM_ICONV -AC_DEFINE_UNQUOTED(DOVECOT_NAME, "$PACKAGE_NAME", Dovecot name) -AC_DEFINE_UNQUOTED(DOVECOT_STRING, "$PACKAGE_STRING", Dovecot string) -AC_DEFINE_UNQUOTED(DOVECOT_VERSION, "$PACKAGE_VERSION", Dovecot version) +AC_DEFINE_UNQUOTED(DOVECOT_NAME, "$PACKAGE_NAME", [Dovecot name]) +AC_DEFINE_UNQUOTED(DOVECOT_STRING, "$PACKAGE_STRING", [Dovecot string]) +AC_DEFINE_UNQUOTED(DOVECOT_VERSION, "$PACKAGE_VERSION", [Dovecot version]) AC_DEFINE([DOVECOT_VERSION_MAJOR], regexp(AC_PACKAGE_VERSION, [^\([0-9]+\)\.\([0-9]+\)], [\1]), [Dovecot major version]) AC_DEFINE([DOVECOT_VERSION_MINOR], regexp(AC_PACKAGE_VERSION, [^\([0-9]+\)\.\([0-9]+\)], [\2]), [Dovecot minor version]) @@ -395,12 +395,12 @@ AC_SEARCH_LIBS(socket, socket) AC_SEARCH_LIBS(inet_addr, nsl) AC_SEARCH_LIBS(fdatasync, rt, [ - AC_DEFINE(HAVE_FDATASYNC,, Define if you have fdatasync()) + AC_DEFINE(HAVE_FDATASYNC,, [Define if you have fdatasync()]) ]) if test $want_libcap != no; then AC_CHECK_LIB(cap, cap_init, [ - AC_DEFINE(HAVE_LIBCAP,, libcap is installed for cap_init()) + AC_DEFINE(HAVE_LIBCAP,, [libcap is installed for cap_init()]) LIBCAP="-lcap" AC_SUBST(LIBCAP) ], [ @@ -429,7 +429,7 @@ ]) ]) if test $i_cv_have_libwrap = yes; then - AC_DEFINE(HAVE_LIBWRAP,, Define if you have libwrap) + AC_DEFINE(HAVE_LIBWRAP,, [Define if you have libwrap]) LIBWRAP_LIBS=-lwrap AC_SUBST(LIBWRAP_LIBS) have_libwrap=yes @@ -447,7 +447,7 @@ fi AM_CONDITIONAL(TCPWRAPPERS, test "$have_libwrap" = "yes") -AC_DEFINE(PACKAGE_WEBPAGE, "http://www.dovecot.org/", Support URL) +AC_DEFINE(PACKAGE_WEBPAGE, "http://www.dovecot.org/", [Support URL]) dnl * after -lsocket and -lnsl tests, inet_aton() may be in them AC_CHECK_FUNCS(fcntl flock lockf inet_aton sigaction getpagesize madvise \ @@ -464,7 +464,7 @@ ]) AC_SEARCH_LIBS(clock_gettime, rt, [ - AC_DEFINE(HAVE_CLOCK_GETTIME,, Define if you have the clock_gettime function) + AC_DEFINE(HAVE_CLOCK_GETTIME,, [Define if you have the clock_gettime function]) ]) AC_CACHE_CHECK([for typeof],i_cv_have_typeof,[ @@ -479,7 +479,7 @@ ]) ]) if test $i_cv_have_typeof = yes; then - AC_DEFINE(HAVE_TYPEOF,, Define if you have typeof()) + AC_DEFINE(HAVE_TYPEOF,, [Define if you have typeof()]) fi dnl strtoimax and strtoumax are macros in HP-UX, so inttypes.h must be included @@ -496,7 +496,7 @@ ]) ]) if test $i_cv_have_strtoimax = yes; then - AC_DEFINE(HAVE_STRTOIMAX,, Define if you have strtoimax function) + AC_DEFINE(HAVE_STRTOIMAX,, [Define if you have strtoimax function]) fi AC_CACHE_CHECK([for strtoumax],i_cv_have_strtoumax,[ @@ -511,7 +511,7 @@ ]) ]) if test $i_cv_have_strtoumax = yes; then - AC_DEFINE(HAVE_STRTOUMAX,, Define if you have strtoumax function) + AC_DEFINE(HAVE_STRTOUMAX,, [Define if you have strtoumax function]) fi dnl * I/O loop function @@ -533,7 +533,7 @@ ]) ]) if test $i_cv_epoll_works = yes; then - AC_DEFINE(IOLOOP_EPOLL,, Implement I/O loop with Linux 2.6 epoll()) + AC_DEFINE(IOLOOP_EPOLL,, [Implement I/O loop with Linux 2.6 epoll()]) have_ioloop=yes ioloop=epoll else @@ -555,14 +555,14 @@ if test "$ioloop" = "best" || test "$ioloop" = "poll"; then AC_CHECK_FUNC(poll, [ - AC_DEFINE(IOLOOP_POLL,, Implement I/O loop with poll()) + AC_DEFINE(IOLOOP_POLL,, [Implement I/O loop with poll()]) ioloop=poll have_ioloop=yes ]) fi if test "$have_ioloop" = "no"; then - AC_DEFINE(IOLOOP_SELECT,, Implement I/O loop with select()) + AC_DEFINE(IOLOOP_SELECT,, [Implement I/O loop with select()]) ioloop="select" fi @@ -575,7 +575,7 @@ have_notify=inotify notify=inotify AC_MSG_RESULT("yes") - AC_DEFINE(IOLOOP_NOTIFY_INOTIFY,, Use Linux inotify) + AC_DEFINE(IOLOOP_NOTIFY_INOTIFY,, [Use Linux inotify]) else AC_MSG_RESULT("no") if test "$notify" = "inotify"; then @@ -592,8 +592,7 @@ have_notify=kqueue notify=kqueue AC_MSG_RESULT("yes") - AC_DEFINE(IOLOOP_NOTIFY_KQUEUE,, - Use BSD kqueue directory changes notificaton) + AC_DEFINE(IOLOOP_NOTIFY_KQUEUE,, [Use BSD kqueue directory changes notificaton]) else AC_MSG_RESULT("no") if test "$notify" = "kqueue" ; then @@ -621,7 +620,7 @@ ]) ]) if test $i_cv_have_dnotify = yes; then - AC_DEFINE(IOLOOP_NOTIFY_DNOTIFY,, Use Linux dnotify) + AC_DEFINE(IOLOOP_NOTIFY_DNOTIFY,, [Use Linux dnotify]) have_notify=dnotify notify=dnotify else @@ -632,7 +631,7 @@ fi if test "$have_notify" = "none"; then - AC_DEFINE(IOLOOP_NOTIFY_NONE,, No special notify support) + AC_DEFINE(IOLOOP_NOTIFY_NONE,, [No special notify support]) fi AC_CACHE_CHECK([whether we have glibc],i_cv_have_glibc,[ @@ -648,7 +647,7 @@ ]) ]) if test "$i_cv_have_glibc" = "yes"; then - AC_DEFINE(PREAD_WRAPPERS,, Define if pread/pwrite needs _XOPEN_SOURCE 500) + AC_DEFINE(PREAD_WRAPPERS,, [Define if pread/pwrite needs _XOPEN_SOURCE 500]) fi dnl * Old glibcs have broken posix_fallocate(). Make sure not to use it. @@ -681,7 +680,7 @@ ]) ]) if test $i_cv_posix_fallocate_works = yes; then - AC_DEFINE(HAVE_POSIX_FALLOCATE,, Define if you have a working posix_fallocate()) + AC_DEFINE(HAVE_POSIX_FALLOCATE,, [Define if you have a working posix_fallocate()]) fi dnl * OS specific options @@ -692,10 +691,10 @@ # for getting fd_send/fd_recv working: LDFLAGS="$LDFLAGS -Wl,+b,:" LIBS="-lxnet $LIBS" - AC_DEFINE(PREAD_BROKEN,, Defint if pread/pwrite implementation is broken) + AC_DEFINE(PREAD_BROKEN,, [Define if pread/pwrite implementation is broken]) ;; linux*|darwin*) - AC_DEFINE(PROCTITLE_HACK,, Define if process title can be changed by modifying argv) + AC_DEFINE(PROCTITLE_HACK,, [Define if process title can be changed by modifying argv]) ;; *) ;; @@ -832,7 +831,7 @@ AC_CHECK_TYPE(uoff_t, [ have_uoff_t=yes - AC_DEFINE(HAVE_UOFF_T,, Define if you have a native uoff_t type) + AC_DEFINE(HAVE_UOFF_T,, [Define if you have a native uoff_t type]) ], [ have_uoff_t=no ]) @@ -843,7 +842,7 @@ offt_max=INT_MAX uofft_fmt="u" if test "$have_uoff_t" != "yes"; then - AC_DEFINE(UOFF_T_INT,, Define if off_t is int) + AC_DEFINE(UOFF_T_INT,, [Define if off_t is int]) fi offt_bits=`expr 8 \* $ac_cv_sizeof_int` ;; @@ -851,7 +850,7 @@ offt_max=LONG_MAX uofft_fmt="lu" if test "$have_uoff_t" != "yes"; then - AC_DEFINE(UOFF_T_LONG,, Define if off_t is long) + AC_DEFINE(UOFF_T_LONG,, [Define if off_t is long]) fi offt_bits=`expr 8 \* $ac_cv_sizeof_long` ;; @@ -859,7 +858,7 @@ offt_max=LLONG_MAX uofft_fmt="llu" if test "$have_uoff_t" != "yes"; then - AC_DEFINE(UOFF_T_LONG_LONG,, Define if off_t is long long) + AC_DEFINE(UOFF_T_LONG_LONG,, [Define if off_t is long long]) fi offt_bits=`expr 8 \* $ac_cv_sizeof_long_long` ;; @@ -875,7 +874,7 @@ struct dirent d; d.d_type = DT_DIR; ]])],[ - AC_DEFINE(HAVE_DIRENT_D_TYPE,, Define if you have struct dirent->d_type) + AC_DEFINE(HAVE_DIRENT_D_TYPE,, [Define if you have struct dirent->d_type]) ],[]) dnl * Do we have OFF_T_MAX? @@ -887,10 +886,10 @@ ]])],[ : ],[ - AC_DEFINE_UNQUOTED(OFF_T_MAX, $offt_max, Maximum value of off_t) + AC_DEFINE_UNQUOTED(OFF_T_MAX, $offt_max, [Maximum value of off_t]) ]) -AC_DEFINE_UNQUOTED(PRIuUOFF_T, "$uofft_fmt", printf() format for uoff_t) +AC_DEFINE_UNQUOTED(PRIuUOFF_T, "$uofft_fmt", [printf() format for uoff_t]) dnl * make sure size_t isn't signed. we'd probably work fine with it, but dnl * it's more likely vulnerable to buffer overflows. Anyway, C99 specifies @@ -938,14 +937,14 @@ sizet_fmt="u" if test "$typeof_size_t" = ""; then - AC_DEFINE(size_t, unsigned int, Define to 'unsigned int' if you don't have it) - AC_DEFINE(ssize_t, int, Define to 'int' if you don't have it) + AC_DEFINE(size_t, unsigned int, [Define to 'unsigned int' if you don't have it]) + AC_DEFINE(ssize_t, int, [Define to 'int' if you don't have it]) fi ;; esac -AC_DEFINE_UNQUOTED(SSIZE_T_MAX, $ssizet_max, Maximum value of ssize_t) -AC_DEFINE_UNQUOTED(PRIuSIZE_T, "$sizet_fmt", printf() format for size_t) +AC_DEFINE_UNQUOTED(SSIZE_T_MAX, $ssizet_max, [Maximum value of ssize_t]) +AC_DEFINE_UNQUOTED(PRIuSIZE_T, "$sizet_fmt", [printf() format for size_t]) AC_DEFUN([AC_CHECKTYPE2], [ AC_MSG_CHECKING([for $1]) @@ -964,27 +963,27 @@ AC_CHECKTYPE2(uintmax_t, [$stdint_include]) if test $i_cv_type_uintmax_t = yes; then - AC_DEFINE(HAVE_UINTMAX_T,, Define if you have uintmax_t (C99 type)) + AC_DEFINE(HAVE_UINTMAX_T,, [Define if you have uintmax_t (C99 type)]) fi dnl use separate check, eg. Solaris 8 has uintmax_t but not uint_fast32_t AC_CHECKTYPE2(uint_fast32_t, [$stdint_include]) if test $i_cv_type_uint_fast32_t = yes; then - AC_DEFINE(HAVE_UINT_FAST32_T,, Define if you have uint_fast32_t (C99 type)) + AC_DEFINE(HAVE_UINT_FAST32_T,, [Define if you have uint_fast32_t (C99 type)]) fi AC_CHECKTYPE2(socklen_t, [#include ]) if test $i_cv_type_socklen_t = yes; then - AC_DEFINE(HAVE_SOCKLEN_T,, Define to 'int' if you don't have socklen_t) From dovecot at dovecot.org Thu May 14 11:04:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 14 May 2015 11:04:14 +0000 Subject: dovecot-2.2: lib-fts: Fixed test-fts-filter when building withou... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d09429bdb340 changeset: 18691:d09429bdb340 user: Timo Sirainen date: Thu May 14 14:02:13 2015 +0300 description: lib-fts: Fixed test-fts-filter when building without libstemmer diffstat: src/lib-fts/test-fts-filter.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (30 lines): diff -r a2c3a0f671a6 -r d09429bdb340 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Thu May 14 12:36:38 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Thu May 14 14:02:13 2015 +0300 @@ -478,6 +478,7 @@ test_end(); } +#ifdef HAVE_FTS_STEMMER static void test_fts_filter_normalizer_stopwords_stemmer_eng(void) { int ret; @@ -533,6 +534,7 @@ test_end(); } #endif +#endif /* TODO: Functions to test 1. ref-unref pairs 2. multiple registers + an unregister + find */ @@ -557,8 +559,10 @@ test_fts_filter_normalizer_french, test_fts_filter_normalizer_empty, test_fts_filter_normalizer_invalid_id, +#ifdef HAVE_FTS_STEMMER test_fts_filter_normalizer_stopwords_stemmer_eng, #endif +#endif NULL }; int ret; From pigeonhole at rename-it.nl Thu May 14 11:13:44 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 14 May 2015 13:13:44 +0200 Subject: dovecot-2.2-pigeonhole: configure: Use consistent quotation for ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/0c4ae064f307 changeset: 2069:0c4ae064f307 user: Stephan Bosch date: Thu May 14 13:13:39 2015 +0200 description: configure: Use consistent quotation for AC_DEFINEs as per autoconf recommendations. diffstat: configure.ac | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diffs (29 lines): diff -r b491aa4139bc -r 0c4ae064f307 configure.ac --- a/configure.ac Wed May 13 23:13:42 2015 +0200 +++ b/configure.ac Thu May 14 13:13:39 2015 +0200 @@ -122,10 +122,10 @@ AC_CHECK_LIB(ldap, ldap_init, [ AC_CHECK_HEADER(ldap.h, [ AC_CHECK_LIB(ldap, ldap_initialize, [ - AC_DEFINE(LDAP_HAVE_INITIALIZE,, Define if you have ldap_initialize) + AC_DEFINE(LDAP_HAVE_INITIALIZE,, [Define if you have ldap_initialize]) ]) AC_CHECK_LIB(ldap, ldap_start_tls_s, [ - AC_DEFINE(LDAP_HAVE_START_TLS_S,, Define if you have ldap_start_tls_s) + AC_DEFINE(LDAP_HAVE_START_TLS_S,, [Define if you have ldap_start_tls_s]) ]) LDAP_LIBS="-lldap" AC_CHECK_LIB(ldap, ber_free, [ @@ -138,10 +138,10 @@ ]) AC_SUBST(LDAP_LIBS) if test $want_ldap != plugin; then - AC_DEFINE(SIEVE_BUILTIN_LDAP,, LDAP support is built in) + AC_DEFINE(SIEVE_BUILTIN_LDAP,, [LDAP support is built in]) fi - AC_DEFINE(STORAGE_LDAP,, Build with LDAP support) + AC_DEFINE(STORAGE_LDAP,, [Build with LDAP support]) AC_CHECK_HEADERS(sasl.h sasl/sasl.h) have_ldap=yes ], [ From dovecot at dovecot.org Thu May 14 18:46:51 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 14 May 2015 18:46:51 +0000 Subject: dovecot-2.2: lib-imap: Fixed crash in IDLE for non-TCP connections. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4c4eff229426 changeset: 18692:4c4eff229426 user: Timo Sirainen date: Thu May 14 21:44:50 2015 +0300 description: lib-imap: Fixed crash in IDLE for non-TCP connections. diffstat: src/lib-imap/imap-keepalive.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d09429bdb340 -r 4c4eff229426 src/lib-imap/imap-keepalive.c --- a/src/lib-imap/imap-keepalive.c Thu May 14 14:02:13 2015 +0300 +++ b/src/lib-imap/imap-keepalive.c Thu May 14 21:44:50 2015 +0300 @@ -42,7 +42,7 @@ { unsigned int client_hash; - client_hash = imap_remote_ip_is_usable(ip) ? + client_hash = ip != NULL && imap_remote_ip_is_usable(ip) ? net_ip_hash(ip) : crc32_str(username); interval_secs -= (time(NULL) + client_hash) % interval_secs; return interval_secs * 1000; From dovecot at dovecot.org Thu May 14 20:51:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 14 May 2015 20:51:13 +0000 Subject: dovecot-2.2: lib: Use /dev/zero instead of /dev/stderr in test-i... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/91bbd6753689 changeset: 18693:91bbd6753689 user: Timo Sirainen date: Thu May 14 23:49:12 2015 +0300 description: lib: Use /dev/zero instead of /dev/stderr in test-istream-unix unit test Some distros don't seem to allow opening stderr. diffstat: src/lib/test-istream-unix.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 4c4eff229426 -r 91bbd6753689 src/lib/test-istream-unix.c --- a/src/lib/test-istream-unix.c Thu May 14 21:44:50 2015 +0300 +++ b/src/lib/test-istream-unix.c Thu May 14 23:49:12 2015 +0300 @@ -166,8 +166,8 @@ test_begin("istream unix"); if ((send_fd = open("/dev/null", O_RDONLY)) == -1) i_fatal("open(/dev/null) failed: %m"); - if ((send_fd2 = open("/dev/stderr", O_WRONLY)) == -1) - i_fatal("open(/dev/stderr) failed: %m"); + if ((send_fd2 = open("/dev/zero", O_RDONLY)) == -1) + i_fatal("open(/dev/zero) failed: %m"); if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) < 0) i_fatal("socketpair() failed: %m"); switch (fork()) { From dovecot at dovecot.org Fri May 15 08:17:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 08:17:50 +0000 Subject: dovecot-2.2: sdbox: When hardlink-copying a mail in alt storage,... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9ad3f11617f1 changeset: 18694:9ad3f11617f1 user: Timo Sirainen date: Fri May 15 11:15:50 2015 +0300 description: sdbox: When hardlink-copying a mail in alt storage, keep it within the alt storage. Also make sure that the alt-flag is set correctly no matter where we copy the mail. diffstat: src/lib-storage/index/dbox-single/sdbox-copy.c | 16 +++++++++++----- src/lib-storage/index/dbox-single/sdbox-file.c | 14 +++++++------- 2 files changed, 18 insertions(+), 12 deletions(-) diffs (81 lines): diff -r 91bbd6753689 -r 9ad3f11617f1 src/lib-storage/index/dbox-single/sdbox-copy.c --- a/src/lib-storage/index/dbox-single/sdbox-copy.c Thu May 14 23:49:12 2015 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-copy.c Fri May 15 11:15:50 2015 +0300 @@ -89,7 +89,7 @@ (struct sdbox_mailbox *)_ctx->transaction->box; struct sdbox_mailbox *src_mbox; struct dbox_file *src_file, *dest_file; - const char *src_path; + const char *src_path, *dest_path; int ret; if (strcmp(mail->box->storage->name, SDBOX_STORAGE_NAME) == 0) @@ -102,11 +102,18 @@ src_file = sdbox_file_init(src_mbox, mail->uid); dest_file = sdbox_file_init(dest_mbox, 0); + ctx->ctx.data.flags &= ~DBOX_INDEX_FLAG_ALT; + src_path = src_file->primary_path; - ret = nfs_safe_link(src_path, dest_file->cur_path, FALSE); + dest_path = dest_file->primary_path; + ret = nfs_safe_link(src_path, dest_path, FALSE); if (ret < 0 && errno == ENOENT && src_file->alt_path != NULL) { src_path = src_file->alt_path; - ret = nfs_safe_link(src_path, dest_file->cur_path, FALSE); + if (dest_file->alt_path != NULL) { + dest_path = dest_file->cur_path = dest_file->alt_path; + ctx->ctx.data.flags |= DBOX_INDEX_FLAG_ALT; + } + ret = nfs_safe_link(src_path, dest_path, FALSE); } if (ret < 0) { if (ECANTLINK(errno)) @@ -119,8 +126,7 @@ } else { mail_storage_set_critical( _ctx->transaction->box->storage, - "link(%s, %s) failed: %m", - src_path, dest_file->cur_path); + "link(%s, %s) failed: %m", src_path, dest_path); } dbox_file_unref(&src_file); dbox_file_unref(&dest_file); diff -r 91bbd6753689 -r 9ad3f11617f1 src/lib-storage/index/dbox-single/sdbox-file.c --- a/src/lib-storage/index/dbox-single/sdbox-file.c Thu May 14 23:49:12 2015 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-file.c Fri May 15 11:15:50 2015 +0300 @@ -44,10 +44,7 @@ sdbox_file_init_paths(file, fname); file->uid = uid; } else { - file->file.primary_path = - i_strdup_printf("%s/%s", - mailbox_get_path(&mbox->box), - dbox_generate_tmp_filename()); + sdbox_file_init_paths(file, dbox_generate_tmp_filename()); } } T_END; dbox_file_init(&file->file); @@ -151,16 +148,19 @@ int sdbox_file_assign_uid(struct sdbox_file *file, uint32_t uid) { - const char *old_path, *new_fname, *new_path; + const char *p, *old_path, *dir, *new_fname, *new_path; struct stat st; i_assert(file->uid == 0); i_assert(uid != 0); old_path = file->file.cur_path; + p = strrchr(old_path, '/'); + i_assert(p != NULL); + dir = t_strdup_until(old_path, p); + new_fname = t_strdup_printf(SDBOX_MAIL_FILE_FORMAT, uid); - new_path = t_strdup_printf("%s/%s", mailbox_get_path(&file->mbox->box), - new_fname); + new_path = t_strdup_printf("%s/%s", dir, new_fname); if (stat(new_path, &st) == 0) { mail_storage_set_critical(&file->file.storage->storage, From dovecot at dovecot.org Fri May 15 09:45:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 09:45:15 +0000 Subject: dovecot-2.2: quota: Fixed dependency tracking in Makefile Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/46969c4cc57e changeset: 18695:46969c4cc57e user: Timo Sirainen date: Fri May 15 12:43:13 2015 +0300 description: quota: Fixed dependency tracking in Makefile The earlier fix attempt was done by creating a .tmp file first, but that didn't really solve anything. diffstat: src/plugins/quota/Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 9ad3f11617f1 -r 46969c4cc57e src/plugins/quota/Makefile.am --- a/src/plugins/quota/Makefile.am Fri May 15 11:15:50 2015 +0300 +++ b/src/plugins/quota/Makefile.am Fri May 15 12:43:13 2015 +0300 @@ -73,7 +73,7 @@ RQUOTA_XDR_LO = rquota_xdr.lo #RQUOTA_X = /usr/include/rpcsvc/rquota.x RQUOTA_X = $(srcdir)/rquota.x -rquota_xdr.c: Makefile $(RQUOTA_X) +rquota_xdr.c: Makefile rquota.h if [ "$(top_srcdir)" != "$(top_builddir)" ]; then \ cp $(RQUOTA_X) $(top_builddir)/src/plugins/quota/; \ fi; \ From dovecot at dovecot.org Fri May 15 09:50:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 09:50:11 +0000 Subject: dovecot-2.2: Reverted d592417ec815 which added unnecessary code ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/55bcb0e792fc changeset: 18696:55bcb0e792fc user: Timo Sirainen date: Fri May 15 12:48:07 2015 +0300 description: Reverted d592417ec815 which added unnecessary code to Makefiles. The original problem it tried to solve was properly fixed by 46969c4cc57e. make will actually wait for processes to finish creating files before it continues to the next program that wants to access the file. As long as the dependencies are correct. diffstat: src/config/Makefile.am | 2 +- src/lib-dict/Makefile.am | 33 +++++++++++++++---------------- src/lib-fts/Makefile.am | 4 +- src/lib-sql/Makefile.am | 17 +++++++-------- src/lib-storage/register/Makefile.am | 38 +++++++++++++++++------------------ src/lib/Makefile.am | 2 +- src/plugins/quota/Makefile.am | 5 +-- 7 files changed, 48 insertions(+), 53 deletions(-) diffs (211 lines): diff -r 46969c4cc57e -r 55bcb0e792fc src/config/Makefile.am --- a/src/config/Makefile.am Fri May 15 12:43:13 2015 +0300 +++ b/src/config/Makefile.am Fri May 15 12:48:07 2015 +0300 @@ -55,7 +55,7 @@ config-request.h all-settings.c: $(SETTING_FILES) $(top_srcdir)/src/config/settings-get.pl - $(top_srcdir)/src/config/settings-get.pl $(SETTING_FILES) > all-settings.c.tmp && mv all-settings.c.tmp all-settings.c + $(top_srcdir)/src/config/settings-get.pl $(SETTING_FILES) > all-settings.c || rm -f all-settings.c EXTRA_DIST = \ config-settings.c \ diff -r 46969c4cc57e -r 55bcb0e792fc src/lib-dict/Makefile.am --- a/src/lib-dict/Makefile.am Fri May 15 12:43:13 2015 +0300 +++ b/src/lib-dict/Makefile.am Fri May 15 12:48:07 2015 +0300 @@ -46,35 +46,34 @@ pkginc_lib_HEADERS = $(headers) dict-drivers-register.c: Makefile $(top_builddir)/config.h - rm -f $@.tmp - echo '/* this file automatically generated by Makefile */' >$@.tmp - echo '#include "lib.h"' >>$@.tmp - echo '#include "dict.h"' >>$@.tmp - echo '#include "dict-sql.h"' >>$@.tmp + rm -f $@ + echo '/* this file automatically generated by Makefile */' >$@ + echo '#include "lib.h"' >>$@ + echo '#include "dict.h"' >>$@ + echo '#include "dict-sql.h"' >>$@ for i in $(dict_drivers) null; do \ if [ "$${i}" != "null" ]; then \ - echo "extern struct dict dict_driver_$${i};" >>$@.tmp ; \ + echo "extern struct dict dict_driver_$${i};" >>$@ ; \ fi; \ done - echo 'void dict_drivers_register_all(void) {' >>$@.tmp - echo 'dict_drivers_register_builtin();' >>$@.tmp - echo 'dict_sql_register();' >>$@.tmp + echo 'void dict_drivers_register_all(void) {' >>$@ + echo 'dict_drivers_register_builtin();' >>$@ + echo 'dict_sql_register();' >>$@ for i in $(dict_drivers) null; do \ if [ "$${i}" != "null" ]; then \ - echo "dict_driver_register(&dict_driver_$${i});" >>$@.tmp ; \ + echo "dict_driver_register(&dict_driver_$${i});" >>$@ ; \ fi; \ done - echo '}' >>$@.tmp - echo 'void dict_drivers_unregister_all(void) {' >>$@.tmp - echo 'dict_drivers_unregister_builtin();' >>$@.tmp - echo 'dict_sql_unregister();' >>$@.tmp + echo '}' >>$@ + echo 'void dict_drivers_unregister_all(void) {' >>$@ + echo 'dict_drivers_unregister_builtin();' >>$@ + echo 'dict_sql_unregister();' >>$@ for i in $(dict_drivers) null; do \ if [ "$${i}" != "null" ]; then \ - echo "dict_driver_unregister(&dict_driver_$${i});" >>$@.tmp ; \ + echo "dict_driver_unregister(&dict_driver_$${i});" >>$@ ; \ fi; \ done - echo '}' >>$@.tmp - mv $@.tmp $@ + echo '}' >>$@ distclean-generic: rm -f Makefile dict-drivers-register.c diff -r 46969c4cc57e -r 55bcb0e792fc src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Fri May 15 12:43:13 2015 +0300 +++ b/src/lib-fts/Makefile.am Fri May 15 12:48:07 2015 +0300 @@ -29,12 +29,12 @@ WordBreakProperty.txt: test -f WordBreakProperty.txt || wget http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakProperty.txt $(srcdir)/word-boundary-data.c: word-properties.pl WordBreakProperty.txt - perl word-properties.pl boundaries WordBreakProperty.txt > $@.tmp && mv $@.tmp $@ + perl word-properties.pl boundaries WordBreakProperty.txt > $@ PropList.txt: test -f PropList.txt || wget http://www.unicode.org/Public/UNIDATA/PropList.txt $(srcdir)/word-break-data.c: word-properties.pl PropList.txt - perl word-properties.pl breaks PropList.txt > $@.tmp && mv $@.tmp $@ + perl word-properties.pl breaks PropList.txt > $@ if BUILD_FTS_STEMMER diff -r 46969c4cc57e -r 55bcb0e792fc src/lib-sql/Makefile.am --- a/src/lib-sql/Makefile.am Fri May 15 12:43:13 2015 +0300 +++ b/src/lib-sql/Makefile.am Fri May 15 12:48:07 2015 +0300 @@ -104,27 +104,26 @@ pkginc_lib_HEADERS = $(headers) sql-drivers-register.c: Makefile - rm -f $@.tmp - echo '/* this file automatically generated by Makefile */' >$@.tmp - echo '#include "lib.h"' >>$@.tmp - echo '#include "sql-api.h"' >>$@.tmp + rm -f $@ + echo '/* this file automatically generated by Makefile */' >$@ + echo '#include "lib.h"' >>$@ + echo '#include "sql-api.h"' >>$@ if ! SQL_PLUGINS for i in $(sql_drivers) null; do \ if [ "$${i}" != "null" ]; then \ - echo "extern struct sql_db driver_$${i}_db;" >>$@.tmp ; \ + echo "extern struct sql_db driver_$${i}_db;" >>$@ ; \ fi; \ done endif - echo 'void sql_drivers_register_all(void) {' >>$@.tmp + echo 'void sql_drivers_register_all(void) {' >>$@ if ! SQL_PLUGINS for i in $(sql_drivers) null; do \ if [ "$${i}" != "null" ]; then \ - echo "sql_driver_register(&driver_$${i}_db);" >>$@.tmp ; \ + echo "sql_driver_register(&driver_$${i}_db);" >>$@ ; \ fi; \ done endif - echo '}' >>$@.tmp - mv $@.tmp $@ + echo '}' >>$@ if SQL_PLUGINS install-exec-local: diff -r 46969c4cc57e -r 55bcb0e792fc src/lib-storage/register/Makefile.am --- a/src/lib-storage/register/Makefile.am Fri May 15 12:43:13 2015 +0300 +++ b/src/lib-storage/register/Makefile.am Fri May 15 12:48:07 2015 +0300 @@ -5,36 +5,34 @@ mailbox_list_drivers = @mailbox_list_drivers@ mail-storage-register.c: Makefile - rm -f $@.tmp - echo '/* this file automatically generated by Makefile */' >$@.tmp - echo '#include "lib.h"' >>$@.tmp - echo '#include "mail-storage.h"' >>$@.tmp + rm -f $@ + echo '/* this file automatically generated by Makefile */' >$@ + echo '#include "lib.h"' >>$@ + echo '#include "mail-storage.h"' >>$@ for i in $(mail_storages) ; do \ - echo "extern struct mail_storage $${i}_storage;" >>$@.tmp ; \ + echo "extern struct mail_storage $${i}_storage;" >>$@ ; \ done - echo 'void mail_storage_register_all(void) {' >>$@.tmp + echo 'void mail_storage_register_all(void) {' >>$@ for i in $(mail_storages) ; do \ - echo "mail_storage_class_register(&$${i}_storage);" >>$@.tmp ; \ + echo "mail_storage_class_register(&$${i}_storage);" >>$@ ; \ done - echo '}' >>$@.tmp - mv $@.tmp $@ + echo '}' >>$@ mailbox-list-register.c: Makefile - rm -f $@.tmp - echo '/* this file automatically generated by Makefile */' >$@.tmp - echo '#include "lib.h"' >>$@.tmp - echo '#include "mailbox-list.h"' >>$@.tmp + rm -f $@ + echo '/* this file automatically generated by Makefile */' >$@ + echo '#include "lib.h"' >>$@ + echo '#include "mailbox-list.h"' >>$@ for i in $(mailbox_list_drivers) ; do \ - echo "extern struct mailbox_list $${i}_mailbox_list;" >>$@.tmp ; \ + echo "extern struct mailbox_list $${i}_mailbox_list;" >>$@ ; \ done - echo "void mailbox_list_index_init(void);" >>$@.tmp - echo 'void mailbox_list_register_all(void) {' >>$@.tmp + echo "void mailbox_list_index_init(void);" >>$@ + echo 'void mailbox_list_register_all(void) {' >>$@ for i in $(mailbox_list_drivers) ; do \ - echo "mailbox_list_register(&$${i}_mailbox_list);" >>$@.tmp ; \ + echo "mailbox_list_register(&$${i}_mailbox_list);" >>$@ ; \ done - echo "mailbox_list_index_init();" >>$@.tmp - echo '}' >>$@.tmp - mv $@.tmp $@ + echo "mailbox_list_index_init();" >>$@ + echo '}' >>$@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ diff -r 46969c4cc57e -r 55bcb0e792fc src/lib/Makefile.am --- a/src/lib/Makefile.am Fri May 15 12:43:13 2015 +0300 +++ b/src/lib/Makefile.am Fri May 15 12:48:07 2015 +0300 @@ -8,7 +8,7 @@ test -f UnicodeData.txt || wget http://www.unicode.org/Public/UNIDATA/UnicodeData.txt $(srcdir)/unicodemap.c: unicodemap.pl UnicodeData.txt - perl $(srcdir)/unicodemap.pl < UnicodeData.txt > $@.tmp && mv $@.tmp $@ + perl $(srcdir)/unicodemap.pl < UnicodeData.txt > $@ liblib_la_SOURCES = \ abspath.c \ diff -r 46969c4cc57e -r 55bcb0e792fc src/plugins/quota/Makefile.am --- a/src/plugins/quota/Makefile.am Fri May 15 12:43:13 2015 +0300 +++ b/src/plugins/quota/Makefile.am Fri May 15 12:48:07 2015 +0300 @@ -83,11 +83,10 @@ sed -e 's/IXDR_PUT/(void)IXDR_PUT/g' \ -e 's,/usr/include/rpcsvc/rquota.h,rquota.h,' \ -e 's/int32_t \*buf/int32_t *buf ATTR_UNUSED/' \ - -e 's/^static char rcsid.*//' ) > rquota_xdr.c.tmp; \ - mv rquota_xdr.c.tmp rquota_xdr.c + -e 's/^static char rcsid.*//' ) > rquota_xdr.c rquota.h: Makefile $(RQUOTA_X) - $(RPCGEN) -h $(RQUOTA_X) > rquota.h.tmp && mv rquota.h.tmp rquota.h + $(RPCGEN) -h $(RQUOTA_X) > rquota.h quota-fs.lo: rquota.h From dovecot at dovecot.org Fri May 15 10:09:29 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 10:09:29 +0000 Subject: dovecot-2.2: pgsql: Log a warning if DNS lookup takes too long. ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/db41043041cb changeset: 18697:db41043041cb user: Timo Sirainen date: Fri May 15 13:07:27 2015 +0300 description: pgsql: Log a warning if DNS lookup takes too long. Don't include it in connect timeout. diffstat: src/lib-sql/driver-pgsql.c | 17 +++++++++++++++++ 1 files changed, 17 insertions(+), 0 deletions(-) diffs (50 lines): diff -r 55bcb0e792fc -r db41043041cb src/lib-sql/driver-pgsql.c --- a/src/lib-sql/driver-pgsql.c Fri May 15 12:48:07 2015 +0300 +++ b/src/lib-sql/driver-pgsql.c Fri May 15 13:07:27 2015 +0300 @@ -3,12 +3,15 @@ #include "lib.h" #include "array.h" #include "ioloop.h" +#include "time-util.h" #include "sql-api-private.h" #ifdef BUILD_PGSQL #include #include +#define PGSQL_DNS_WARN_MSECS 500 + struct pgsql_db { struct sql_db api; @@ -195,9 +198,14 @@ static int driver_pgsql_connect(struct sql_db *_db) { struct pgsql_db *db = (struct pgsql_db *)_db; + struct timeval tv_start; + int msecs; i_assert(db->api.state == SQL_DB_STATE_DISCONNECTED); + io_loop_time_refresh(); + tv_start = ioloop_timeval; + db->pg = PQconnectStart(db->connect_string); if (db->pg == NULL) { i_fatal("%s: PQconnectStart() failed (out of memory)", @@ -210,6 +218,15 @@ driver_pgsql_close(db); return -1; } + /* PQconnectStart() blocks on host name resolving. Log a warning if + it takes too long. Also don't include time spent on that in the + connect timeout (by refreshing ioloop time). */ + io_loop_time_refresh(); + msecs = timeval_diff_msecs(&ioloop_timeval, &tv_start); + if (msecs > PGSQL_DNS_WARN_MSECS) { + i_warning("%s: DNS lookup took %d.%03d s", + pgsql_prefix(db), msecs/1000, msecs % 1000); + } /* nonblocking connecting begins. */ if (PQsetnonblocking(db->pg, 1) < 0) From dovecot at dovecot.org Fri May 15 10:29:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 10:29:30 +0000 Subject: dovecot-2.2: pgsql: Don't crash at disconnect/deinit if there's ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b19ca4214e63 changeset: 18698:b19ca4214e63 user: Timo Sirainen date: Fri May 15 13:27:29 2015 +0300 description: pgsql: Don't crash at disconnect/deinit if there's an unfinished query. diffstat: src/lib-sql/driver-pgsql.c | 12 +++++------- 1 files changed, 5 insertions(+), 7 deletions(-) diffs (29 lines): diff -r db41043041cb -r b19ca4214e63 src/lib-sql/driver-pgsql.c --- a/src/lib-sql/driver-pgsql.c Fri May 15 13:07:27 2015 +0300 +++ b/src/lib-sql/driver-pgsql.c Fri May 15 13:27:29 2015 +0300 @@ -245,8 +245,10 @@ { struct pgsql_db *db = (struct pgsql_db *)_db; - if (db->cur_result != NULL && db->cur_result->to != NULL) - result_finish(db->cur_result); + if (db->cur_result != NULL && db->cur_result->to != NULL) { + driver_pgsql_stop_io(db); + result_finish(db->cur_result); + } _db->no_reconnect = TRUE; driver_pgsql_close(db); @@ -276,11 +278,7 @@ { struct pgsql_db *db = (struct pgsql_db *)_db; - if (db->cur_result != NULL && db->cur_result->to != NULL) - result_finish(db->cur_result); - - _db->no_reconnect = TRUE; - driver_pgsql_close(db); + driver_pgsql_disconnect(_db); i_free(db->host); i_free(db->error); i_free(db->connect_string); From dovecot at dovecot.org Fri May 15 10:57:53 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 10:57:53 +0000 Subject: dovecot-2.2: replication plugin: A small optimization to check t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6a40cb15e48f changeset: 18699:6a40cb15e48f user: Timo Sirainen date: Fri May 15 13:48:45 2015 +0300 description: replication plugin: A small optimization to check the user's "dsyncing" status only once. diffstat: src/plugins/replication/replication-plugin.c | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-) diffs (29 lines): diff -r b19ca4214e63 -r 6a40cb15e48f src/plugins/replication/replication-plugin.c --- a/src/plugins/replication/replication-plugin.c Fri May 15 13:27:29 2015 +0300 +++ b/src/plugins/replication/replication-plugin.c Fri May 15 13:48:45 2015 +0300 @@ -184,12 +184,6 @@ { struct replication_user *ruser; - if (ns->user->dsyncing) { - /* we're running dsync, which means that the remote is telling - us about a change. don't trigger a replication back to it */ - return; - } - ruser = REPLICATION_USER_CONTEXT(ns->user); if (ruser == NULL) return; @@ -320,6 +314,12 @@ if (value == NULL || value[0] == '\0') return; + if (user->dsyncing) { + /* we're running dsync, which means that the remote is telling + us about a change. don't trigger a replication back to it */ + return; + } + ruser = p_new(user->pool, struct replication_user, 1); ruser->module_ctx.super = *v; user->vlast = &ruser->module_ctx.super; From dovecot at dovecot.org Fri May 15 10:57:54 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 10:57:54 +0000 Subject: dovecot-2.2: replication plugin: Added debug logging when mail_d... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7cf62f28fd62 changeset: 18700:7cf62f28fd62 user: Timo Sirainen date: Fri May 15 13:54:49 2015 +0300 description: replication plugin: Added debug logging when mail_debug=yes diffstat: src/plugins/replication/replication-plugin.c | 25 ++++++++++++++++++------- 1 files changed, 18 insertions(+), 7 deletions(-) diffs (88 lines): diff -r 6a40cb15e48f -r 7cf62f28fd62 src/plugins/replication/replication-plugin.c --- a/src/plugins/replication/replication-plugin.c Fri May 15 13:48:45 2015 +0300 +++ b/src/plugins/replication/replication-plugin.c Fri May 15 13:54:49 2015 +0300 @@ -180,7 +180,8 @@ } static void replication_notify(struct mail_namespace *ns, - enum replication_priority priority) + enum replication_priority priority, + const char *event) { struct replication_user *ruser; @@ -188,6 +189,11 @@ if (ruser == NULL) return; + if (ns->user->mail_debug) { + i_debug("replication: Replication requested by '%s', priority=%d", + event, priority); + } + if (priority == REPLICATION_PRIORITY_SYNC) { if (replication_notify_sync(ns->user) == 0) { timeout_remove(&ruser->to); @@ -254,7 +260,7 @@ priority = !ctx->new_messages ? REPLICATION_PRIORITY_LOW : ruser->sync_secs == 0 ? REPLICATION_PRIORITY_HIGH : REPLICATION_PRIORITY_SYNC; - replication_notify(ctx->ns, priority); + replication_notify(ctx->ns, priority, "transction commit"); } i_free(ctx); } @@ -262,7 +268,7 @@ static void replication_mailbox_create(struct mailbox *box) { replication_notify(mailbox_get_namespace(box), - REPLICATION_PRIORITY_LOW); + REPLICATION_PRIORITY_LOW, "mailbox create"); } static void @@ -270,7 +276,7 @@ struct mailbox *box) { replication_notify(mailbox_get_namespace(box), - REPLICATION_PRIORITY_LOW); + REPLICATION_PRIORITY_LOW, "mailbox delete"); } static void @@ -278,14 +284,14 @@ struct mailbox *dest) { replication_notify(mailbox_get_namespace(dest), - REPLICATION_PRIORITY_LOW); + REPLICATION_PRIORITY_LOW, "mailbox rename"); } static void replication_mailbox_set_subscribed(struct mailbox *box, bool subscribed ATTR_UNUSED) { replication_notify(mailbox_get_namespace(box), - REPLICATION_PRIORITY_LOW); + REPLICATION_PRIORITY_LOW, "mailbox subscribe"); } static void replication_user_deinit(struct mail_user *user) @@ -311,12 +317,17 @@ const char *value; value = mail_user_plugin_getenv(user, "mail_replica"); - if (value == NULL || value[0] == '\0') + if (value == NULL || value[0] == '\0') { + if (user->mail_debug) + i_debug("replication: No mail_replica setting - replication disabled"); return; + } if (user->dsyncing) { /* we're running dsync, which means that the remote is telling us about a change. don't trigger a replication back to it */ + if (user->mail_debug) + i_debug("replication: We're running dsync - replication disabled"); return; } From dovecot at dovecot.org Fri May 15 10:57:54 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 10:57:54 +0000 Subject: dovecot-2.2: replication plugin: Fixed handling EAGAIN errors wh... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d46270397a2f changeset: 18701:d46270397a2f user: Timo Sirainen date: Fri May 15 13:55:46 2015 +0300 description: replication plugin: Fixed handling EAGAIN errors when writing to replication-pipe. diffstat: src/plugins/replication/replication-plugin.c | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-) diffs (26 lines): diff -r 7cf62f28fd62 -r d46270397a2f src/plugins/replication/replication-plugin.c --- a/src/plugins/replication/replication-plugin.c Fri May 15 13:54:49 2015 +0300 +++ b/src/plugins/replication/replication-plugin.c Fri May 15 13:55:46 2015 +0300 @@ -80,16 +80,16 @@ } str_append_c(str, '\n'); ret = write(fifo_fd, str_data(str), str_len(str)); - if (ret == 0) { - /* busy, try again later */ - return 0; - } + i_assert(ret != 0); if (ret != (ssize_t)str_len(str)) { if (ret > 0) i_error("write(%s) wrote partial data", fifo_path); - else if (errno != EPIPE) + else if (errno == EAGAIN) { + /* busy, try again later */ + return 0; + } else if (errno != EPIPE) { i_error("write(%s) failed: %m", fifo_path); - else { + } else { /* server was probably restarted, don't bother logging this. */ } From dovecot at dovecot.org Fri May 15 11:15:18 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 11:15:18 +0000 Subject: dovecot-2.2: Released v2.2.18. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/917d027836d0 changeset: 18702:917d027836d0 user: Timo Sirainen date: Fri May 15 14:02:58 2015 +0300 description: Released v2.2.18. diffstat: NEWS | 7 +++++++ configure.ac | 2 +- 2 files changed, 8 insertions(+), 1 deletions(-) diffs (26 lines): diff -r d46270397a2f -r 917d027836d0 NEWS --- a/NEWS Fri May 15 13:55:46 2015 +0300 +++ b/NEWS Fri May 15 14:02:58 2015 +0300 @@ -1,3 +1,10 @@ +v2.2.18 2015-05-15 Timo Sirainen + + - director: Login UNIX sockets were normally detected as doveadm or + director ring sockets, causing it to break in existing installations. + - sdbox: When copying a mail in alt storage, place the destination to + alt storage as well. + v2.2.17 2015-05-13 Timo Sirainen * Dovecot no longer checks or warns if a mountpoint is removed. This diff -r d46270397a2f -r 917d027836d0 configure.ac --- a/configure.ac Fri May 15 13:55:46 2015 +0300 +++ b/configure.ac Fri May 15 14:02:58 2015 +0300 @@ -2,7 +2,7 @@ # Be sure to update ABI version also if anything changes that might require # recompiling plugins. Most importantly that means if any structs are changed. -AC_INIT([Dovecot],[2.2.17],[dovecot at dovecot.org]) +AC_INIT([Dovecot],[2.2.18],[dovecot at dovecot.org]) AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv17($PACKAGE_VERSION)", [Dovecot ABI version]) AC_CONFIG_SRCDIR([src]) From dovecot at dovecot.org Fri May 15 11:15:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 11:15:19 +0000 Subject: dovecot-2.2: Added tag 2.2.18 for changeset 917d027836d0 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9e2df74a37c7 changeset: 18703:9e2df74a37c7 user: Timo Sirainen date: Fri May 15 14:02:58 2015 +0300 description: Added tag 2.2.18 for changeset 917d027836d0 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 917d027836d0 -r 9e2df74a37c7 .hgtags --- a/.hgtags Fri May 15 14:02:58 2015 +0300 +++ b/.hgtags Fri May 15 14:02:58 2015 +0300 @@ -126,3 +126,4 @@ da685736985a17c2b6b1ff24c4f762a4009ce598 2.2.17.rc1 2c1ce55de520484431a63041602829d21195b4e9 2.2.17.rc2 166106aaefc5e6cf3850dba9534ca90153931ad9 2.2.17 +917d027836d0f9d7d4b740dc33ae48cc5beead6f 2.2.18 From dovecot at dovecot.org Fri May 15 11:15:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 11:15:24 +0000 Subject: dovecot-2.2: Added signature for changeset 917d027836d0 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c627a0f64719 changeset: 18704:c627a0f64719 user: Timo Sirainen date: Fri May 15 14:03:02 2015 +0300 description: Added signature for changeset 917d027836d0 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 9e2df74a37c7 -r c627a0f64719 .hgsigs --- a/.hgsigs Fri May 15 14:02:58 2015 +0300 +++ b/.hgsigs Fri May 15 14:03:02 2015 +0300 @@ -89,3 +89,4 @@ da685736985a17c2b6b1ff24c4f762a4009ce598 0 iEYEABECAAYFAlVORBAACgkQyUhSUUBViskBnQCfRmUjoFty2pUJrrrdIvkEgOCiVEUAnRK0OYM4z5bxs9xcBlXZn8n2qyFx 2c1ce55de520484431a63041602829d21195b4e9 0 iEYEABECAAYFAlVSXUYACgkQyUhSUUBVislsrwCdEtWDWrQPsJua75RL83lpkeh7DagAoKW+534P0iziiX/nD6QzNmBTHODe 166106aaefc5e6cf3850dba9534ca90153931ad9 0 iEYEABECAAYFAlVTf5MACgkQyUhSUUBViskLHgCeMooopuB7egroMj8w78YarjjRieYAnjVSKcz0sXShlbYRsLRPOnyfq/pS +917d027836d0f9d7d4b740dc33ae48cc5beead6f 0 iEYEABECAAYFAlVV0mIACgkQyUhSUUBVismFtwCeIVNrGFguuXIhDoCSUm86Q4uTpQUAnAi6UfVL9/+QWC5LpR7jvY97QH0c From dovecot at dovecot.org Fri May 15 11:37:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 11:37:10 +0000 Subject: dovecot-2.2: lib-imap-storage: Added imap_msgpart_contains_body() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4cd9e46e0c78 changeset: 18705:4cd9e46e0c78 user: Timo Sirainen date: Fri May 15 14:34:54 2015 +0300 description: lib-imap-storage: Added imap_msgpart_contains_body() diffstat: src/lib-imap-storage/imap-msgpart.c | 16 ++++++++++++++++ src/lib-imap-storage/imap-msgpart.h | 5 +++++ 2 files changed, 21 insertions(+), 0 deletions(-) diffs (41 lines): diff -r c627a0f64719 -r 4cd9e46e0c78 src/lib-imap-storage/imap-msgpart.c --- a/src/lib-imap-storage/imap-msgpart.c Fri May 15 14:03:02 2015 +0300 +++ b/src/lib-imap-storage/imap-msgpart.c Fri May 15 14:34:54 2015 +0300 @@ -306,6 +306,22 @@ pool_unref(&msgpart->pool); } +bool imap_msgpart_contains_body(const struct imap_msgpart *msgpart) +{ + switch (msgpart->fetch_type) { + case FETCH_HEADER: + case FETCH_HEADER_FIELDS: + case FETCH_HEADER_FIELDS_NOT: + return FALSE; + case FETCH_FULL: + case FETCH_MIME: + case FETCH_MIME_BODY: + case FETCH_BODY: + break; + } + return TRUE; +} + void imap_msgpart_set_decode_to_binary(struct imap_msgpart *msgpart) { msgpart->decode_cte_to_binary = TRUE; diff -r c627a0f64719 -r 4cd9e46e0c78 src/lib-imap-storage/imap-msgpart.h --- a/src/lib-imap-storage/imap-msgpart.h Fri May 15 14:03:02 2015 +0300 +++ b/src/lib-imap-storage/imap-msgpart.h Fri May 15 14:34:54 2015 +0300 @@ -24,6 +24,11 @@ int imap_msgpart_parse(const char *section, struct imap_msgpart **msgpart_r); void imap_msgpart_free(struct imap_msgpart **msgpart); +/* Returns TRUE if the msgpart might return at least part of the message body. + Or alternatively: If FALSE is returned, the msgpart will never return + anything except (part of) the message header. MIME headers are counted + as part of the message body. */ +bool imap_msgpart_contains_body(const struct imap_msgpart *msgpart); /* Decode MIME parts with Content-Transfer-Encoding: base64/quoted-printable to binary data (IMAP BINARY extension). If something can't be decoded, fails with storage error set to MAIL_ERROR_CONVERSION. */ From dovecot at dovecot.org Fri May 15 11:37:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 11:37:11 +0000 Subject: dovecot-2.2: Added %{fetch_hdr/body_count/bytes} variables to im... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/93bba97afb2a changeset: 18706:93bba97afb2a user: Timo Sirainen date: Fri May 15 14:35:01 2015 +0300 description: Added %{fetch_hdr/body_count/bytes} variables to imap_logout_format diffstat: doc/example-config/conf.d/20-imap.conf | 4 ++++ src/imap/imap-client.c | 8 ++++++++ src/imap/imap-client.h | 4 ++++ src/imap/imap-fetch-body.c | 21 ++++++++++++++++++++- src/imap/imap-fetch.h | 1 + 5 files changed, 37 insertions(+), 1 deletions(-) diffs (131 lines): diff -r 4cd9e46e0c78 -r 93bba97afb2a doc/example-config/conf.d/20-imap.conf --- a/doc/example-config/conf.d/20-imap.conf Fri May 15 14:34:54 2015 +0300 +++ b/doc/example-config/conf.d/20-imap.conf Fri May 15 14:35:01 2015 +0300 @@ -10,6 +10,10 @@ # IMAP logout format string: # %i - total number of bytes read from client # %o - total number of bytes sent to client +# %{fetch_hdr_count} - Number of mails with mail header data sent to client +# %{fetch_hdr_bytes} - Number of bytes with mail header data sent to client +# %{fetch_body_count} - Number of mails with mail body data sent to client +# %{fetch_body_bytes} - Number of bytes with mail body data sent to client #imap_logout_format = in=%i out=%o # Override the IMAP CAPABILITY response. If the value begins with '+', diff -r 4cd9e46e0c78 -r 93bba97afb2a src/imap/imap-client.c --- a/src/imap/imap-client.c Fri May 15 14:34:54 2015 +0300 +++ b/src/imap/imap-client.c Fri May 15 14:35:01 2015 +0300 @@ -217,6 +217,10 @@ { 'i', NULL, "input" }, { 'o', NULL, "output" }, { '\0', NULL, "session" }, + { '\0', NULL, "fetch_hdr_count" }, + { '\0', NULL, "fetch_hdr_bytes" }, + { '\0', NULL, "fetch_body_count" }, + { '\0', NULL, "fetch_body_bytes" }, { '\0', NULL, NULL } }; struct var_expand_table *tab; @@ -228,6 +232,10 @@ tab[0].value = dec2str(i_stream_get_absolute_offset(client->input)); tab[1].value = dec2str(client->output->offset); tab[2].value = client->session_id; + tab[3].value = dec2str(client->fetch_hdr_count); + tab[4].value = dec2str(client->fetch_hdr_bytes); + tab[5].value = dec2str(client->fetch_body_count); + tab[6].value = dec2str(client->fetch_body_bytes); str = t_str_new(128); var_expand(str, client->set->imap_logout_format, tab); diff -r 4cd9e46e0c78 -r 93bba97afb2a src/imap/imap-client.h --- a/src/imap/imap-client.h Fri May 15 14:34:54 2015 +0300 +++ b/src/imap/imap-client.h Fri May 15 14:35:01 2015 +0300 @@ -136,6 +136,10 @@ uint64_t sync_last_full_modseq; uint64_t highest_fetch_modseq; + /* For imap_logout_format statistics: */ + unsigned int fetch_hdr_count, fetch_body_count; + uint64_t fetch_hdr_bytes, fetch_body_bytes; + /* SEARCHRES extension: Last saved SEARCH result */ ARRAY_TYPE(seq_range) search_saved_uidset; /* SEARCH=CONTEXT extension: Searches that get updated */ diff -r 4cd9e46e0c78 -r 93bba97afb2a src/imap/imap-fetch-body.c --- a/src/imap/imap-fetch-body.c Fri May 15 14:34:54 2015 +0300 +++ b/src/imap/imap-fetch-body.c Fri May 15 14:35:01 2015 +0300 @@ -99,8 +99,11 @@ ret = o_stream_send_istream(ctx->client->output, state->cur_input); o_stream_set_max_buffer_size(ctx->client->output, (size_t)-1); - if (ret > 0) + if (ret > 0) { state->cur_offset += ret; + if (ctx->state.cur_stats_sizep != NULL) + *ctx->state.cur_stats_sizep += ret; + } if (state->cur_offset != state->cur_size) { /* unfinished */ @@ -160,6 +163,18 @@ return p_strdup(pool, str_c(str)); } +static void fetch_state_update_stats(struct imap_fetch_context *ctx, + const struct imap_msgpart *msgpart) +{ + if (!imap_msgpart_contains_body(msgpart)) { + ctx->client->fetch_hdr_count++; + ctx->state.cur_stats_sizep = &ctx->client->fetch_hdr_bytes; + } else { + ctx->client->fetch_body_count++; + ctx->state.cur_stats_sizep = &ctx->client->fetch_body_bytes; + } +} + static int fetch_body_msgpart(struct imap_fetch_context *ctx, struct mail *mail, struct imap_fetch_body_data *body) { @@ -178,6 +193,7 @@ ctx->state.cur_size_field = result.size_field; ctx->state.cur_human_name = get_body_human_name(ctx->ctx_pool, body); + fetch_state_update_stats(ctx, body->msgpart); str = get_prefix(&ctx->state, body, ctx->state.cur_size, result.binary_decoded_input_has_nuls); o_stream_nsend(ctx->client->output, str_data(str), str_len(str)); @@ -487,6 +503,7 @@ str++; ctx->state.cur_first = FALSE; } o_stream_nsend_str(ctx->client->output, str); + fetch_state_update_stats(ctx, msgpart); ctx->state.cur_human_name = "RFC822"; return ctx->state.cont_handler(ctx); @@ -509,6 +526,7 @@ str++; ctx->state.cur_first = FALSE; } o_stream_nsend_str(ctx->client->output, str); + fetch_state_update_stats(ctx, msgpart); ctx->state.cur_human_name = "RFC822.HEADER"; return ctx->state.cont_handler(ctx); @@ -531,6 +549,7 @@ str++; ctx->state.cur_first = FALSE; } o_stream_nsend_str(ctx->client->output, str); + fetch_state_update_stats(ctx, msgpart); ctx->state.cur_human_name = "RFC822.TEXT"; return ctx->state.cont_handler(ctx); diff -r 4cd9e46e0c78 -r 93bba97afb2a src/imap/imap-fetch.h --- a/src/imap/imap-fetch.h Fri May 15 14:34:54 2015 +0300 +++ b/src/imap/imap-fetch.h Fri May 15 14:35:01 2015 +0300 @@ -59,6 +59,7 @@ struct istream *cur_input; bool skip_cr; int (*cont_handler)(struct imap_fetch_context *ctx); + uint64_t *cur_stats_sizep; unsigned int fetching:1; unsigned int seen_flags_changed:1; From dovecot at dovecot.org Fri May 15 11:57:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 11:57:22 +0000 Subject: dovecot-2.2: imap: Added %{deleted}, %{expunged} and %{trashed} ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e0a17714f0c9 changeset: 18707:e0a17714f0c9 user: Timo Sirainen date: Fri May 15 14:55:21 2015 +0300 description: imap: Added %{deleted}, %{expunged} and %{trashed} to imap_logout_format diffstat: doc/example-config/conf.d/20-imap.conf | 4 ++++ src/imap/cmd-close.c | 2 +- src/imap/cmd-copy.c | 15 +++++++++++++++ src/imap/cmd-expunge.c | 2 +- src/imap/cmd-store.c | 10 ++++++++++ src/imap/imap-client.c | 6 ++++++ src/imap/imap-client.h | 1 + src/imap/imap-expunge.c | 4 +++- src/imap/imap-expunge.h | 3 ++- 9 files changed, 43 insertions(+), 4 deletions(-) diffs (186 lines): diff -r 93bba97afb2a -r e0a17714f0c9 doc/example-config/conf.d/20-imap.conf --- a/doc/example-config/conf.d/20-imap.conf Fri May 15 14:35:01 2015 +0300 +++ b/doc/example-config/conf.d/20-imap.conf Fri May 15 14:55:21 2015 +0300 @@ -14,6 +14,10 @@ # %{fetch_hdr_bytes} - Number of bytes with mail header data sent to client # %{fetch_body_count} - Number of mails with mail body data sent to client # %{fetch_body_bytes} - Number of bytes with mail body data sent to client +# %{deleted} - Number of mails where client added \Deleted flag +# %{expunged} - Number of mails that client expunged +# %{trashed} - Number of mails that client copied/moved to the +# special_use=\Trash mailbox. #imap_logout_format = in=%i out=%o # Override the IMAP CAPABILITY response. If the value begins with '+', diff -r 93bba97afb2a -r e0a17714f0c9 src/imap/cmd-close.c --- a/src/imap/cmd-close.c Fri May 15 14:35:01 2015 +0300 +++ b/src/imap/cmd-close.c Fri May 15 14:55:21 2015 +0300 @@ -19,7 +19,7 @@ client->mailbox = NULL; storage = mailbox_get_storage(mailbox); - if (imap_expunge(mailbox, NULL) < 0) { + if (imap_expunge(mailbox, NULL, &client->expunged_count) < 0) { errstr = mailbox_get_last_error(mailbox, &error); if (error != MAIL_ERROR_PERM) client_send_untagged_storage_error(client, storage); diff -r 93bba97afb2a -r e0a17714f0c9 src/imap/cmd-copy.c --- a/src/imap/cmd-copy.c Fri May 15 14:35:01 2015 +0300 +++ b/src/imap/cmd-copy.c Fri May 15 14:55:21 2015 +0300 @@ -86,6 +86,19 @@ return ret; } +static void copy_update_trashed(struct client *client, struct mailbox *box, + unsigned int count) +{ + const struct mailbox_settings *set; + + set = mailbox_settings_find(mailbox_get_namespace(box), + mailbox_get_vname(box)); + if (set != NULL && set->special_use[0] != '\0' && + str_array_icase_find(t_strsplit_spaces(set->special_use, " "), + "\\Trash")) + client->trashed_count += count; +} + static bool cmd_copy_full(struct client_command_context *cmd, bool move) { struct client *client = cmd->client; @@ -147,6 +160,7 @@ pool_unref(&changes.pool); } else if (move) { i_assert(copy_count == seq_range_count(&changes.saved_uids)); + copy_update_trashed(client, destbox, copy_count); str_printfa(msg, "* OK [COPYUID %u %s ", changes.uid_validity, src_uidset); @@ -159,6 +173,7 @@ pool_unref(&changes.pool); } else { i_assert(copy_count == seq_range_count(&changes.saved_uids)); + copy_update_trashed(client, destbox, copy_count); str_printfa(msg, "OK [COPYUID %u %s ", changes.uid_validity, src_uidset); diff -r 93bba97afb2a -r e0a17714f0c9 src/imap/cmd-expunge.c --- a/src/imap/cmd-expunge.c Fri May 15 14:35:01 2015 +0300 +++ b/src/imap/cmd-expunge.c Fri May 15 14:55:21 2015 +0300 @@ -29,7 +29,7 @@ int ret; ret = imap_expunge(client->mailbox, search_args == NULL ? NULL : - search_args->args); + search_args->args, &client->expunged_count); if (search_args != NULL) mail_search_args_unref(&search_args); if (ret < 0) { diff -r 93bba97afb2a -r e0a17714f0c9 src/imap/cmd-store.c --- a/src/imap/cmd-store.c Fri May 15 14:35:01 2015 +0300 +++ b/src/imap/cmd-store.c Fri May 15 14:55:21 2015 +0300 @@ -136,6 +136,8 @@ const char *set, *reply, *tagged_reply; string_t *str; int ret; + bool update_deletes; + unsigned int deleted_count; if (!client_read_args(cmd, 0, 0, &args)) return FALSE; @@ -188,6 +190,9 @@ &modified_set); } + update_deletes = (ctx.flags & MAIL_DELETED) != 0 && + ctx.modify_type != MODIFY_REMOVE; + deleted_count = 0; while (mailbox_search_next(search_ctx, &mail)) { if (ctx.max_modseq < (uint64_t)-1) { /* check early so there's less work for transaction @@ -197,6 +202,10 @@ continue; } } + if (update_deletes) { + if ((mail_get_flags(mail) & MAIL_DELETED) == 0) + deleted_count++; + } if (ctx.modify_type == MODIFY_REPLACE || ctx.flags != 0) mail_update_flags(mail, ctx.modify_type, ctx.flags); if (ctx.modify_type == MODIFY_REPLACE || ctx.keywords != NULL) { @@ -218,6 +227,7 @@ client_send_box_error(cmd, client->mailbox); return TRUE; } + client->deleted_count += deleted_count; if (array_count(&modified_set) == 0) tagged_reply = "OK Store completed."; diff -r 93bba97afb2a -r e0a17714f0c9 src/imap/imap-client.c --- a/src/imap/imap-client.c Fri May 15 14:35:01 2015 +0300 +++ b/src/imap/imap-client.c Fri May 15 14:55:21 2015 +0300 @@ -221,6 +221,9 @@ { '\0', NULL, "fetch_hdr_bytes" }, { '\0', NULL, "fetch_body_count" }, { '\0', NULL, "fetch_body_bytes" }, + { '\0', NULL, "deleted" }, + { '\0', NULL, "expunged" }, + { '\0', NULL, "trashed" }, { '\0', NULL, NULL } }; struct var_expand_table *tab; @@ -236,6 +239,9 @@ tab[4].value = dec2str(client->fetch_hdr_bytes); tab[5].value = dec2str(client->fetch_body_count); tab[6].value = dec2str(client->fetch_body_bytes); + tab[7].value = dec2str(client->deleted_count); + tab[8].value = dec2str(client->expunged_count); + tab[9].value = dec2str(client->trashed_count); str = t_str_new(128); var_expand(str, client->set->imap_logout_format, tab); diff -r 93bba97afb2a -r e0a17714f0c9 src/imap/imap-client.h --- a/src/imap/imap-client.h Fri May 15 14:35:01 2015 +0300 +++ b/src/imap/imap-client.h Fri May 15 14:55:21 2015 +0300 @@ -139,6 +139,7 @@ /* For imap_logout_format statistics: */ unsigned int fetch_hdr_count, fetch_body_count; uint64_t fetch_hdr_bytes, fetch_body_bytes; + unsigned int deleted_count, expunged_count, trashed_count; /* SEARCHRES extension: Last saved SEARCH result */ ARRAY_TYPE(seq_range) search_saved_uidset; diff -r 93bba97afb2a -r e0a17714f0c9 src/imap/imap-expunge.c --- a/src/imap/imap-expunge.c Fri May 15 14:35:01 2015 +0300 +++ b/src/imap/imap-expunge.c Fri May 15 14:55:21 2015 +0300 @@ -5,7 +5,8 @@ #include "mail-search-build.h" #include "imap-expunge.h" -int imap_expunge(struct mailbox *box, struct mail_search_arg *next_search_arg) +int imap_expunge(struct mailbox *box, struct mail_search_arg *next_search_arg, + unsigned int *expunged_count) { struct mail_search_context *ctx; struct mailbox_transaction_context *t; @@ -31,6 +32,7 @@ mail_search_args_unref(&search_args); while (mailbox_search_next(ctx, &mail)) { + *expunged_count += 1; mail_expunge(mail); expunges = TRUE; } diff -r 93bba97afb2a -r e0a17714f0c9 src/imap/imap-expunge.h --- a/src/imap/imap-expunge.h Fri May 15 14:35:01 2015 +0300 +++ b/src/imap/imap-expunge.h Fri May 15 14:55:21 2015 +0300 @@ -3,7 +3,8 @@ struct mail_search_arg; -int imap_expunge(struct mailbox *box, struct mail_search_arg *next_search_arg) +int imap_expunge(struct mailbox *box, struct mail_search_arg *next_search_arg, + unsigned int *expunged_count) ATTR_NULL(2); #endif From dovecot at dovecot.org Fri May 15 13:52:29 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 13:52:29 +0000 Subject: dovecot-2.2: director: Moving a user to another host sometimes c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/51ee438392c7 changeset: 18708:51ee438392c7 user: Timo Sirainen date: Fri May 15 16:44:45 2015 +0300 description: director: Moving a user to another host sometimes caused the move to fail. It could have given "User hash .. is being redirected to two hosts" error and afterwards moved the user back to its original host. diffstat: src/director/director-connection.c | 9 +++++++++ src/director/user-directory.h | 5 ++++- 2 files changed, 13 insertions(+), 1 deletions(-) diffs (34 lines): diff -r e0a17714f0c9 -r 51ee438392c7 src/director/director-connection.c --- a/src/director/director-connection.c Fri May 15 14:55:21 2015 +0300 +++ b/src/director/director-connection.c Fri May 15 16:44:45 2015 +0300 @@ -506,6 +506,15 @@ "replacing host %s with %s", username_hash, net_ip2addr(&user->host->ip), net_ip2addr(&host->ip)); ret = TRUE; + } else if (user->kill_state != USER_KILL_STATE_NONE && + user->kill_state < USER_KILL_STATE_DELAY) { + /* user is still being moved - ignore conflicting host updates + from other directors who don't yet know about the move. */ + dir_debug("user refresh: %u is being moved, " + "preserve its host %s instead of replacing with %s", + username_hash, net_ip2addr(&user->host->ip), + net_ip2addr(&host->ip)); + host = user->host; } else { /* non-weak user received a non-weak update with conflicting host. this shouldn't happen. */ diff -r e0a17714f0c9 -r 51ee438392c7 src/director/user-directory.h --- a/src/director/user-directory.h Fri May 15 14:55:21 2015 +0300 +++ b/src/director/user-directory.h Fri May 15 16:44:45 2015 +0300 @@ -16,7 +16,10 @@ /* We're done killing, but waiting for USER-KILLED-EVERYWHERE notification until this state gets reset. */ USER_KILL_STATE_KILLED_WAITING_FOR_EVERYONE, - /* Wait for a while for the user connections to actually die */ + /* Wait for a while for the user connections to actually die. Note that + only at this stage we can be sure that all the directors know about + the user move (although it could be earlier if we added a new + USER-MOVED notification). */ USER_KILL_STATE_DELAY }; From dovecot at dovecot.org Fri May 15 13:52:29 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 13:52:29 +0000 Subject: dovecot-2.2: director: Added HOST-RESET-USERS command to move us... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/618c8d76bc5d changeset: 18709:618c8d76bc5d user: Timo Sirainen date: Fri May 15 16:45:14 2015 +0300 description: director: Added HOST-RESET-USERS command to move users to their hash-assigned hosts. diffstat: src/director/doveadm-connection.c | 64 +++++++++++++++++++++++++++++++++++++++ 1 files changed, 64 insertions(+), 0 deletions(-) diffs (81 lines): diff -r 51ee438392c7 -r 618c8d76bc5d src/director/doveadm-connection.c --- a/src/director/doveadm-connection.c Fri May 15 16:44:45 2015 +0300 +++ b/src/director/doveadm-connection.c Fri May 15 16:45:14 2015 +0300 @@ -348,6 +348,68 @@ return TRUE; } +static void +director_host_reset_users(struct director *dir, struct director_host *src, + struct mail_host *host) +{ + struct user_directory_iter *iter; + struct user *user; + struct mail_host *new_host; + + if (dir->right != NULL) + director_connection_cork(dir->right); + + iter = user_directory_iter_init(dir->users); + while ((user = user_directory_iter_next(iter)) != NULL) { + if (user->host != host) + continue; + new_host = mail_host_get_by_hash(dir->mail_hosts, + user->username_hash, host->tag); + if (new_host != host) T_BEGIN { + director_move_user(dir, src, NULL, + user->username_hash, new_host); + } T_END; + } + user_directory_iter_deinit(&iter); + if (dir->right != NULL) + director_connection_uncork(dir->right); +} + +static void +doveadm_cmd_host_reset_users_all(struct doveadm_connection *conn) +{ + struct mail_host *const *hostp; + + array_foreach(mail_hosts_get(conn->dir->mail_hosts), hostp) + director_host_reset_users(conn->dir, conn->dir->self_host, *hostp); + o_stream_nsend(conn->output, "OK\n", 3); +} + +static bool +doveadm_cmd_host_reset_users(struct doveadm_connection *conn, const char *line) +{ + struct mail_host *host; + struct ip_addr ip; + + if (line[0] == '\0') { + doveadm_cmd_host_reset_users_all(conn); + return TRUE; + } + + if (net_addr2ip(line, &ip) < 0) { + i_error("doveadm sent invalid HOST-RESET-USERS parameters"); + return FALSE; + } + host = mail_host_lookup(conn->dir->mail_hosts, &ip); + if (host == NULL) + o_stream_nsend_str(conn->output, "NOTFOUND\n"); + else { + director_host_reset_users(conn->dir, conn->dir->self_host, host); + o_stream_nsend(conn->output, "OK\n", 3); + } + return TRUE; +} + static bool doveadm_cmd_user_lookup(struct doveadm_connection *conn, const char *line) { @@ -529,6 +591,8 @@ ret = doveadm_cmd_host_remove(conn, args); else if (strcmp(cmd, "HOST-FLUSH") == 0) ret = doveadm_cmd_host_flush(conn, args); + else if (strcmp(cmd, "HOST-RESET-USERS") == 0) + ret = doveadm_cmd_host_reset_users(conn, args); else if (strcmp(cmd, "USER-LOOKUP") == 0) ret = doveadm_cmd_user_lookup(conn, args); else if (strcmp(cmd, "USER-LIST") == 0) From dovecot at dovecot.org Fri May 15 13:52:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 13:52:36 +0000 Subject: dovecot-2.2: lib-master: Added data stack frames to handling IPC... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5175f8e600e9 changeset: 18710:5175f8e600e9 user: Timo Sirainen date: Fri May 15 16:46:14 2015 +0300 description: lib-master: Added data stack frames to handling IPC input. diffstat: src/lib-master/ipc-client.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (14 lines): diff -r 618c8d76bc5d -r 5175f8e600e9 src/lib-master/ipc-client.c --- a/src/lib-master/ipc-client.c Fri May 15 16:45:14 2015 +0300 +++ b/src/lib-master/ipc-client.c Fri May 15 16:46:14 2015 +0300 @@ -74,8 +74,9 @@ ipc_client_disconnect(client); return; } - while ((line = i_stream_next_line(client->input)) != NULL) + while ((line = i_stream_next_line(client->input)) != NULL) T_BEGIN { ipc_client_input_line(client, line); + } T_END; } static int ipc_client_connect(struct ipc_client *client) From dovecot at dovecot.org Fri May 15 13:52:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 13:52:36 +0000 Subject: dovecot-2.2: doveadm director flush: Unless -F parameter is used... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2336a3127a0f changeset: 18711:2336a3127a0f user: Timo Sirainen date: Fri May 15 16:50:27 2015 +0300 description: doveadm director flush: Unless -F parameter is used, do the flush by moving users. User moving causes the users to be properly kicked out of the old backends before new connections are made to the new backends. This way the user isn't accessed simultaneously by different backends. diffstat: src/doveadm/doveadm-director.c | 17 +++++++++++------ 1 files changed, 11 insertions(+), 6 deletions(-) diffs (62 lines): diff -r 5175f8e600e9 -r 2336a3127a0f src/doveadm/doveadm-director.c --- a/src/doveadm/doveadm-director.c Fri May 15 16:46:14 2015 +0300 +++ b/src/doveadm/doveadm-director.c Fri May 15 16:50:27 2015 +0300 @@ -24,7 +24,7 @@ const char *tag; struct istream *input; bool explicit_socket_path; - bool hash_map, user_map; + bool hash_map, user_map, force_flush; }; struct user_list { @@ -106,6 +106,9 @@ case 'f': ctx->users_path = optarg; break; + case 'F': + ctx->force_flush = TRUE; + break; case 'h': ctx->hash_map = TRUE; break; @@ -549,7 +552,8 @@ { const char *line; - director_send(ctx, "HOST-FLUSH\n"); + director_send(ctx, ctx->force_flush ? + "HOST-FLUSH\n" : "HOST-RESET-USERS\n"); line = i_stream_read_next_line(ctx->input); if (line == NULL) { @@ -572,7 +576,7 @@ const char *host, *line; int ret; - ctx = cmd_director_init(argc, argv, "a:", cmd_director_flush); + ctx = cmd_director_init(argc, argv, "a:F", cmd_director_flush); host = argv[optind++]; if (host == NULL || argv[optind] != NULL) director_cmd_help(cmd_director_flush); @@ -593,8 +597,9 @@ } for (i = 0; i < ips_count; i++) { - director_send(ctx, - t_strdup_printf("HOST-FLUSH\t%s\n", net_ip2addr(&ip))); + director_send(ctx, t_strdup_printf("%s\t%s\n", + ctx->force_flush ? "HOST-FLUSH" : "HOST-RESET-USERS", + net_ip2addr(&ip))); } for (i = 0; i < ips_count; i++) { line = i_stream_read_next_line(ctx->input); @@ -779,7 +784,7 @@ { cmd_director_kick, "director kick", "[-a ] " }, { cmd_director_flush, "director flush", - "[-a ] |all" }, + "[-a ] [-f] |all" }, { cmd_director_dump, "director dump", "[-a ]" }, { cmd_director_ring_add, "director ring add", From dovecot at dovecot.org Fri May 15 18:38:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 15 May 2015 18:38:08 +0000 Subject: dovecot-2.2: replication plugin: s/transction/transaction/ in de... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0b13bfe5d09b changeset: 18712:0b13bfe5d09b user: Timo Sirainen date: Fri May 15 21:36:07 2015 +0300 description: replication plugin: s/transction/transaction/ in debug log messages diffstat: src/plugins/replication/replication-plugin.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 2336a3127a0f -r 0b13bfe5d09b src/plugins/replication/replication-plugin.c --- a/src/plugins/replication/replication-plugin.c Fri May 15 16:50:27 2015 +0300 +++ b/src/plugins/replication/replication-plugin.c Fri May 15 21:36:07 2015 +0300 @@ -260,7 +260,7 @@ priority = !ctx->new_messages ? REPLICATION_PRIORITY_LOW : ruser->sync_secs == 0 ? REPLICATION_PRIORITY_HIGH : REPLICATION_PRIORITY_SYNC; - replication_notify(ctx->ns, priority, "transction commit"); + replication_notify(ctx->ns, priority, "transaction commit"); } i_free(ctx); } From pigeonhole at rename-it.nl Fri May 15 18:52:03 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 15 May 2015 20:52:03 +0200 Subject: dovecot-2.2-pigeonhole: Added tag 0.4.8 for changeset baa15dd77f9e Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/54827cd6eb23 changeset: 2071:54827cd6eb23 user: Stephan Bosch date: Fri May 15 20:22:32 2015 +0200 description: Added tag 0.4.8 for changeset baa15dd77f9e diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r baa15dd77f9e -r 54827cd6eb23 .hgtags --- a/.hgtags Fri May 15 20:22:20 2015 +0200 +++ b/.hgtags Fri May 15 20:22:32 2015 +0200 @@ -30,3 +30,4 @@ fcc97e953584269e5fa140363804f4016c33ef1c 0.4.8.rc1 44d41235776de692f1f4ef9843ee6f3901fa3138 0.4.8.rc2 7b154d69394fcdbbc0b066978d864e551b736ce9 0.4.8.rc3 +baa15dd77f9e2a1b7794771ce7deb6f961ed0360 0.4.8 From pigeonhole at rename-it.nl Fri May 15 18:52:03 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 15 May 2015 20:52:03 +0200 Subject: dovecot-2.2-pigeonhole: Released v0.4.8 for Dovecot v2.2.18. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/baa15dd77f9e changeset: 2070:baa15dd77f9e user: Stephan Bosch date: Fri May 15 20:22:20 2015 +0200 description: Released v0.4.8 for Dovecot v2.2.18. diffstat: NEWS | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 0c4ae064f307 -r baa15dd77f9e NEWS --- a/NEWS Thu May 14 13:13:39 2015 +0200 +++ b/NEWS Fri May 15 20:22:20 2015 +0200 @@ -1,4 +1,4 @@ -v0.4.8 xx-05-2015 Stephan Bosch +v0.4.8 15-05-2015 Stephan Bosch * LDA Sieve plugin: Dovecot changed the deliver_log_format setting to include %{delivery_time}. This prompted changes in Pigeonhole that make this release diff -r 0c4ae064f307 -r baa15dd77f9e configure.ac --- a/configure.ac Thu May 14 13:13:39 2015 +0200 +++ b/configure.ac Fri May 15 20:22:20 2015 +0200 @@ -1,4 +1,4 @@ -AC_INIT([Pigeonhole], [0.4.8.rc3], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) +AC_INIT([Pigeonhole], [0.4.8], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_SRCDIR([src]) AC_CONFIG_MACRO_DIR([m4]) From pigeonhole at rename-it.nl Fri May 15 18:52:52 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 15 May 2015 20:52:52 +0200 Subject: dovecot-2.2-pigeonhole: Added signature for changeset baa15dd77f9e Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/5b17177abce9 changeset: 2072:5b17177abce9 user: Stephan Bosch date: Fri May 15 20:54:00 2015 +0200 description: Added signature for changeset baa15dd77f9e diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 54827cd6eb23 -r 5b17177abce9 .hgsigs --- a/.hgsigs Fri May 15 20:22:32 2015 +0200 +++ b/.hgsigs Fri May 15 20:54:00 2015 +0200 @@ -24,3 +24,4 @@ fcc97e953584269e5fa140363804f4016c33ef1c 0 iQEcBAABAgAGBQJVT9pVAAoJEATWKx49+7T0/14H/3iejYaCOqm5ftfQDMlebUhbMa9LGEMwOPRoBxnzGOBoJGEQ7MS/aqk8ARM1i016jFtDa0oVWkTDyWZhDwQHS6yA2HkXPxizKHf9+9Kc2Lxjew4GIyTGSNmkaJb34XUKGhKx4YMfDXQiFsc0G/vVSO8UoZzvqho6g7bUmuWeuCrXmEW+M+bUG4KMCF0abYba7iasYKaV+1fSnbAnh5kRQqwfUfs/ugdU4f9VZxmnGdJymf540KdoqEzePMaxhEp6OPhhr/eVGabfWhV+RJiRI7atWQl7I6uoEkvziObuUKx3rl0xVoSVcKeCC4APsJdQQT/HDoPmWfyaQCV+WORmz6U= 44d41235776de692f1f4ef9843ee6f3901fa3138 0 iQEcBAABAgAGBQJVUP1CAAoJEATWKx49+7T0+GMH/3QKZewiy+jeeHmb8wYExW6eiAEprb7lq2U8UjN5iOuID7OnDbkZ0/jr/oU0zsxpch/lacpAa0cIvgvujhGuXzeR2XNO3MWyiOPe4t0WcFaoP4Hl+EV7faZipCdPx7mGbxFYSEx9rz4Kp0MBnG/I+yE2+I1AbBsHqQMC1meAA3llu1n/Wf9BOhyzUP4/AL9CSNUtO/6tUZriXQOrRZpwkcyc7mrH6Trpyy0YXkEVWAZ8jdusiVuLmwMzEHUaF4SXc2daHhRRRiwp3wCeIkyVvqm8BbxFGZ7ZMM7fKnhj3LGmSLfeloHHmCC3HU6SfzvNmv8Fcieah/thpl/EOdKiWeI= 7b154d69394fcdbbc0b066978d864e551b736ce9 0 iQEcBAABAgAGBQJVU7n1AAoJEATWKx49+7T0sOYH/jL46oo70cwImQN5pyAHudSarM9Mtls8//vX7JmJ/b8OPftwwx4ExBNYmvHSv315CPjq6ntMA7GMZ1tAd9lHk+ZB2WFL/Y7MQZUGgowo6K/F36MW1Z6tGN2rtnooqglyamKUPqrLIurs5Hc9444VrGMcdeMecGn9mEpxULy1jsMmo0AVVsyf50PHbYRwSgq960onz93lKhTcaD7/Sc76FPbJenqstP9iPQBuXugVxv/p9gN31Xx/V8RlxmHIjeHljx33m76sTGQ/8ca2XKvBwft8dZU4lTsrSmcvt+0wxATurjtfJ+aqPlH5pPXmqydbsEe6F3fjZTblwexc4kFNDsQ= +baa15dd77f9e2a1b7794771ce7deb6f961ed0360 0 iQEcBAABAgAGBQJVVkC/AAoJEATWKx49+7T0nhoIAInNseUHHa4TUCeKH5NOr4f8d5Sj1arnxr1IGPuNlhz/Naeu2999cyVLHWGfOyWN7/B3SCmbJ/Vb/41lJLikC7Ma01IEskMDIcvJca/Nx8ZR5mcDxwpxma7k0III+gxf0pP2HWvqYXOZD+JXEGmvjCXSJyMAYPkEUEO/aVH83fNQVaPAqvvrl3cZz53TF8U9l01DecduRwLm+od2zHiFByB4bJuKhIvcAq2ksrjiHyfLJv9MXLZnhSbTLbWr4SXHs8bctzgBg6TXl6Fgp6ehOR3UQTR+k3xFOfXJY2GBfNPzBUi+rY8g8DgirNDDYjSwrc1W1+QRD5d9ANwmppeDDzg= From dovecot at dovecot.org Sat May 16 08:49:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 May 2015 08:49:11 +0000 Subject: dovecot-2.2: imap: Fixed crash in FETCH RFC822* caused by earlie... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1044c55fb4ef changeset: 18713:1044c55fb4ef user: Timo Sirainen date: Sat May 16 11:47:09 2015 +0300 description: imap: Fixed crash in FETCH RFC822* caused by earlier commit diffstat: src/imap/imap-fetch-body.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (51 lines): diff -r 0b13bfe5d09b -r 1044c55fb4ef src/imap/imap-fetch-body.c --- a/src/imap/imap-fetch-body.c Fri May 15 21:36:07 2015 +0300 +++ b/src/imap/imap-fetch-body.c Sat May 16 11:47:09 2015 +0300 @@ -494,6 +494,7 @@ const char *str; msgpart = imap_msgpart_full(); + fetch_state_update_stats(ctx, msgpart); if (fetch_and_free_msgpart(ctx, mail, &msgpart) < 0) return -1; @@ -503,7 +504,6 @@ str++; ctx->state.cur_first = FALSE; } o_stream_nsend_str(ctx->client->output, str); - fetch_state_update_stats(ctx, msgpart); ctx->state.cur_human_name = "RFC822"; return ctx->state.cont_handler(ctx); @@ -517,6 +517,7 @@ const char *str; msgpart = imap_msgpart_header(); + fetch_state_update_stats(ctx, msgpart); if (fetch_and_free_msgpart(ctx, mail, &msgpart) < 0) return -1; @@ -526,7 +527,6 @@ str++; ctx->state.cur_first = FALSE; } o_stream_nsend_str(ctx->client->output, str); - fetch_state_update_stats(ctx, msgpart); ctx->state.cur_human_name = "RFC822.HEADER"; return ctx->state.cont_handler(ctx); @@ -540,6 +540,7 @@ const char *str; msgpart = imap_msgpart_body(); + fetch_state_update_stats(ctx, msgpart); if (fetch_and_free_msgpart(ctx, mail, &msgpart) < 0) return -1; @@ -549,7 +550,6 @@ str++; ctx->state.cur_first = FALSE; } o_stream_nsend_str(ctx->client->output, str); - fetch_state_update_stats(ctx, msgpart); ctx->state.cur_human_name = "RFC822.TEXT"; return ctx->state.cont_handler(ctx); From dovecot at dovecot.org Sat May 16 09:48:41 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 May 2015 09:48:41 +0000 Subject: dovecot-2.2: lib-charset: Added a minimal unit test Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dcaf508860ad changeset: 18714:dcaf508860ad user: Timo Sirainen date: Sat May 16 12:46:38 2015 +0300 description: lib-charset: Added a minimal unit test diffstat: src/lib-charset/Makefile.am | 23 ++++++++++- src/lib-charset/charset-iconv.c | 3 +- src/lib-charset/test-charset.c | 90 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 3 deletions(-) diffs (144 lines): diff -r 1044c55fb4ef -r dcaf508860ad src/lib-charset/Makefile.am --- a/src/lib-charset/Makefile.am Sat May 16 11:47:09 2015 +0300 +++ b/src/lib-charset/Makefile.am Sat May 16 12:46:38 2015 +0300 @@ -1,7 +1,8 @@ noinst_LTLIBRARIES = libcharset.la AM_CPPFLAGS = \ - -I$(top_srcdir)/src/lib + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test libcharset_la_LIBADD = $(LTLIBICONV) libcharset_la_SOURCES = \ @@ -13,3 +14,23 @@ pkginc_libdir=$(pkgincludedir) pkginc_lib_HEADERS = $(headers) + +test_programs = \ + test-charset + +noinst_PROGRAMS = $(test_programs) + +test_libs = \ + ../lib-test/libtest.la \ + ../lib/liblib.la +test_deps = $(noinst_LTLIBRARIES) $(test_libs) + +test_charset_SOURCES = test-charset.c +test_charset_LDADD = libcharset.la $(test_libs) +test_charset_DEPENDENCIES = libcharset.la $(test_deps) + +check: check-am check-test +check-test: all-am + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done diff -r 1044c55fb4ef -r dcaf508860ad src/lib-charset/charset-iconv.c --- a/src/lib-charset/charset-iconv.c Sat May 16 11:47:09 2015 +0300 +++ b/src/lib-charset/charset-iconv.c Sat May 16 12:46:38 2015 +0300 @@ -118,8 +118,7 @@ if (result == CHARSET_RET_INVALID_INPUT) { if (prev_invalid_pos != dest->used) { - uni_ucs4_to_utf8_c(UNICODE_REPLACEMENT_CHAR, - dest); + str_append(dest, UNICODE_REPLACEMENT_CHAR_UTF8); prev_invalid_pos = dest->used; } pos++; diff -r 1044c55fb4ef -r dcaf508860ad src/lib-charset/test-charset.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-charset/test-charset.c Sat May 16 12:46:38 2015 +0300 @@ -0,0 +1,90 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "test-common.h" +#include "charset-utf8.h" + +static void test_charset_is_utf8(void) +{ + test_begin("charset_is_utf8"); + test_assert(charset_is_utf8("AScII")); + test_assert(charset_is_utf8("us-AScII")); + test_assert(charset_is_utf8("uTF8")); + test_assert(charset_is_utf8("uTF-8")); + test_end(); +} + +static void test_charset_utf8_common(const char *input_charset) +{ + struct { + const char *input; + const char *output; + enum charset_result result; + } tests[] = { + { "p???", "p??", CHARSET_RET_INCOMPLETE_INPUT }, + { "p???a", "p??"UNICODE_REPLACEMENT_CHAR_UTF8"a", CHARSET_RET_INVALID_INPUT } + }; + string_t *str = t_str_new(128); + enum charset_result result; + unsigned int i; + + for (i = 0; i < N_ELEMENTS(tests); i++) { + str_truncate(str, 0); + test_assert_idx(charset_to_utf8_str(input_charset, NULL, + tests[i].input, str, &result) == 0, i); + test_assert_idx(strcmp(tests[i].output, str_c(str)) == 0, i); + test_assert_idx(result == tests[i].result, i); + } +} + +static void test_charset_utf8(void) +{ + test_begin("charset utf8"); + test_charset_utf8_common("UTF-8"); + test_end(); +} + +#ifdef HAVE_ICONV +static void test_charset_iconv(void) +{ + struct { + const char *charset; + const char *input; + const char *output; + enum charset_result result; + } tests[] = { + { "ISO-8859-1", "p\xE4\xE4", "p????", CHARSET_RET_OK } + }; + string_t *str = t_str_new(128); + enum charset_result result; + unsigned int i; + + test_begin("charset iconv"); + for (i = 0; i < N_ELEMENTS(tests); i++) { + str_truncate(str, 0); + test_assert_idx(charset_to_utf8_str(tests[i].charset, NULL, + tests[i].input, str, &result) == 0, i); + test_assert_idx(strcmp(tests[i].output, str_c(str)) == 0, i); + test_assert_idx(result == tests[i].result, i); + } + /* Use //IGNORE just to force handling to be done by iconv + instead of our own UTF-8 routines. */ + test_charset_utf8_common("UTF-8//IGNORE"); + test_end(); +} +#endif + +int main(void) +{ + static void (*test_functions[])(void) = { + test_charset_is_utf8, + test_charset_utf8, +#ifdef HAVE_ICONV + test_charset_iconv, +#endif + NULL + }; + + return test_run(test_functions); +} From dovecot at dovecot.org Sat May 16 10:22:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 May 2015 10:22:25 +0000 Subject: dovecot-2.2: lib: Added p_memdup() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aa216b4f1e0e changeset: 18715:aa216b4f1e0e user: Timo Sirainen date: Sat May 16 12:51:08 2015 +0300 description: lib: Added p_memdup() diffstat: src/lib/strfuncs.c | 9 +++++++++ src/lib/strfuncs.h | 1 + 2 files changed, 10 insertions(+), 0 deletions(-) diffs (30 lines): diff -r dcaf508860ad -r aa216b4f1e0e src/lib/strfuncs.c --- a/src/lib/strfuncs.c Sat May 16 12:46:38 2015 +0300 +++ b/src/lib/strfuncs.c Sat May 16 12:51:08 2015 +0300 @@ -44,6 +44,15 @@ return mem; } +void *p_memdup(pool_t pool, const void *data, size_t size) +{ + void *mem; + + mem = p_malloc(pool, size); + memcpy(mem, data, size); + return mem; +} + char *p_strdup_empty(pool_t pool, const char *str) { if (str == NULL || *str == '\0') diff -r dcaf508860ad -r aa216b4f1e0e src/lib/strfuncs.h --- a/src/lib/strfuncs.h Sat May 16 12:46:38 2015 +0300 +++ b/src/lib/strfuncs.h Sat May 16 12:51:08 2015 +0300 @@ -10,6 +10,7 @@ ATTR_FORMAT(3, 4); char *p_strdup(pool_t pool, const char *str) ATTR_MALLOC; +void *p_memdup(pool_t pool, const void *data, size_t size) ATTR_MALLOC; /* return NULL if str = "" */ char *p_strdup_empty(pool_t pool, const char *str) ATTR_MALLOC; /* *end isn't included */ From dovecot at dovecot.org Sat May 16 10:22:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 May 2015 10:22:25 +0000 Subject: dovecot-2.2: lib: Added UNICODE_REPLACEMENT_CHAR_UTF8 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ee240e7e4b6e changeset: 18716:ee240e7e4b6e user: Timo Sirainen date: Sat May 16 13:20:22 2015 +0300 description: lib: Added UNICODE_REPLACEMENT_CHAR_UTF8 diffstat: src/lib/unichar.h | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r aa216b4f1e0e -r ee240e7e4b6e src/lib/unichar.h --- a/src/lib/unichar.h Sat May 16 12:51:08 2015 +0300 +++ b/src/lib/unichar.h Sat May 16 13:20:22 2015 +0300 @@ -3,6 +3,7 @@ /* Character used to replace invalid input. */ #define UNICODE_REPLACEMENT_CHAR 0xfffd +#define UNICODE_REPLACEMENT_CHAR_UTF8 "\xEF\xBF\xBD" /* Characters >= base require surrogates */ #define UTF16_SURROGATE_BASE 0x10000 From dovecot at dovecot.org Sat May 16 15:49:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 May 2015 15:49:28 +0000 Subject: dovecot-2.2: lib-charset: Fixed compile warning caused by earlie... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5c905c8afb02 changeset: 18717:5c905c8afb02 user: Timo Sirainen date: Sat May 16 13:50:48 2015 +0300 description: lib-charset: Fixed compile warning caused by earlier commit. This UNICODE_REPLACEMENT_CHAR_UTF8 part of the change was actually supposed to be a separate commit.. diffstat: src/lib-charset/charset-iconv.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r ee240e7e4b6e -r 5c905c8afb02 src/lib-charset/charset-iconv.c --- a/src/lib-charset/charset-iconv.c Sat May 16 13:20:22 2015 +0300 +++ b/src/lib-charset/charset-iconv.c Sat May 16 13:50:48 2015 +0300 @@ -118,7 +118,8 @@ if (result == CHARSET_RET_INVALID_INPUT) { if (prev_invalid_pos != dest->used) { - str_append(dest, UNICODE_REPLACEMENT_CHAR_UTF8); + buffer_append(dest, UNICODE_REPLACEMENT_CHAR_UTF8, + strlen(UNICODE_REPLACEMENT_CHAR_UTF8)); prev_invalid_pos = dest->used; } pos++; From dovecot at dovecot.org Sat May 16 15:49:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 May 2015 15:49:28 +0000 Subject: dovecot-2.2: fts: Avoid excessive data stack usage with lib-fts Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a7175a018348 changeset: 18718:a7175a018348 user: Timo Sirainen date: Sat May 16 18:20:08 2015 +0300 description: fts: Avoid excessive data stack usage with lib-fts diffstat: src/plugins/fts/fts-build-mail.c | 27 ++++++++++++--------------- 1 files changed, 12 insertions(+), 15 deletions(-) diffs (40 lines): diff -r 5c905c8afb02 -r a7175a018348 src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Sat May 16 13:50:48 2015 +0300 +++ b/src/plugins/fts/fts-build-mail.c Sat May 16 18:20:08 2015 +0300 @@ -265,24 +265,21 @@ { struct fts_tokenizer *tokenizer; struct fts_filter *filter = ctx->cur_user_lang->filter; - const char *token; - const char *error; - int ret; + const char *token, *error; + int ret = 1, ret2; tokenizer = fts_user_get_index_tokenizer(ctx->update_ctx->backend->ns->user); - while ((ret = fts_tokenizer_next(tokenizer, data, size, &token, &error)) > 0) { - if (filter != NULL) { - ret = fts_filter_filter(filter, &token, &error); - if (ret == 0) - continue; - if (ret < 0) - break; + while (ret > 0) T_BEGIN { + ret = ret2 = fts_tokenizer_next(tokenizer, data, size, &token, &error); + if (ret2 > 0 && filter != NULL) + ret2 = fts_filter_filter(filter, &token, &error); + if (ret2 > 0) { + if (fts_backend_update_build_more(ctx->update_ctx, + (const void *)token, + strlen(token)) < 0) + ret = -1; } - if (fts_backend_update_build_more(ctx->update_ctx, - (const void *)token, - strlen(token)) < 0) - return -1; - } + } T_END; if (ret < 0) i_error("fts: Couldn't create indexable tokens: %s", error); return ret; From dovecot at dovecot.org Sat May 16 15:49:38 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 May 2015 15:49:38 +0000 Subject: dovecot-2.2: lib: Added buffer_get_writable_size() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3deb3fd654c6 changeset: 18719:3deb3fd654c6 user: Timo Sirainen date: Sat May 16 18:41:44 2015 +0300 description: lib: Added buffer_get_writable_size() diffstat: src/lib/buffer.c | 14 ++++++++++++++ src/lib/buffer.h | 4 ++++ 2 files changed, 18 insertions(+), 0 deletions(-) diffs (38 lines): diff -r a7175a018348 -r 3deb3fd654c6 src/lib/buffer.c --- a/src/lib/buffer.c Sat May 16 18:20:08 2015 +0300 +++ b/src/lib/buffer.c Sat May 16 18:41:44 2015 +0300 @@ -322,6 +322,20 @@ return buf->alloc; } +size_t buffer_get_writable_size(const buffer_t *_buf) +{ + const struct real_buffer *buf = (const struct real_buffer *)_buf; + + if (!buf->dynamic || buf->alloc == 0) + return buf->alloc; + + /* we reserve +1 for str_c() NUL in buffer_check_limits(), so don't + include that in our return value. otherwise the caller might + increase the buffer's alloc size unnecessarily when it just wants + to access the entire buffer. */ + return buf->alloc-1; +} + bool buffer_cmp(const buffer_t *buf1, const buffer_t *buf2) { if (buf1->used != buf2->used) diff -r a7175a018348 -r 3deb3fd654c6 src/lib/buffer.h --- a/src/lib/buffer.h Sat May 16 18:20:08 2015 +0300 +++ b/src/lib/buffer.h Sat May 16 18:41:44 2015 +0300 @@ -91,6 +91,10 @@ /* Returns the current buffer size. */ size_t buffer_get_size(const buffer_t *buf) ATTR_PURE; +/* Returns how many bytes we can write to buffer without increasing its size. + With dynamic buffers this is buffer_get_size()-1, because the extra 1 byte + is reserved for str_c()'s NUL. */ +size_t buffer_get_writable_size(const buffer_t *buf) ATTR_PURE; /* Returns TRUE if buffer contents are identical. */ bool buffer_cmp(const buffer_t *buf1, const buffer_t *buf2); From dovecot at dovecot.org Sat May 16 15:49:38 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 May 2015 15:49:38 +0000 Subject: dovecot-2.2: lib-fts: Rewrite ICU handling functions. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/309863bb69cc changeset: 18720:309863bb69cc user: Timo Sirainen date: Sat May 16 18:47:20 2015 +0300 description: lib-fts: Rewrite ICU handling functions. Some of the changes: - Use buffers instead of allocating everything from data stack. - Optimistically attempt to write the data directly to the buffers without first calculating their size. Grow the buffer if it doesn't fit first. - Use u_strFromUTF8Lenient() instead of u_strFromUTF8(). Our input is already supposed to be valid UTF-8, although we don't check if all code points are valid, while u_strFromUTF8() does check them and return failures. We don't really care about if code points are valid or not and u_strFromUTF8Lenient() passes through everything. Added unit tests to make sure all the functions work as intended and all the UTF-8 input passes through them successfully. diffstat: src/lib-fts/Makefile.am | 13 ++- src/lib-fts/fts-filter-normalizer-icu.c | 163 +++++-------------------------- src/lib-fts/fts-icu.c | 110 +++++++++++++++++++++ src/lib-fts/fts-icu.h | 17 +++ src/lib-fts/test-fts-filter.c | 34 ++++++ src/lib-fts/test-fts-icu.c | 152 +++++++++++++++++++++++++++++ 6 files changed, 351 insertions(+), 138 deletions(-) diffs (truncated from 612 to 300 lines): diff -r 3deb3fd654c6 -r 309863bb69cc src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Sat May 16 18:41:44 2015 +0300 +++ b/src/lib-fts/Makefile.am Sat May 16 18:47:20 2015 +0300 @@ -50,7 +50,9 @@ endif if BUILD_LIBICU +ICU_SOURCES = fts-icu.c NORMALIZER_LIBS = $(LIBICU_LIBS) +ICU_TESTS = test-fts-icu endif libfts_la_LIBADD = \ @@ -67,19 +69,22 @@ fts-language.c \ fts-tokenizer.c \ fts-tokenizer-address.c \ - fts-tokenizer-generic.c + fts-tokenizer-generic.c \ + $(ICU_SOURCES) noinst_HEADERS = \ fts-filter.h \ fts-filter-private.h \ + fts-icu.h \ fts-language.h \ fts-tokenizer.h \ fts-tokenizer-private.h \ fts-tokenizer-generic-private.h test_programs = \ + $(ICU_TESTS) \ + $(TEST_FTS_LANGUAGE) \ test-fts-filter \ - $(TEST_FTS_LANGUAGE) \ test-fts-tokenizer noinst_PROGRAMS = $(test_programs) @@ -89,6 +94,10 @@ ../lib/liblib.la test_deps = $(noinst_LTLIBRARIES) $(test_libs) +test_fts_icu_SOURCES = test-fts-icu.c +test_fts_icu_LDADD = fts-icu.lo $(LIBICU_LIBS) $(test_libs) +test_fts_icu_DEPENDENCIES = fts-icu.lo $(test_deps) + test_fts_filter_SOURCES = test-fts-filter.c test_fts_filter_LDADD = libfts.la $(test_libs) test_fts_filter_DEPENDENCIES = libfts.la $(test_deps) diff -r 3deb3fd654c6 -r 309863bb69cc src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Sat May 16 18:41:44 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Sat May 16 18:47:20 2015 +0300 @@ -8,112 +8,20 @@ #include "fts-language.h" #ifdef HAVE_LIBICU - -#include -#include -#include -#include -#include +#include "fts-icu.h" struct fts_filter_normalizer_icu { struct fts_filter filter; pool_t pool; const char *transliterator_id; + const UChar *transliterator_id_utf16; + unsigned int transliterator_id_utf16_len; + UTransliterator *transliterator; + buffer_t *utf16_token, *trans_token; + string_t *utf8_token; }; -/* Helper to create UTF16, which libicu wants as input. - - On input, if *dst_uchars_r > 0, it indicates the number of UChar - sized units that should be allocated for the text. However, the - function will not use the number, if the text will not fit in that - amount. - - On return *dst_uchars_r will contain the number of UChar sized units - allocated for the dst. NOT the number of bytes nor the length of the - text. */ -static void make_uchar(const char *src, UChar **dst, int32_t *dst_uchars_r) -{ - UErrorCode err = U_ZERO_ERROR; - int32_t len = strlen(src); - int32_t ustr_len = 0; - int32_t ustr_len_actual = 0; - UChar *retp = NULL; - int32_t alloc_uchars = 0; - - i_assert(dst_uchars_r != NULL); - - /* Check length required for encoded dst. */ - retp = u_strFromUTF8(NULL, 0, &ustr_len, src, len, &err); - - /* When preflighting a successful call returns a buffer overflow - error. */ - if (U_BUFFER_OVERFLOW_ERROR != err && U_FAILURE(err)) { - i_panic("Failed to estimate allocation size with lib ICU" - " u_strFromUTF8(): %s",u_errorName(err)); - } - i_assert(NULL == retp); - - err = U_ZERO_ERROR; - if (*dst_uchars_r > 0 && *dst_uchars_r > ustr_len) - alloc_uchars = *dst_uchars_r; - else - alloc_uchars = ustr_len; - alloc_uchars++; /* room for null bytes(2) */ - *dst = t_malloc(alloc_uchars * sizeof(UChar)); - *dst_uchars_r = alloc_uchars; - retp = u_strFromUTF8(*dst, alloc_uchars, &ustr_len_actual, - src, len, &err); - - if (U_FAILURE(err)) - i_panic("Lib ICU u_strFromUTF8 failed: %s", u_errorName(err)); - i_assert(retp == *dst); - i_assert(ustr_len == ustr_len_actual); -} - -static void make_utf8(const UChar *src, const char **_dst) -{ - char *dst; - char *retp = NULL; - int32_t dsize = 0; - int32_t dsize_actual = 0; - int32_t sub_num = 0; - UErrorCode err = U_ZERO_ERROR; - int32_t usrc_len = u_strlen(src); /* libicu selects different codepaths - depending if srclen -1 or not */ - - retp = u_strToUTF8WithSub(NULL, 0, &dsize, src, usrc_len, - UNICODE_REPLACEMENT_CHAR, &sub_num, &err); - - /* Preflighting can cause buffer overflow to be reported */ - if (U_BUFFER_OVERFLOW_ERROR != err && U_FAILURE(err)) { - i_panic("Failed to estimate allocation size with lib ICU" - " u_strToUTF8(): %s",u_errorName(err)); - } - i_assert(0 == sub_num); - i_assert(NULL == retp); - - dsize++; /* room for '\0' byte */ - dst = t_malloc(dsize); - err = U_ZERO_ERROR; - retp = u_strToUTF8WithSub(dst, dsize, &dsize_actual, src, usrc_len, - UNICODE_REPLACEMENT_CHAR, &sub_num, &err); - if (U_FAILURE(err)) - i_panic("Lib ICU u_strToUTF8WithSub() failed: %s", - u_errorName(err)); - if (dsize_actual >= dsize) { - i_panic("Produced UTF8 string length (%d) does not fit in " - "preflighted(%d). Buffer overflow?", - dsize_actual, dsize); - } - if (0 != sub_num) { - i_panic("UTF8 string not well formed. " - "Substitutions (%d) were made.", sub_num); - } - i_assert(retp == dst); - *_dst = dst; -} - static void fts_filter_normalizer_icu_destroy(struct fts_filter *filter) { struct fts_filter_normalizer_icu *np = @@ -152,6 +60,13 @@ np->pool = pp; np->filter = *fts_filter_normalizer_icu; np->transliterator_id = p_strdup(pp, id); + np->utf16_token = buffer_create_dynamic(pp, 128); + np->trans_token = buffer_create_dynamic(pp, 128); + np->utf8_token = buffer_create_dynamic(pp, 128); + fts_icu_utf8_to_utf16(np->utf16_token, id); + np->transliterator_id_utf16 = + p_memdup(pp, np->utf16_token->data, np->utf16_token->used); + np->transliterator_id_utf16_len = np->utf16_token->used / sizeof(UChar); *filter_r = &np->filter; return 0; } @@ -162,14 +77,11 @@ { UErrorCode err = U_ZERO_ERROR; UParseError perr; - UChar *id_uchar = NULL; - int32_t id_len_uchar = 0; memset(&perr, 0, sizeof(perr)); - make_uchar(np->transliterator_id, &id_uchar, &id_len_uchar); - - np->transliterator = utrans_openU(id_uchar, u_strlen(id_uchar), + np->transliterator = utrans_openU(np->transliterator_id_utf16, + np->transliterator_id_utf16_len, UTRANS_FORWARD, NULL, 0, &perr, &err); if (U_FAILURE(err)) { string_t *str = t_str_new(128); @@ -193,48 +105,27 @@ { struct fts_filter_normalizer_icu *np = (struct fts_filter_normalizer_icu *)filter; - UErrorCode err = U_ZERO_ERROR; - UChar *utext = NULL; - int32_t utext_cap = 0; - int32_t utext_len = -1; - int32_t utext_limit; if (np->transliterator == NULL) { if (fts_filter_normalizer_icu_create_trans(np, error_r) < 0) return -1; } - make_uchar(*token, &utext, &utext_cap); - utext_limit = u_strlen(utext); - utrans_transUChars(np->transliterator, utext, &utext_len, - utext_cap, 0, &utext_limit, &err); + fts_icu_utf8_to_utf16(np->utf16_token, *token); + buffer_append_zero(np->utf16_token, 2); + buffer_set_used_size(np->utf16_token, np->utf16_token->used-2); + buffer_set_used_size(np->trans_token, 0); + if (fts_icu_translate(np->trans_token, np->utf16_token->data, + np->utf16_token->used / sizeof(UChar), + np->transliterator, error_r) < 0) + return -1; - /* Data did not fit into utext. */ - if (utext_len > utext_cap || err == U_BUFFER_OVERFLOW_ERROR) { - /* This is a crude retry fix... Make a new utext of the - size utrans_transUChars indicated */ - utext_len++; /* room for '\0' bytes(2) */ - utext_cap = utext_len; - make_uchar(*token, &utext, &utext_cap); - i_assert(utext_cap == utext_len); - utext_limit = u_strlen(utext); - utext_len = -1; - err = U_ZERO_ERROR; - utrans_transUChars(np->transliterator, utext, - &utext_len, utext_cap, 0, - &utext_limit, &err); - } - - if (U_FAILURE(err)) { - *error_r = t_strdup_printf("utrans_transUChars() failed: %s\n", - u_errorName(err)); - return -1; - } - - if (utext_len == 0) + if (np->trans_token->used == 0) return 0; - make_utf8(utext, token); + fts_icu_utf16_to_utf8(np->utf8_token, np->trans_token->data, + np->trans_token->used / sizeof(UChar)); + *token = str_c(np->utf8_token); return 1; } diff -r 3deb3fd654c6 -r 309863bb69cc src/lib-fts/fts-icu.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/fts-icu.c Sat May 16 18:47:20 2015 +0300 @@ -0,0 +1,110 @@ +/* Copyright (c) 2014-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "unichar.h" +#include "fts-icu.h" + +void fts_icu_utf8_to_utf16(buffer_t *dest_utf16, const char *src_utf8) +{ + UErrorCode err = U_ZERO_ERROR; + unsigned int src_bytes = strlen(src_utf8); + int32_t utf16_len; + UChar *dest_data, *retp = NULL; + int32_t avail_uchars = 0; + + /* try to encode with the current buffer size */ + avail_uchars = buffer_get_writable_size(dest_utf16) / sizeof(UChar); + dest_data = buffer_get_space_unsafe(dest_utf16, 0, + buffer_get_writable_size(dest_utf16)); + retp = u_strFromUTF8Lenient(dest_data, avail_uchars, + &utf16_len, src_utf8, src_bytes, &err); + if (err == U_BUFFER_OVERFLOW_ERROR) { + /* try again with a larger buffer */ + dest_data = buffer_get_space_unsafe(dest_utf16, 0, + utf16_len * sizeof(UChar)); + err = U_ZERO_ERROR; + retp = u_strFromUTF8Lenient(dest_data, utf16_len, + &utf16_len, src_utf8, + src_bytes, &err); + } + if (U_FAILURE(err)) { + i_panic("LibICU u_strFromUTF8Lenient() failed: %s", + u_errorName(err)); From dovecot at dovecot.org Sat May 16 21:53:53 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 May 2015 21:53:53 +0000 Subject: dovecot-2.2: lib-mail: Make sure iconv state is reset between MI... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9809f68aaa36 changeset: 18721:9809f68aaa36 user: Timo Sirainen date: Sun May 17 00:51:50 2015 +0300 description: lib-mail: Make sure iconv state is reset between MIME parts. diffstat: src/lib-mail/message-decoder.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 309863bb69cc -r 9809f68aaa36 src/lib-mail/message-decoder.c --- a/src/lib-mail/message-decoder.c Sat May 16 18:47:20 2015 +0300 +++ b/src/lib-mail/message-decoder.c Sun May 17 00:51:50 2015 +0300 @@ -248,6 +248,7 @@ if (ctx->charset_trans != NULL && ctx->content_charset != NULL && strcasecmp(ctx->content_charset, ctx->charset_trans_charset) == 0) { /* already have the correct translation selected */ + charset_to_utf8_reset(ctx->charset_trans); return; } From dovecot at dovecot.org Mon May 18 10:24:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 18 May 2015 10:24:10 +0000 Subject: dovecot-2.2: lib-storage: Cleanup - separate search arg values t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e6513dd519d5 changeset: 18722:e6513dd519d5 user: Timo Sirainen date: Mon May 18 06:21:50 2015 -0400 description: lib-storage: Cleanup - separate search arg values that are set by mail_search_init() diffstat: src/lib-storage/index/index-search.c | 14 +++++++------- src/lib-storage/mail-search-args-imap.c | 2 +- src/lib-storage/mail-search.c | 28 ++++++++++++++-------------- src/lib-storage/mail-search.h | 11 +++++++---- 4 files changed, 29 insertions(+), 26 deletions(-) diffs (165 lines): diff -r 9809f68aaa36 -r e6513dd519d5 src/lib-storage/index/index-search.c --- a/src/lib-storage/index/index-search.c Sun May 17 00:51:50 2015 +0300 +++ b/src/lib-storage/index/index-search.c Mon May 18 06:21:50 2015 -0400 @@ -127,7 +127,7 @@ struct mail_search_arg *arg) { ARRAY_TYPE(keyword_indexes) keyword_indexes_arr; - const struct mail_keywords *search_kws = arg->value.keywords; + const struct mail_keywords *search_kws = arg->initialized.keywords; const unsigned int *keyword_indexes; unsigned int i, j, count; @@ -202,9 +202,9 @@ if (arg->value.flags != 0) { modseq = mail_index_modseq_lookup_flags(ctx->view, arg->value.flags, ctx->mail_ctx.seq); - } else if (arg->value.keywords != NULL) { + } else if (arg->initialized.keywords != NULL) { modseq = mail_index_modseq_lookup_keywords(ctx->view, - arg->value.keywords, ctx->mail_ctx.seq); + arg->initialized.keywords, ctx->mail_ctx.seq); } else { modseq = mail_index_modseq_lookup(ctx->view, ctx->mail_ctx.seq); @@ -260,12 +260,12 @@ return strcasecmp(arg->value.str, "INBOX") == 0; return strcmp(str, arg->value.str) == 0; case SEARCH_MAILBOX_GLOB: - if (imap_match(arg->value.mailbox_glob, box->vname) == IMAP_MATCH_YES) + if (imap_match(arg->initialized.mailbox_glob, box->vname) == IMAP_MATCH_YES) return 1; if (mail_get_special(ctx->cur_mail, MAIL_FETCH_MAILBOX_NAME, &str) < 0) return -1; - return imap_match(arg->value.mailbox_glob, str) == IMAP_MATCH_YES; + return imap_match(arg->initialized.mailbox_glob, str) == IMAP_MATCH_YES; default: return -1; } @@ -1073,11 +1073,11 @@ int ret = 0; /* mail_search_args_init() must have been called by now */ - i_assert(arg->value.search_args != NULL); + i_assert(arg->initialized.search_args != NULL); p_array_init(&arg->value.seqset, ctx->mail_ctx.args->pool, 64); if (mailbox_search_result_build(ctx->mail_ctx.transaction, - arg->value.search_args, + arg->initialized.search_args, MAILBOX_SEARCH_RESULT_FLAG_UPDATE | MAILBOX_SEARCH_RESULT_FLAG_QUEUE_SYNC, &arg->value.search_result) < 0) diff -r 9809f68aaa36 -r e6513dd519d5 src/lib-storage/mail-search-args-imap.c --- a/src/lib-storage/mail-search-args-imap.c Sun May 17 00:51:50 2015 +0300 +++ b/src/lib-storage/mail-search-args-imap.c Mon May 18 06:21:50 2015 -0400 @@ -98,7 +98,7 @@ str_append_c(dest, ')'); break; case SEARCH_KEYWORDS: { - const struct mail_keywords *kw = arg->value.keywords; + const struct mail_keywords *kw = arg->initialized.keywords; const ARRAY_TYPE(keywords) *names_arr; const char *const *namep; unsigned int i; diff -r 9809f68aaa36 -r e6513dd519d5 src/lib-storage/mail-search.c --- a/src/lib-storage/mail-search.c Sun May 17 00:51:50 2015 +0300 +++ b/src/lib-storage/mail-search.c Mon May 18 06:21:50 2015 -0400 @@ -85,8 +85,8 @@ keywords[0] = arg->value.str; keywords[1] = NULL; - i_assert(arg->value.keywords == NULL); - arg->value.keywords = + i_assert(arg->initialized.keywords == NULL); + arg->initialized.keywords = mailbox_keywords_create_valid(args->box, keywords); break; @@ -95,15 +95,15 @@ struct mail_namespace *ns = mailbox_get_namespace(args->box); - arg->value.mailbox_glob = + arg->initialized.mailbox_glob = imap_match_init(default_pool, arg->value.str, TRUE, mail_namespace_get_sep(ns)); break; } case SEARCH_INTHREAD: - thread_args = arg->value.search_args; + thread_args = arg->initialized.search_args; if (thread_args == NULL) { - arg->value.search_args = thread_args = + arg->initialized.search_args = thread_args = p_new(args->pool, struct mail_search_args, 1); thread_args->pool = args->pool; @@ -153,25 +153,25 @@ switch (arg->type) { case SEARCH_MODSEQ: case SEARCH_KEYWORDS: - if (arg->value.keywords == NULL) + if (arg->initialized.keywords == NULL) break; - mailbox_keywords_unref(&arg->value.keywords); + mailbox_keywords_unref(&arg->initialized.keywords); break; case SEARCH_MAILBOX_GLOB: - if (arg->value.mailbox_glob == NULL) + if (arg->initialized.mailbox_glob == NULL) break; - imap_match_deinit(&arg->value.mailbox_glob); + imap_match_deinit(&arg->initialized.mailbox_glob); break; case SEARCH_INTHREAD: - i_assert(arg->value.search_args->refcount > 0); + i_assert(arg->initialized.search_args->refcount > 0); if (args->refcount == 0 && arg->value.search_result != NULL) { mailbox_search_result_free( &arg->value.search_result); } - arg->value.search_args->refcount--; - arg->value.search_args->box = NULL; + arg->initialized.search_args->refcount--; + arg->initialized.search_args->box = NULL; /* fall through */ case SEARCH_SUB: case SEARCH_OR: @@ -650,8 +650,8 @@ m1->type == m2->type; } case SEARCH_INTHREAD: - return mail_search_args_equal(arg1->value.search_args, - arg2->value.search_args); + return mail_search_args_equal(arg1->initialized.search_args, + arg2->initialized.search_args); } i_unreached(); return FALSE; diff -r 9809f68aaa36 -r e6513dd519d5 src/lib-storage/mail-search.h --- a/src/lib-storage/mail-search.h Sun May 17 00:51:50 2015 +0300 +++ b/src/lib-storage/mail-search.h Mon May 18 06:21:50 2015 -0400 @@ -83,12 +83,15 @@ enum mail_search_arg_flag search_flags; enum mail_search_date_type date_type; enum mail_thread_type thread_type; + struct mail_search_modseq *modseq; + struct mail_search_result *search_result; + } value; + /* set by mail_search_args_init(): */ + struct { + struct mail_search_args *search_args; struct mail_keywords *keywords; - struct mail_search_modseq *modseq; - struct mail_search_args *search_args; - struct mail_search_result *search_result; struct imap_match_glob *mailbox_glob; - } value; + } initialized; void *context; const char *hdr_field_name; /* for SEARCH_HEADER* */ From dovecot at dovecot.org Mon May 18 10:39:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 18 May 2015 10:39:32 +0000 Subject: dovecot-2.2: lib-storage: Added mail_search_init/deinit_arg() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/eafbf9a13bbd changeset: 18723:eafbf9a13bbd user: Timo Sirainen date: Mon May 18 06:36:56 2015 -0400 description: lib-storage: Added mail_search_init/deinit_arg() For forcibly initializing/deinitializing search args. diffstat: src/lib-storage/mail-search.c | 23 ++++++++++------------- src/lib-storage/mail-search.h | 13 +++++++++++-- 2 files changed, 21 insertions(+), 15 deletions(-) diffs (94 lines): diff -r e6513dd519d5 -r eafbf9a13bbd src/lib-storage/mail-search.c --- a/src/lib-storage/mail-search.c Mon May 18 06:21:50 2015 -0400 +++ b/src/lib-storage/mail-search.c Mon May 18 06:36:56 2015 -0400 @@ -60,8 +60,7 @@ } } -static void -mail_search_args_init_sub(struct mail_search_args *args, +void mail_search_arg_init(struct mail_search_args *args, struct mail_search_arg *arg, bool change_uidsets, const ARRAY_TYPE(seq_range) *search_saved_uidset) @@ -118,9 +117,9 @@ /* fall through */ case SEARCH_SUB: case SEARCH_OR: - mail_search_args_init_sub(args, arg->value.subargs, - change_uidsets, - search_saved_uidset); + mail_search_arg_init(args, arg->value.subargs, + change_uidsets, + search_saved_uidset); break; default: break; @@ -142,12 +141,11 @@ args->box = box; if (!args->simplified) mail_search_args_simplify(args); - mail_search_args_init_sub(args, args->args, change_uidsets, - search_saved_uidset); + mail_search_arg_init(args, args->args, change_uidsets, + search_saved_uidset); } -static void mail_search_args_deinit_sub(struct mail_search_args *args, - struct mail_search_arg *arg) +void mail_search_arg_deinit(struct mail_search_arg *arg) { for (; arg != NULL; arg = arg->next) { switch (arg->type) { @@ -165,8 +163,7 @@ break; case SEARCH_INTHREAD: i_assert(arg->initialized.search_args->refcount > 0); - if (args->refcount == 0 && - arg->value.search_result != NULL) { + if (arg->value.search_result != NULL) { mailbox_search_result_free( &arg->value.search_result); } @@ -175,7 +172,7 @@ /* fall through */ case SEARCH_SUB: case SEARCH_OR: - mail_search_args_deinit_sub(args, arg->value.subargs); + mail_search_arg_deinit(arg->value.subargs); break; default: break; @@ -188,7 +185,7 @@ if (--args->init_refcount > 0) return; - mail_search_args_deinit_sub(args, args->args); + mail_search_arg_deinit(args->args); args->box = NULL; } diff -r e6513dd519d5 -r eafbf9a13bbd src/lib-storage/mail-search.h --- a/src/lib-storage/mail-search.h Mon May 18 06:21:50 2015 -0400 +++ b/src/lib-storage/mail-search.h Mon May 18 06:36:56 2015 -0400 @@ -132,9 +132,18 @@ struct mailbox *box, bool change_uidsets, const ARRAY_TYPE(seq_range) *search_saved_uidset) ATTR_NULL(4); -/* Free keywords. The args can initialized afterwards again if needed. - The args can be reused for other queries after calling this. */ +/* Initialize arg and its children. args is used for getting mailbox and + pool. */ +void mail_search_arg_init(struct mail_search_args *args, + struct mail_search_arg *arg, + bool change_uidsets, + const ARRAY_TYPE(seq_range) *search_saved_uidset); +/* Free memory allocated by mail_search_args_init(). The args can initialized + afterwards again if needed. The args can be reused for other queries after + calling this. */ void mail_search_args_deinit(struct mail_search_args *args); +/* Free arg and its children. */ +void mail_search_arg_deinit(struct mail_search_arg *arg); /* Convert sequence sets in args to UIDs. */ void mail_search_args_seq2uid(struct mail_search_args *args); /* Returns TRUE if the two search arguments are fully compatible. From dovecot at dovecot.org Mon May 18 10:39:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 18 May 2015 10:39:34 +0000 Subject: dovecot-2.2: fts + lib-fts: Fixed crash with some search queries... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3d2bd49e6b4a changeset: 18724:3d2bd49e6b4a user: Timo Sirainen date: Mon May 18 06:37:28 2015 -0400 description: fts + lib-fts: Fixed crash with some search queries that contained uninitialized search args. diffstat: src/plugins/fts/fts-search-args.c | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diffs (28 lines): diff -r eafbf9a13bbd -r 3d2bd49e6b4a src/plugins/fts/fts-search-args.c --- a/src/plugins/fts/fts-search-args.c Mon May 18 06:36:56 2015 -0400 +++ b/src/plugins/fts/fts-search-args.c Mon May 18 06:37:28 2015 -0400 @@ -183,11 +183,12 @@ int fts_search_args_expand(struct fts_backend *backend, struct mail_search_args *args) { - struct mail_search_arg *args_dup; + struct mail_search_arg *args_dup, *orig_args = args->args; /* duplicate the args, so if expansion fails we haven't changed anything */ args_dup = mail_search_arg_dup(args->pool, args->args); + if (fts_search_args_expand_tree(backend, args->pool, &args_dup) < 0) return -1; @@ -195,5 +196,10 @@ args->simplified = FALSE; args->args = args_dup; mail_search_args_simplify(args); + + /* duplicated args aren't initialized */ + i_assert(args->init_refcount > 0); + mail_search_arg_init(args, args_dup, FALSE, NULL); + mail_search_arg_deinit(orig_args); return 0; } From dovecot at dovecot.org Mon May 18 10:48:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 18 May 2015 10:48:39 +0000 Subject: dovecot-2.2: lib-fts: test-fts-icu memory leak fix Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c2ce65bb1caf changeset: 18725:c2ce65bb1caf user: Timo Sirainen date: Mon May 18 06:46:32 2015 -0400 description: lib-fts: test-fts-icu memory leak fix diffstat: src/lib-fts/test-fts-icu.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (19 lines): diff -r 3d2bd49e6b4a -r c2ce65bb1caf src/lib-fts/test-fts-icu.c --- a/src/lib-fts/test-fts-icu.c Mon May 18 06:37:28 2015 -0400 +++ b/src/lib-fts/test-fts-icu.c Mon May 18 06:46:32 2015 -0400 @@ -108,6 +108,7 @@ translit, &error) == 0); test_assert(dest->used == i * sizeof(UChar)); } + utrans_close(translit); test_end(); } @@ -134,6 +135,7 @@ translit, &error) == 0); } + utrans_close(translit); test_end(); } From dovecot at dovecot.org Mon May 18 10:55:26 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 18 May 2015 10:55:26 +0000 Subject: dovecot-2.2: lib-fts: Call u_clean() at deinit to free up all of... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d097a9779c37 changeset: 18726:d097a9779c37 user: Timo Sirainen date: Mon May 18 06:51:24 2015 -0400 description: lib-fts: Call u_clean() at deinit to free up all of libicu's memory. diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 9 +++++++++ src/lib-fts/test-fts-icu.c | 6 +++++- 2 files changed, 14 insertions(+), 1 deletions(-) diffs (54 lines): diff -r c2ce65bb1caf -r d097a9779c37 src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Mon May 18 06:46:32 2015 -0400 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Mon May 18 06:51:24 2015 -0400 @@ -10,6 +10,8 @@ #ifdef HAVE_LIBICU #include "fts-icu.h" +#include + struct fts_filter_normalizer_icu { struct fts_filter filter; pool_t pool; @@ -22,6 +24,8 @@ string_t *utf8_token; }; +static bool icu_exit_callback_set = FALSE; + static void fts_filter_normalizer_icu_destroy(struct fts_filter *filter) { struct fts_filter_normalizer_icu *np = @@ -54,6 +58,11 @@ } } + if (!icu_exit_callback_set) { + icu_exit_callback_set = TRUE; + lib_atexit(u_cleanup); + } + pp = pool_alloconly_create(MEMPOOL_GROWING"fts_filter_normalizer_icu", sizeof(struct fts_filter_normalizer_icu)); np = p_new(pp, struct fts_filter_normalizer_icu, 1); diff -r c2ce65bb1caf -r d097a9779c37 src/lib-fts/test-fts-icu.c --- a/src/lib-fts/test-fts-icu.c Mon May 18 06:46:32 2015 -0400 +++ b/src/lib-fts/test-fts-icu.c Mon May 18 06:51:24 2015 -0400 @@ -7,6 +7,8 @@ #include "test-common.h" #include "fts-icu.h" +#include + static void test_fts_icu_utf8_to_utf16_ascii_resize(void) { buffer_t *dest = buffer_create_dynamic(pool_datastack_create(), 5); @@ -150,5 +152,7 @@ test_fts_icu_translate_resize, NULL }; - return test_run(test_functions); + int ret = test_run(test_functions); + u_cleanup(); + return ret; } From dovecot at dovecot.org Mon May 18 10:55:26 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 18 May 2015 10:55:26 +0000 Subject: dovecot-2.2: lib-fts: Fixed memory leaks in test-fts-filter unit... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/713476fa84af changeset: 18727:713476fa84af user: Timo Sirainen date: Mon May 18 06:53:20 2015 -0400 description: lib-fts: Fixed memory leaks in test-fts-filter unit test diffstat: src/lib-fts/test-fts-filter.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (19 lines): diff -r d097a9779c37 -r 713476fa84af src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Mon May 18 06:51:24 2015 -0400 +++ b/src/lib-fts/test-fts-filter.c Mon May 18 06:53:20 2015 -0400 @@ -200,6 +200,7 @@ test_assert(fts_filter_create(fts_filter_stopwords, NULL, &unknown, stopword_settings, &filter, &error) == 0); test_assert(filter != NULL && error == NULL); test_assert(fts_filter_filter(filter, &token, &error) < 0 && error != NULL); + fts_filter_unref(&filter); test_end(); } @@ -508,6 +509,7 @@ test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0); test_assert(error == NULL); test_assert(fts_filter_filter(norm, &token, &error) < 0 && error != NULL); + fts_filter_unref(&norm); test_end(); } From dovecot at dovecot.org Mon May 18 10:58:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 18 May 2015 10:58:11 +0000 Subject: dovecot-2.2: lib-charset: test-charset unit test - Don't use inv... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0d815b4db957 changeset: 18728:0d815b4db957 user: Timo Sirainen date: Mon May 18 13:56:07 2015 +0300 description: lib-charset: test-charset unit test - Don't use invalid UTF-8 text in source code diffstat: src/lib-charset/test-charset.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 713476fa84af -r 0d815b4db957 src/lib-charset/test-charset.c --- a/src/lib-charset/test-charset.c Mon May 18 06:53:20 2015 -0400 +++ b/src/lib-charset/test-charset.c Mon May 18 13:56:07 2015 +0300 @@ -22,8 +22,8 @@ const char *output; enum charset_result result; } tests[] = { - { "p???", "p??", CHARSET_RET_INCOMPLETE_INPUT }, - { "p???a", "p??"UNICODE_REPLACEMENT_CHAR_UTF8"a", CHARSET_RET_INVALID_INPUT } + { "p\xC3\xA4\xC3", "p\xC3\xA4", CHARSET_RET_INCOMPLETE_INPUT }, + { "p\xC3\xA4\xC3""a", "p\xC3\xA4"UNICODE_REPLACEMENT_CHAR_UTF8"a", CHARSET_RET_INVALID_INPUT } }; string_t *str = t_str_new(128); enum charset_result result; From dovecot at dovecot.org Mon May 18 11:51:21 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 18 May 2015 11:51:21 +0000 Subject: dovecot-2.2: director: Added "up" vs "down" states and doveadm d... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0ee3e734249a changeset: 18729:0ee3e734249a user: Timo Sirainen date: Mon May 18 07:49:15 2015 -0400 description: director: Added "up" vs "down" states and doveadm director up/down commands. These commands are intended to be used by automated watchdogs that detect if backends are up or down. This way the vhost count doesn't get forgotten after server goes down. It also means that admin can manually take down a server by setting its vhost count to 0 without the watchdog automatically bringing it back up. diffstat: src/director/director-connection.c | 36 ++++++++++++++++++++---- src/director/director.c | 16 ++++++---- src/director/director.h | 4 ++- src/director/doveadm-connection.c | 33 ++++++++++++++++++++++ src/director/mail-host.c | 15 ++++++++++ src/director/mail-host.h | 6 ++++ src/doveadm/doveadm-director.c | 57 ++++++++++++++++++++++++++++++------- 7 files changed, 142 insertions(+), 25 deletions(-) diffs (truncated from 364 to 300 lines): diff -r 0d815b4db957 -r 0ee3e734249a src/director/director-connection.c --- a/src/director/director-connection.c Mon May 18 13:56:07 2015 +0300 +++ b/src/director/director-connection.c Mon May 18 07:49:15 2015 -0400 @@ -846,7 +846,8 @@ struct ip_addr ip; const char *tag = ""; unsigned int vhost_count; - bool update; + bool update, down = FALSE; + time_t last_updown_change = 0; if (str_array_length(args) < 2 || net_addr2ip(args[0], &ip) < 0 || @@ -854,8 +855,17 @@ director_cmd_error(conn, "Invalid parameters"); return FALSE; } - if (args[2] != NULL) + if (args[2] != NULL) { tag = args[2]; + if (args[3] != NULL) { + if ((args[3][0] != 'D' && args[3][0] != 'U') || + str_to_time(args[3]+1, &last_updown_change) < 0) { + director_cmd_error(conn, "Invalid updown parameters"); + return FALSE; + } + down = args[3][0] == 'D'; + } + } if (conn->ignore_host_events) { /* remote is sending hosts in a handshake, but it doesn't have a completed ring and we do. */ @@ -868,7 +878,10 @@ host = mail_host_add_ip(conn->dir->mail_hosts, &ip, tag); update = TRUE; } else { - update = host->vhost_count != vhost_count; + update = host->vhost_count != vhost_count || + host->down != down || + host->last_updown_change != last_updown_change; +; if (strcmp(tag, host->tag) != 0) { i_error("director(%s): Host %s changed tag from '%s' to '%s'", conn->name, net_ip2addr(&host->ip), @@ -879,6 +892,8 @@ } if (update) { + mail_host_set_down(conn->dir->mail_hosts, host, + down, last_updown_change); mail_host_set_vhost_count(conn->dir->mail_hosts, host, vhost_count); director_update_host(conn->dir, conn->host, dir_host, host); @@ -1581,14 +1596,23 @@ director_connection_send_hosts(struct director_connection *conn, string_t *str) { struct mail_host *const *hostp; + bool send_updowns; + + send_updowns = conn->minor_version >= DIRECTOR_VERSION_UPDOWN; str_printfa(str, "HOST-HAND-START\t%u\n", conn->dir->ring_handshaked); array_foreach(mail_hosts_get(conn->dir->mail_hosts), hostp) { + struct mail_host *host = *hostp; + str_printfa(str, "HOST\t%s\t%u", - net_ip2addr(&(*hostp)->ip), (*hostp)->vhost_count); - if ((*hostp)->tag[0] != '\0') { + net_ip2addr(&host->ip), host->vhost_count); + if (host->tag[0] != '\0' || send_updowns) { str_append_c(str, '\t'); - str_append_tabescaped(str, (*hostp)->tag); + str_append_tabescaped(str, host->tag); + } + if (send_updowns) { + str_printfa(str, "\t%c%ld", host->down ? 'D' : 'U', + (long)host->last_updown_change); } str_append_c(str, '\n'); } diff -r 0d815b4db957 -r 0ee3e734249a src/director/director.c --- a/src/director/director.c Mon May 18 13:56:07 2015 +0300 +++ b/src/director/director.c Mon May 18 07:49:15 2015 -0400 @@ -533,17 +533,19 @@ net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq, net_ip2addr(&host->ip), host->vhost_count); - if (host->tag[0] == '\0') - ; - else if (dir->ring_handshaked && - dir->ring_min_version < DIRECTOR_VERSION_TAGS) { + if (dir->ring_min_version >= DIRECTOR_VERSION_TAGS) { + str_append_c(str, '\t'); + str_append_tabescaped(str, host->tag); + } else if (host->tag[0] != '\0' && + dir->ring_min_version < DIRECTOR_VERSION_TAGS) { i_error("Ring has directors that don't support tags - removing host %s with tag '%s'", net_ip2addr(&host->ip), host->tag); director_remove_host(dir, NULL, NULL, host); return; - } else { - str_append_c(str, '\t'); - str_append_tabescaped(str, host->tag); + } + if (dir->ring_min_version >= DIRECTOR_VERSION_UPDOWN) { + str_printfa(str, "\t%c%ld", host->down ? 'D' : 'U', + (long)host->last_updown_change); } str_append_c(str, '\n'); director_update_send(dir, src, str_c(str)); diff -r 0d815b4db957 -r 0ee3e734249a src/director/director.h --- a/src/director/director.h Mon May 18 13:56:07 2015 +0300 +++ b/src/director/director.h Mon May 18 07:49:15 2015 -0400 @@ -6,7 +6,7 @@ #define DIRECTOR_VERSION_NAME "director" #define DIRECTOR_VERSION_MAJOR 1 -#define DIRECTOR_VERSION_MINOR 5 +#define DIRECTOR_VERSION_MINOR 6 /* weak users supported in protocol */ #define DIRECTOR_VERSION_WEAK_USERS 1 @@ -20,6 +20,8 @@ #define DIRECTOR_VERSION_OPTIONS 5 /* user tags supported */ #define DIRECTOR_VERSION_TAGS 5 +/* up/down state is tracked */ +#define DIRECTOR_VERSION_UPDOWN 6 /* Minimum time between even attempting to communicate with a director that failed due to a protocol error. */ diff -r 0d815b4db957 -r 0ee3e734249a src/director/doveadm-connection.c --- a/src/director/doveadm-connection.c Mon May 18 13:56:07 2015 +0300 +++ b/src/director/doveadm-connection.c Mon May 18 07:49:15 2015 -0400 @@ -51,6 +51,8 @@ net_ip2addr(&(*hostp)->ip), (*hostp)->vhost_count, (*hostp)->user_count); str_append_tabescaped(str, (*hostp)->tag); + str_printfa(str, "\t%c\t%ld", (*hostp)->down ? 'D' : 'U', + (long)(*hostp)->last_updown_change); str_append_c(str, '\n'); } str_append_c(str, '\n'); @@ -285,6 +287,33 @@ } static bool +doveadm_cmd_host_updown(struct doveadm_connection *conn, bool down, + const char *line) +{ + struct mail_host *host; + struct ip_addr ip; + + if (net_addr2ip(line, &ip) < 0) { + i_error("doveadm sent invalid %s parameters: %s", + down ? "HOST-DOWN" : "HOST-UP", line); + return FALSE; + } + host = mail_host_lookup(conn->dir->mail_hosts, &ip); + if (host == NULL) { + o_stream_nsend_str(conn->output, "NOTFOUND\n"); + return TRUE; + } + if (host->down != down) { + mail_host_set_down(conn->dir->mail_hosts, host, + down, ioloop_time); + director_update_host(conn->dir, conn->dir->self_host, + NULL, host); + } + o_stream_nsend(conn->output, "OK\n", 3); + return TRUE; +} + +static bool doveadm_cmd_host_remove(struct doveadm_connection *conn, const char *line) { struct mail_host *host; @@ -587,6 +616,10 @@ ret = doveadm_cmd_director_remove(conn, args); else if (strcmp(cmd, "HOST-SET") == 0) ret = doveadm_cmd_host_set(conn, args); + else if (strcmp(cmd, "HOST-UP") == 0) + ret = doveadm_cmd_host_updown(conn, FALSE, args); + else if (strcmp(cmd, "HOST-DOWN") == 0) + ret = doveadm_cmd_host_updown(conn, TRUE, args); else if (strcmp(cmd, "HOST-REMOVE") == 0) ret = doveadm_cmd_host_remove(conn, args); else if (strcmp(cmd, "HOST-FLUSH") == 0) diff -r 0d815b4db957 -r 0ee3e734249a src/director/mail-host.c --- a/src/director/mail-host.c Mon May 18 13:56:07 2015 +0300 +++ b/src/director/mail-host.c Mon May 18 07:49:15 2015 -0400 @@ -58,6 +58,9 @@ char num_str[MAX_INT_STRLEN]; unsigned int i, j; + if (host->down) + return; + ip_str = net_ip2addr(&host->ip); md5_init(&md5_ctx); @@ -99,6 +102,8 @@ /* rebuild vhosts */ array_clear(&list->vhosts); array_foreach(&list->hosts, hostp) { + if ((*hostp)->down) + continue; for (i = 0; i < (*hostp)->vhost_count; i++) { vhost = array_append_space(&list->vhosts); vhost->host = *hostp; @@ -269,6 +274,16 @@ host->tag = i_strdup(tag); } +void mail_host_set_down(struct mail_host_list *list, + struct mail_host *host, bool down, time_t timestamp) +{ + if (host->down != down) { + host->down = down; + host->last_updown_change = timestamp; + list->hosts_unsorted = TRUE; + } +} + void mail_host_set_vhost_count(struct mail_host_list *list, struct mail_host *host, unsigned int vhost_count) { diff -r 0d815b4db957 -r 0ee3e734249a src/director/mail-host.h --- a/src/director/mail-host.h Mon May 18 13:56:07 2015 +0300 +++ b/src/director/mail-host.h Mon May 18 07:49:15 2015 -0400 @@ -8,6 +8,10 @@ struct mail_host { unsigned int user_count; unsigned int vhost_count; + /* server up/down. down=TRUE has effectively the same result as if + vhost_count=0. */ + bool down; + time_t last_updown_change; struct ip_addr ip; char *tag; @@ -26,6 +30,8 @@ int mail_hosts_parse_and_add(struct mail_host_list *list, const char *hosts_string); void mail_host_set_tag(struct mail_host *host, const char *tag); +void mail_host_set_down(struct mail_host_list *list, + struct mail_host *host, bool down, time_t timestamp); void mail_host_set_vhost_count(struct mail_host_list *list, struct mail_host *host, unsigned int vhost_count); diff -r 0d815b4db957 -r 0ee3e734249a src/doveadm/doveadm-director.c --- a/src/doveadm/doveadm-director.c Mon May 18 13:56:07 2015 +0300 +++ b/src/doveadm/doveadm-director.c Mon May 18 07:49:15 2015 -0400 @@ -176,21 +176,32 @@ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); doveadm_print_header_simple("mail server ip"); doveadm_print_header_simple("tag"); - doveadm_print_header("vhosts", "vhosts", - DOVEADM_PRINT_HEADER_FLAG_RIGHT_JUSTIFY); - doveadm_print_header("users", "users", - DOVEADM_PRINT_HEADER_FLAG_RIGHT_JUSTIFY); + doveadm_print_header_simple("vhosts"); + doveadm_print_header_simple("state"); + doveadm_print_header("state-changed", "state changed", 0); + doveadm_print_header_simple("users"); director_send(ctx, "HOST-LIST\n"); while ((line = i_stream_read_next_line(ctx->input)) != NULL) { if (*line == '\0') break; T_BEGIN { + unsigned int arg_count; + time_t ts; + args = t_strsplit_tab(line); - if (str_array_length(args) >= 4) { - doveadm_print(args[0]); + arg_count = str_array_length(args); + if (arg_count >= 6) { + /* ip vhosts users tag updown updown-ts */ + doveadm_print(args[0]); doveadm_print(args[3]); doveadm_print(args[1]); + doveadm_print(args[4][0] == 'D' ? "down" : "up"); + if (str_to_time(args[5], &ts) < 0 || + ts <= 0) + doveadm_print("-"); + else + doveadm_print(unixdate2str(ts)); doveadm_print(args[2]); } } T_END; @@ -447,22 +458,24 @@ director_disconnect(ctx); } -static void cmd_director_remove(int argc, char *argv[]) +static void +cmd_director_ipcmd(const char *cmd_name, doveadm_command_t *cmd, + const char *success_result, int argc, char *argv[]) From dovecot at dovecot.org Mon May 18 11:55:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 18 May 2015 11:55:55 +0000 Subject: dovecot-2.2: lib-fts: Partially reverted d097a9779c37 - don't us... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/45013c8cf69c changeset: 18730:45013c8cf69c user: Timo Sirainen date: Mon May 18 14:53:52 2015 +0300 description: lib-fts: Partially reverted d097a9779c37 - don't use lib_atexit() Because fts is loaded as plugin lib_atexit() is called after the plugin is already unloaded, so it crashes. diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 9 --------- 1 files changed, 0 insertions(+), 9 deletions(-) diffs (33 lines): diff -r 0ee3e734249a -r 45013c8cf69c src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Mon May 18 07:49:15 2015 -0400 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Mon May 18 14:53:52 2015 +0300 @@ -10,8 +10,6 @@ #ifdef HAVE_LIBICU #include "fts-icu.h" -#include - struct fts_filter_normalizer_icu { struct fts_filter filter; pool_t pool; @@ -24,8 +22,6 @@ string_t *utf8_token; }; -static bool icu_exit_callback_set = FALSE; - static void fts_filter_normalizer_icu_destroy(struct fts_filter *filter) { struct fts_filter_normalizer_icu *np = @@ -58,11 +54,6 @@ } } - if (!icu_exit_callback_set) { - icu_exit_callback_set = TRUE; - lib_atexit(u_cleanup); - } - pp = pool_alloconly_create(MEMPOOL_GROWING"fts_filter_normalizer_icu", sizeof(struct fts_filter_normalizer_icu)); np = p_new(pp, struct fts_filter_normalizer_icu, 1); From pigeonhole at rename-it.nl Tue May 19 19:31:19 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 19 May 2015 21:31:19 +0200 Subject: dovecot-2.2-pigeonhole: managesieve-login: settings: Changed def... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/9caff604a21e changeset: 2073:9caff604a21e user: Stephan Bosch date: Tue May 19 21:23:52 2015 +0200 description: managesieve-login: settings: Changed default listener initialization to include field names. diffstat: src/managesieve-login/managesieve-login-settings.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 5b17177abce9 -r 9caff604a21e src/managesieve-login/managesieve-login-settings.c --- a/src/managesieve-login/managesieve-login-settings.c Fri May 15 20:54:00 2015 +0200 +++ b/src/managesieve-login/managesieve-login-settings.c Tue May 19 21:23:52 2015 +0200 @@ -23,7 +23,7 @@ /* */ static struct inet_listener_settings managesieve_login_inet_listeners_array[] = { - { "sieve", "", 4190, FALSE, FALSE }, + { .name = "sieve", .address = "", .port = 4190 }, }; static struct inet_listener_settings *managesieve_login_inet_listeners[] = { &managesieve_login_inet_listeners_array[0] From dovecot at dovecot.org Thu May 21 10:39:12 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 21 May 2015 10:39:12 +0000 Subject: dovecot-2.2: lib-fts: Fix tr29 tokenizer apostrophe handling. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5ca59cffbf2f changeset: 18731:5ca59cffbf2f user: Teemu Huovila date: Thu May 21 06:17:32 2015 -0400 description: lib-fts: Fix tr29 tokenizer apostrophe handling. U+0027, which is called Single Quote in tr29, was not properly handled as a word boundary. diffstat: src/lib-fts/fts-tokenizer-generic.c | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) diffs (26 lines): diff -r 45013c8cf69c -r 5ca59cffbf2f src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon May 18 14:53:52 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Thu May 21 06:17:32 2015 -0400 @@ -464,8 +464,8 @@ if (lt == LETTER_TYPE_REGIONAL_INDICATOR || lt == LETTER_TYPE_KATAKANA || lt == LETTER_TYPE_HEBREW_LETTER || lt == LETTER_TYPE_ALETTER || - lt == LETTER_TYPE_SINGLE_QUOTE || lt == LETTER_TYPE_NUMERIC) - return FALSE; /* TODO: Include LETTER_TYPE_DOUBLE_QUOTE? */ + lt == LETTER_TYPE_NUMERIC) + return FALSE; return TRUE; } @@ -535,8 +535,9 @@ http://www.unicode.org/reports/tr29/ Adaptions: No word boundary at Start-Of-Text or End-of-Text (Wb1 and - WB2). Break just once, not before and after. Other things also, not - really pure tr29. Meant to assist in finding individual words. + WB2). Break just once, not before and after. Other things also + (e.g. is_nonword(), not really pure tr29. Meant to assist in finding + individual words. TODO: If this letter_fns based approach is too kludgy, do a FSM with function pointers and transition tables. From dovecot at dovecot.org Thu May 21 10:39:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 21 May 2015 10:39:16 +0000 Subject: dovecot-2.2: lib-fts: Fix simple tokenizer apostrophe handling. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6c655ce3b857 changeset: 18732:6c655ce3b857 user: Teemu Huovila date: Thu May 21 06:29:15 2015 -0400 description: lib-fts: Fix simple tokenizer apostrophe handling. Apostrophes and quotation marks are now treated as word breaks, except U+0027 between non-wordbrek characters. The characters U+2019 and U+FF07 are transformed to U+0027 before processing. diffstat: src/lib-fts/fts-tokenizer-generic-private.h | 3 +- src/lib-fts/fts-tokenizer-generic.c | 110 ++++++++++++++++++++------- src/lib-fts/test-fts-tokenizer.c | 9 ++ src/lib-fts/word-properties.pl | 2 +- 4 files changed, 93 insertions(+), 31 deletions(-) diffs (254 lines): diff -r 5ca59cffbf2f -r 6c655ce3b857 src/lib-fts/fts-tokenizer-generic-private.h --- a/src/lib-fts/fts-tokenizer-generic-private.h Thu May 21 06:17:32 2015 -0400 +++ b/src/lib-fts/fts-tokenizer-generic-private.h Thu May 21 06:29:15 2015 -0400 @@ -40,8 +40,7 @@ struct fts_tokenizer tokenizer; unsigned int max_length; enum boundary_algorithm algorithm; - enum letter_type prev_letter; /* These two are basically the - state of the parsing. */ + enum letter_type prev_letter; enum letter_type prev_prev_letter; size_t last_size; /* Bytes in latest utf8 character. */ buffer_t *token; diff -r 5ca59cffbf2f -r 6c655ce3b857 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Thu May 21 06:17:32 2015 -0400 +++ b/src/lib-fts/fts-tokenizer-generic.c Thu May 21 06:29:15 2015 -0400 @@ -11,7 +11,12 @@ #define FTS_DEFAULT_TOKEN_MAX_LENGTH 30 -static unsigned char fts_ascii_word_boundaries[128] = { +#define IS_NONASCII_APOSTROPHE(c) \ + ((c) == 0x2019 || (c) == 0xFF07) +#define IS_APOSTROPHE(c) \ + ((c) == 0x0027 || IS_NONASCII_APOSTROPHE(c)) + +static unsigned char fts_ascii_word_breaks[128] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0-15 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 16-31 */ @@ -95,34 +100,60 @@ return t_strndup(data, pos); } -static void +static bool fts_tokenizer_generic_simple_current_token(struct generic_fts_tokenizer *tok, const char **token_r) { - *token_r = fts_uni_strndup(tok->token->data, tok->token->used); + const unsigned char *data; + size_t start = 0, len; + + /* clean trailing and starting apostrophes. they were all made + into U+0027 earlier. */ + data = tok->token->data; + len = tok->token->used; + while (len > 0 && data[len - 1] == '\'') + len--; + while (start < len && data[start] == '\'') + start++; + + *token_r = len - start == 0 ? "" : + fts_uni_strndup(CONST_PTR_OFFSET(tok->token->data, start), + len - start); buffer_set_used_size(tok->token, 0); + return (*token_r)[0] != '\0'; } -/* TODO: This is duplicated from unichar.c */ static bool uint32_find(const uint32_t *data, unsigned int count, uint32_t value, unsigned int *idx_r) { BINARY_NUMBER_SEARCH(data, count, value, idx_r); } -static bool is_word_break(unichar_t c) +static bool fts_ascii_word_break(unsigned char c) +{ + if (c < 0x80) + return fts_ascii_word_breaks[c] != 0; + return FALSE; +} + +static bool fts_uni_word_break(unichar_t c) { unsigned int idx; + /* Override some apostrophes, which get special treatment. */ + if (IS_APOSTROPHE(c)) + return FALSE; + /* Unicode General Punctuation, including deprecated characters. */ if (c >= 0x2000 && c <= 0x206f) return TRUE; - /* From word-break-data.c, which is generated from PropList.txt. */ if (uint32_find(White_Space, N_ELEMENTS(White_Space), c, &idx)) return TRUE; if (uint32_find(Dash, N_ELEMENTS(Dash), c, &idx)) return TRUE; + if (uint32_find(Quotation_Mark, N_ELEMENTS(Quotation_Mark), c, &idx)) + return TRUE; if (uint32_find(Terminal_Punctuation, N_ELEMENTS(Terminal_Punctuation), c, &idx)) return TRUE; if (uint32_find(STerm, N_ELEMENTS(STerm), c, &idx)) @@ -133,17 +164,17 @@ } static bool -data_is_word_boundary(const unsigned char *data, size_t size, size_t *i) +fts_apostrophe_word_break(struct generic_fts_tokenizer *tok, unichar_t c) { - unichar_t c; - - if (data[*i] < 0x80) - return fts_ascii_word_boundaries[data[*i]] != 0; - /* unicode punctuation? */ - if (uni_utf8_get_char_n(data + *i, size - *i, &c) <= 0) - i_unreached(); - *i += uni_utf8_char_bytes(data[*i]) - 1; - return is_word_break(c); + if (IS_APOSTROPHE(c)) { + if (tok->prev_letter == LETTER_TYPE_SINGLE_QUOTE) + return TRUE; + else + tok->prev_letter = LETTER_TYPE_SINGLE_QUOTE; + } else { + tok->prev_letter = LETTER_TYPE_NONE; + } + return FALSE; } static void fts_tokenizer_generic_reset(struct fts_tokenizer *_tok) @@ -160,10 +191,26 @@ static void tok_append_truncated(struct generic_fts_tokenizer *tok, const unsigned char *data, size_t size) { + size_t append_len, pos = 0, appended = 0; + unichar_t c; + i_assert(tok->max_length >= tok->token->used); + append_len = I_MIN(size, tok->max_length - tok->token->used); - buffer_append(tok->token, data, - I_MIN(size, tok->max_length - tok->token->used)); + /* Append only one kind of apostrophes. Simplifies things when returning + token. */ + while (pos < append_len) { + if (uni_utf8_get_char_n(data + pos, size - pos, &c) <= 0) + i_unreached(); + if (IS_NONASCII_APOSTROPHE(c)) { + buffer_append(tok->token, data, pos); + buffer_append_c(tok->token, '\''); + appended = pos + 1; + } + pos += uni_utf8_char_bytes(data[pos]); + } + if (appended < append_len) + buffer_append(tok->token, data + appended, append_len - appended); } static int @@ -175,21 +222,27 @@ struct generic_fts_tokenizer *tok = (struct generic_fts_tokenizer *)_tok; size_t i, char_start_i, len, start = 0; + unsigned int char_size; + unichar_t c; - for (i = 0; i < size; i++) { + for (i = 0; i < size; i += char_size) { char_start_i = i; - if (data_is_word_boundary(data, size, &i)) { + if (uni_utf8_get_char_n(data + i, size - i, &c) <= 0) + i_unreached(); + char_size = uni_utf8_char_bytes(data[i]); + if (fts_ascii_word_break(data[i]) || fts_uni_word_break(c) || + fts_apostrophe_word_break(tok, c)) { len = char_start_i - start; tok_append_truncated(tok, data + start, len); if (tok->token->used == 0) { - /* no text read yet */ - start = i + 1; + start = i + char_size; continue; } - /* word boundary found - return a new token */ - *skip_r = i + 1; - fts_tokenizer_generic_simple_current_token(tok, token_r); - return 1; + + if (fts_tokenizer_generic_simple_current_token(tok, token_r)) { + *skip_r = i + char_size; + return 1; + } } } /* word boundary not found yet */ @@ -199,9 +252,10 @@ /* return the last token */ if (size == 0 && tok->token->used > 0) { - fts_tokenizer_generic_simple_current_token(tok, token_r); - return 1; + if (fts_tokenizer_generic_simple_current_token(tok, token_r)) + return 1; } + return 0; } diff -r 5ca59cffbf2f -r 6c655ce3b857 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Thu May 21 06:17:32 2015 -0400 +++ b/src/lib-fts/test-fts-tokenizer.c Thu May 21 06:29:15 2015 -0400 @@ -29,6 +29,8 @@ "1.", + "'quoted text' 'word' 'hlo words' you're bad'''word '''pre post'''", + /* whitespace: with Unicode(utf8) U+FF01(ef bc 81)(U+2000(e2 80 80) and U+205A(e2 81 9a) and U+205F(e2 81 9f) */ "hello\xEF\xBC\x81world\r\nAnd\xE2\x80\x80there\twas: text " @@ -99,6 +101,7 @@ outi++; } test_assert_idx(expected_output[outi] == NULL, outi); + return outi+1; } @@ -130,6 +133,9 @@ "1", NULL, + "quoted", "text", "word", "hlo", "words", "you're", "bad", + "word", "pre", "post", NULL, + "hello", "world", "And", "there", "was", "text", "galore", "and", "more", NULL, @@ -169,6 +175,9 @@ "1", NULL, + "quoted", "text", "word", "hlo", "words", "you're", "bad", + "word", "pre", "post", NULL, + "hello", "world", "And", "there", "was", "text", "galore", "and", "more", NULL, diff -r 5ca59cffbf2f -r 6c655ce3b857 src/lib-fts/word-properties.pl --- a/src/lib-fts/word-properties.pl Thu May 21 06:17:32 2015 -0400 +++ b/src/lib-fts/word-properties.pl Thu May 21 06:29:15 2015 -0400 @@ -8,7 +8,7 @@ @categories = qw(CR LF Newline Extend Regional_Indicator Format Katakana Hebrew_Letter ALetter Single_Quote Double_Quote MidNumLet MidLetter MidNum Numeric ExtendNumLet); } elsif ($which eq 'breaks') { - @categories = qw(White_Space Dash Terminal_Punctuation STerm Pattern_White_Space); + @categories = qw(White_Space Dash Quotation_Mark Terminal_Punctuation STerm Pattern_White_Space); } else { die "specify 'boundaries' or 'breaks'"; } From dovecot at dovecot.org Thu May 21 10:39:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 21 May 2015 10:39:16 +0000 Subject: dovecot-2.2: lib-fts: Fixed handling tokens that contain only ap... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5b902db0cabc changeset: 18733:5b902db0cabc user: Timo Sirainen date: Thu May 21 06:35:59 2015 -0400 description: lib-fts: Fixed handling tokens that contain only apostrophes diffstat: src/lib-fts/fts-tokenizer-generic.c | 9 +++------ src/lib-fts/test-fts-tokenizer.c | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diffs (34 lines): diff -r 6c655ce3b857 -r 5b902db0cabc src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Thu May 21 06:29:15 2015 -0400 +++ b/src/lib-fts/fts-tokenizer-generic.c Thu May 21 06:35:59 2015 -0400 @@ -234,15 +234,12 @@ fts_apostrophe_word_break(tok, c)) { len = char_start_i - start; tok_append_truncated(tok, data + start, len); - if (tok->token->used == 0) { - start = i + char_size; - continue; - } - - if (fts_tokenizer_generic_simple_current_token(tok, token_r)) { + if (tok->token->used > 0 && + fts_tokenizer_generic_simple_current_token(tok, token_r)) { *skip_r = i + char_size; return 1; } + start = i + char_size; } } /* word boundary not found yet */ diff -r 6c655ce3b857 -r 5b902db0cabc src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Thu May 21 06:29:15 2015 -0400 +++ b/src/lib-fts/test-fts-tokenizer.c Thu May 21 06:35:59 2015 -0400 @@ -29,7 +29,7 @@ "1.", - "'quoted text' 'word' 'hlo words' you're bad'''word '''pre post'''", + "' ' '' ''' 'quoted text' 'word' 'hlo words' you're bad'''word '''pre post'''", /* whitespace: with Unicode(utf8) U+FF01(ef bc 81)(U+2000(e2 80 80) and U+205A(e2 81 9a) and U+205F(e2 81 9f) */ From dovecot at dovecot.org Thu May 21 12:41:06 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 21 May 2015 12:41:06 +0000 Subject: dovecot-2.2: lib: Cork connection output while handling input Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/191eaf662c21 changeset: 18734:191eaf662c21 user: Timo Sirainen date: Thu May 21 08:38:56 2015 -0400 description: lib: Cork connection output while handling input diffstat: src/lib/connection.c | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-) diffs (35 lines): diff -r 5b902db0cabc -r 191eaf662c21 src/lib/connection.c --- a/src/lib/connection.c Thu May 21 06:35:59 2015 -0400 +++ b/src/lib/connection.c Thu May 21 08:38:56 2015 -0400 @@ -27,6 +27,7 @@ { const char *line; struct istream *input; + struct ostream *output; int ret = 0; switch (connection_input_read(conn)) { @@ -40,7 +41,12 @@ } input = conn->input; + output = conn->output; i_stream_ref(input); + if (output != NULL) { + o_stream_ref(output); + o_stream_cork(output); + } while (!input->closed && (line = i_stream_next_line(input)) != NULL) { T_BEGIN { ret = conn->list->v.input_line(conn, line); @@ -48,6 +54,10 @@ if (ret <= 0) break; } + if (output != NULL) { + o_stream_uncork(output); + o_stream_unref(&output); + } if (ret < 0 && !input->closed) { conn->disconnect_reason = CONNECTION_DISCONNECT_DEINIT; conn->list->v.destroy(conn); From pigeonhole at rename-it.nl Thu May 21 19:47:50 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 21 May 2015 21:47:50 +0200 Subject: dovecot-2.2-pigeonhole: managesieve-login: Implemented proxy XCL... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/ad7fc0736050 changeset: 2074:ad7fc0736050 user: Stephan Bosch date: Thu May 21 21:45:40 2015 +0200 description: managesieve-login: Implemented proxy XCLIENT support. diffstat: src/managesieve-login/client.c | 53 +++++++++++++++++++++++++++- src/managesieve-login/client.h | 1 + src/managesieve-login/managesieve-proxy.c | 57 ++++++++++++++++++++++++++++-- 3 files changed, 105 insertions(+), 6 deletions(-) diffs (198 lines): diff -r 9caff604a21e -r ad7fc0736050 src/managesieve-login/client.c --- a/src/managesieve-login/client.c Tue May 19 21:23:52 2015 +0200 +++ b/src/managesieve-login/client.c Thu May 21 21:45:40 2015 +0200 @@ -80,6 +80,10 @@ /* Protocol version */ client_send_raw(client, "\"VERSION\" \"1.0\"\r\n"); + + /* XCLIENT */ + if (client->trusted) + client_send_raw(client, "\"XCLIENT\"\r\n"); } T_END; } @@ -151,13 +155,58 @@ return 1; } +static int cmd_xclient +(struct managesieve_client *client, + const struct managesieve_arg *args) +{ + unsigned int remote_port; + const char *arg; + bool args_ok = TRUE; + + if ( !client->common.trusted ) { + client_send_no(&client->common, + "You are not from trusted IP"); + return 1; + } + while (!MANAGESIEVE_ARG_IS_EOL(&args[0]) && + managesieve_arg_get_atom(&args[0], &arg)) { + if ( strncasecmp(arg, "ADDR=", 5) == 0 ) { + if (net_addr2ip(arg + 5, &client->common.ip) < 0) + args_ok = FALSE; + } else if ( strncasecmp(arg, "PORT=", 5) == 0 ) { + if (str_to_uint(arg + 5, &remote_port) < 0 || + remote_port == 0 || remote_port > 65535) + args_ok = FALSE; + else + client->common.remote_port = remote_port; + } else if ( strncasecmp(arg, "SESSION=", 8) == 0 ) { + const char *value = arg + 8; + + if (strlen(value) <= LOGIN_MAX_SESSION_ID_LEN) { + client->common.session_id = + p_strdup(client->common.pool, value); + } + } else if ( strncasecmp(arg, "TTL=", 4 ) == 0) { + if (str_to_uint(arg + 4, &client->common.proxy_ttl) < 0) + args_ok = FALSE; + } + args++; + } + if ( !args_ok || !MANAGESIEVE_ARG_IS_EOL(&args[0])) + return -1; + + client_send_ok(&client->common, "Updated"); + return 1; +} + static struct managesieve_command commands[] = { { "AUTHENTICATE", cmd_authenticate, 1 }, { "CAPABILITY", cmd_capability, -1 }, { "STARTTLS", cmd_starttls, -1 }, { "NOOP", cmd_noop, 0 }, - { "LOGOUT", cmd_logout, -1}, - { NULL, NULL, 0} + { "LOGOUT", cmd_logout, -1 }, + { "XCLIENT", cmd_xclient, 0 }, + { NULL, NULL, 0 } }; static bool client_handle_input(struct managesieve_client *client) diff -r 9caff604a21e -r ad7fc0736050 src/managesieve-login/client.h --- a/src/managesieve-login/client.h Tue May 19 21:23:52 2015 +0200 +++ b/src/managesieve-login/client.h Thu May 21 21:45:40 2015 +0200 @@ -32,6 +32,7 @@ unsigned int proxy_starttls:1; unsigned int proxy_sasl_plain:1; + unsigned int proxy_xclient:1; }; bool client_skip_line(struct managesieve_client *client); diff -r 9caff604a21e -r ad7fc0736050 src/managesieve-login/managesieve-proxy.c --- a/src/managesieve-login/managesieve-proxy.c Tue May 19 21:23:52 2015 +0200 +++ b/src/managesieve-login/managesieve-proxy.c Thu May 21 21:45:40 2015 +0200 @@ -23,6 +23,7 @@ PROXY_STATE_INITIAL, PROXY_STATE_TLS_START, PROXY_STATE_TLS_READY, + PROXY_STATE_XCLIENT, PROXY_STATE_AUTHENTICATE, }; @@ -64,8 +65,21 @@ managesieve_quote_append_string(dest, str_c(base64), FALSE); } +static void proxy_write_xclient +(struct managesieve_client *client, string_t *str) +{ + str_printfa(str, + "XCLIENT ADDR=%s PORT=%u SESSION=%s TTL=%u\r\n", + net_ip2addr(&client->common.ip), + client->common.remote_port, + client_get_session_id(&client->common), + client->common.proxy_ttl - 1); +} + static int proxy_write_login(struct managesieve_client *client, string_t *str) { + i_assert(client->common.proxy_ttl > 1); + if ( !client->proxy_sasl_plain ) { client_log_err(&client->common, "proxy: " "Server does not support required PLAIN SASL mechanism"); @@ -77,7 +91,6 @@ get_plain_auth(&client->common, str); proxy_free_password(&client->common); str_append(str, "\r\n"); - return 1; } @@ -171,6 +184,8 @@ } else if ( strcasecmp(capability, "STARTTLS") == 0 ) { client->proxy_starttls = TRUE; + } else if ( strcasecmp(capability, "XCLIENT") == 0 ) { + client->proxy_xclient = TRUE; } } else { @@ -249,6 +264,11 @@ str_append(command, "STARTTLS\r\n"); msieve_client->proxy_state = PROXY_STATE_TLS_START; + + } else if (msieve_client->proxy_xclient) { + proxy_write_xclient(msieve_client, command); + msieve_client->proxy_state = PROXY_STATE_XCLIENT; + } else { if ( proxy_write_login(msieve_client, command) < 0 ) { client_proxy_failed(client, TRUE); @@ -296,17 +316,46 @@ } command = t_str_new(128); + if ( msieve_client->proxy_xclient ) { + proxy_write_xclient(msieve_client, command); + msieve_client->proxy_state = PROXY_STATE_XCLIENT; + + } else { + if ( proxy_write_login(msieve_client, command) < 0 ) { + client_proxy_failed(client, TRUE); + return -1; + } + msieve_client->proxy_state = PROXY_STATE_AUTHENTICATE; + } + (void)o_stream_send(output, str_data(command), str_len(command)); + } + return 0; + + case PROXY_STATE_XCLIENT: + if ( strncasecmp(line, "OK", 2) == 0 && + ( strlen(line) == 2 || line[2] == ' ' ) ) { + + /* STARTTLS successful, begin TLS negotiation. */ + if ( login_proxy_starttls(client->login_proxy) < 0 ) { + client_proxy_failed(client, TRUE); + return -1; + } + + command = t_str_new(128); if ( proxy_write_login(msieve_client, command) < 0 ) { client_proxy_failed(client, TRUE); return -1; } - (void)o_stream_send(output, str_data(command), str_len(command)); - msieve_client->proxy_state = PROXY_STATE_AUTHENTICATE; + return 0; } - return 0; + client_log_err(client, t_strdup_printf( + "proxy: Remote XCLIENT failed: %s", + str_sanitize(line, 160))); + client_proxy_failed(client, TRUE); + return -1; case PROXY_STATE_AUTHENTICATE: From pigeonhole at rename-it.nl Fri May 22 00:02:59 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 22 May 2015 02:02:59 +0200 Subject: dovecot-2.2-pigeonhole: managesieve-login: Copied too much in pr... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/d08c11c6f618 changeset: 2075:d08c11c6f618 user: Stephan Bosch date: Fri May 22 02:02:38 2015 +0200 description: managesieve-login: Copied too much in previous change. diffstat: src/managesieve-login/managesieve-proxy.c | 6 ------ 1 files changed, 0 insertions(+), 6 deletions(-) diffs (16 lines): diff -r ad7fc0736050 -r d08c11c6f618 src/managesieve-login/managesieve-proxy.c --- a/src/managesieve-login/managesieve-proxy.c Thu May 21 21:45:40 2015 +0200 +++ b/src/managesieve-login/managesieve-proxy.c Fri May 22 02:02:38 2015 +0200 @@ -335,12 +335,6 @@ if ( strncasecmp(line, "OK", 2) == 0 && ( strlen(line) == 2 || line[2] == ' ' ) ) { - /* STARTTLS successful, begin TLS negotiation. */ - if ( login_proxy_starttls(client->login_proxy) < 0 ) { - client_proxy_failed(client, TRUE); - return -1; - } - command = t_str_new(128); if ( proxy_write_login(msieve_client, command) < 0 ) { client_proxy_failed(client, TRUE); From pigeonhole at rename-it.nl Fri May 22 00:11:55 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 22 May 2015 02:11:55 +0200 Subject: dovecot-2.2-pigeonhole: managesieve-login: Changed proxy to bett... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/3df7e50f986d changeset: 2076:3df7e50f986d user: Stephan Bosch date: Fri May 22 02:11:03 2015 +0200 description: managesieve-login: Changed proxy to better match imap equivalent. It now uses dsasl-client, so it should now be possible to use SASL mechanisms other than PLAIN. Cleaned up the code a bit. diffstat: src/managesieve-login/managesieve-proxy.c | 253 ++++++++++++++++++++++------- 1 files changed, 188 insertions(+), 65 deletions(-) diffs (truncated from 424 to 300 lines): diff -r d08c11c6f618 -r 3df7e50f986d src/managesieve-login/managesieve-proxy.c --- a/src/managesieve-login/managesieve-proxy.c Fri May 22 02:02:38 2015 +0200 +++ b/src/managesieve-login/managesieve-proxy.c Fri May 22 02:11:03 2015 +0200 @@ -11,6 +11,7 @@ #include "safe-memset.h" #include "buffer.h" #include "base64.h" +#include "dsasl-client.h" #include "client.h" #include "client-authenticate.h" @@ -20,11 +21,11 @@ #include "managesieve-parser.h" enum { - PROXY_STATE_INITIAL, - PROXY_STATE_TLS_START, - PROXY_STATE_TLS_READY, - PROXY_STATE_XCLIENT, - PROXY_STATE_AUTHENTICATE, + MSIEVE_PROXY_STATE_NONE, + MSIEVE_PROXY_STATE_TLS_START, + MSIEVE_PROXY_STATE_TLS_READY, + MSIEVE_PROXY_STATE_XCLIENT, + MSIEVE_PROXY_STATE_AUTH, }; typedef enum { @@ -43,28 +44,6 @@ i_free_and_null(client->proxy_password); } -static void get_plain_auth(struct client *client, string_t *dest) -{ - string_t *str, *base64; - - str = t_str_new(128); - if ( client->proxy_master_user == NULL ) { - str_append_c(str, '\0'); - str_append(str, client->proxy_user); - } else { - str_append(str, client->proxy_user); - str_append_c(str, '\0'); - str_append(str, client->proxy_master_user); - } - str_append_c(str, '\0'); - str_append(str, client->proxy_password); - - base64 = t_str_new(128); - base64_encode(str_data(str), str_len(str), base64); - - managesieve_quote_append_string(dest, str_c(base64), FALSE); -} - static void proxy_write_xclient (struct managesieve_client *client, string_t *str) { @@ -76,8 +55,14 @@ client->common.proxy_ttl - 1); } -static int proxy_write_login(struct managesieve_client *client, string_t *str) +static int proxy_write_auth +(struct managesieve_client *client, string_t *str) { + struct dsasl_client_settings sasl_set; + const unsigned char *output; + unsigned int len; + const char *mech_name, *error; + i_assert(client->common.proxy_ttl > 1); if ( !client->proxy_sasl_plain ) { @@ -86,12 +71,134 @@ return -1; } - /* Send command */ - str_append(str, "AUTHENTICATE \"PLAIN\" "); - get_plain_auth(&client->common, str); + if (client->common.proxy_mech == NULL) + client->common.proxy_mech = &dsasl_client_mech_plain; + + i_assert(client->common.proxy_sasl_client == NULL); + memset(&sasl_set, 0, sizeof(sasl_set)); + sasl_set.authid = client->common.proxy_master_user != NULL ? + client->common.proxy_master_user : client->common.proxy_user; + sasl_set.authzid = client->common.proxy_user; + sasl_set.password = client->common.proxy_password; + client->common.proxy_sasl_client = + dsasl_client_new(client->common.proxy_mech, &sasl_set); + mech_name = dsasl_client_mech_get_name(client->common.proxy_mech); + + str_append(str, "AUTHENTICATE "); + managesieve_quote_append_string(str, mech_name, FALSE); + if (dsasl_client_output(client->common.proxy_sasl_client, + &output, &len, &error) < 0) { + client_log_err(&client->common, t_strdup_printf( + "proxy: SASL mechanism %s init failed: %s", + mech_name, error)); + return -1; + } + str_append_c(str, ' '); + if (len == 0) + str_append(str, "\"\""); + else { + string_t *resp = t_str_new(128); + base64_encode(output, len, resp); + managesieve_quote_append_string(str, str_c(resp), FALSE); + } + str_append(str, "\r\n"); proxy_free_password(&client->common); + return 0; +} + +static int proxy_input_auth_challenge +(struct managesieve_client *client, const char *line, + const char **challenge_r) +{ + struct istream *input; + struct managesieve_parser *parser; + const struct managesieve_arg *args; + const char *challenge; + bool fatal = FALSE; + int ret; + + i_assert(client->common.proxy_sasl_client != NULL); + *challenge_r = NULL; + + /* Build an input stream for the managesieve parser + * FIXME: Ugly, see proxy_input_capability(). + */ + line = t_strconcat(line, "\r\n", NULL); + input = i_stream_create_from_data(line, strlen(line)); + parser = managesieve_parser_create(input, MAX_MANAGESIEVE_LINE); + managesieve_parser_reset(parser); + + (void)i_stream_read(input); + ret = managesieve_parser_read_args(parser, 1, 0, &args); + + if ( ret >= 1 ) { + if ( managesieve_arg_get_string(&args[0], &challenge) ) { + *challenge_r = t_strdup(challenge); + } else { + client_log_err(&client->common, t_strdup_printf("proxy: " + "Server sent invalid SASL challenge line: %s", + str_sanitize(line,160))); + fatal = TRUE; + } + + } else if ( ret == -2 ) { + /* Parser needs more data (not possible on mem stream) */ + i_unreached(); + + } else if ( ret < 0 ) { + const char *error_str = managesieve_parser_get_error(parser, &fatal); + error_str = (error_str != NULL ? error_str : "unknown (bug)" ); + + /* Do not accept faulty server */ + client_log_err(&client->common, t_strdup_printf("proxy: " + "Protocol parse error(%d) int SASL challenge line: %s (line=`%s')", + ret, error_str, line)); + fatal = TRUE; + } + + + /* Cleanup parser */ + managesieve_parser_destroy(&parser); + i_stream_destroy(&input); + + /* Time to exit if greeting was not accepted */ + if ( fatal ) return -1; + return 0; +} + +static int proxy_write_auth_response +(struct managesieve_client *client, + const char *challenge, string_t *str) +{ + const unsigned char *data; + unsigned int data_len; + const char *error; + int ret; + + str = t_str_new(256); + if (base64_decode(challenge, strlen(challenge), NULL, str) < 0) { + client_log_err(&client->common, + "proxy: Server sent invalid base64 data in AUTHENTICATE response"); + return -1; + } + ret = dsasl_client_input(client->common.proxy_sasl_client, + str_data(str), str_len(str), &error); + if (ret == 0) { + ret = dsasl_client_output(client->common.proxy_sasl_client, + &data, &data_len, &error); + } + if (ret < 0) { + client_log_err(&client->common, t_strdup_printf( + "proxy: Server sent invalid authentication data: %s", + error)); + return -1; + } + i_assert(ret == 0); + + str_truncate(str, 0); + base64_encode(data, data_len, str); str_append(str, "\r\n"); - return 1; + return 0; } static managesieve_response_t proxy_read_response @@ -100,22 +207,19 @@ const char *response; if ( managesieve_arg_get_atom(&args[0], &response) ) { - if ( strcasecmp(response, "OK") == 0 ) { /* Received OK response; greeting is finished */ return MANAGESIEVE_RESPONSE_OK; - } else if ( strcasecmp(response, "NO") == 0 ) { + } else if ( strcasecmp(response, "NO") == 0 ) { /* Received OK response; greeting is finished */ return MANAGESIEVE_RESPONSE_NO; - } else if ( strcasecmp(response, "BYE") == 0 ) { + } else if ( strcasecmp(response, "BYE") == 0 ) { /* Received OK response; greeting is finished */ return MANAGESIEVE_RESPONSE_BYE; - } } - return MANAGESIEVE_RESPONSE_NONE; } @@ -200,19 +304,19 @@ /* Parser needs more data (not possible on mem stream) */ i_unreached(); - } else if ( ret < 0 ) { + } else if ( ret < 0 ) { const char *error_str = managesieve_parser_get_error(parser, &fatal); error_str = (error_str != NULL ? error_str : "unknown (bug)" ); /* Do not accept faulty server */ client_log_err(&client->common, t_strdup_printf("proxy: " - "Protocol parse error(%d) in capability/greeting line: %s (line='%s')", + "Protocol parse error(%d) in capability/greeting line: %s (line=`%s')", ret, error_str, line)); fatal = TRUE; } /* Cleanup parser */ - managesieve_parser_destroy(&parser); + managesieve_parser_destroy(&parser); i_stream_destroy(&input); /* Time to exit if greeting was not accepted */ @@ -238,8 +342,9 @@ output = login_proxy_get_ostream(client->login_proxy); switch ( msieve_client->proxy_state ) { - case PROXY_STATE_INITIAL: - if ( (ret=proxy_input_capability(msieve_client, line, &response)) < 0 ) { + case MSIEVE_PROXY_STATE_NONE: + if ( (ret=proxy_input_capability + (msieve_client, line, &response)) < 0 ) { client_proxy_failed(client, TRUE); return -1; } @@ -263,27 +368,27 @@ } str_append(command, "STARTTLS\r\n"); - msieve_client->proxy_state = PROXY_STATE_TLS_START; + msieve_client->proxy_state = MSIEVE_PROXY_STATE_TLS_START; } else if (msieve_client->proxy_xclient) { proxy_write_xclient(msieve_client, command); - msieve_client->proxy_state = PROXY_STATE_XCLIENT; + msieve_client->proxy_state = MSIEVE_PROXY_STATE_XCLIENT; } else { - if ( proxy_write_login(msieve_client, command) < 0 ) { + if ( proxy_write_auth(msieve_client, command) < 0 ) { client_proxy_failed(client, TRUE); return -1; } - msieve_client->proxy_state = PROXY_STATE_AUTHENTICATE; + msieve_client->proxy_state = MSIEVE_PROXY_STATE_AUTH; } (void)o_stream_send(output, str_data(command), str_len(command)); } - return 0; - case PROXY_STATE_TLS_START: From dovecot at dovecot.org Fri May 22 02:05:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 May 2015 02:05:24 +0000 Subject: dovecot-2.2: lib-fts: ICU normalization changes some characters ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5aae57dc5ad6 changeset: 18735:5aae57dc5ad6 user: Timo Sirainen date: Thu May 21 22:03:10 2015 -0400 description: lib-fts: ICU normalization changes some characters to spaces - remove them. We don't really want to add spaces to our index. It would be nice if the words between spaces were actually split to different tokens, but that's more of the fts-tokenizer's job and at filter stage that's probably not wanted anymore. diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 2 +- src/lib-fts/test-fts-filter.c | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diffs (39 lines): diff -r 191eaf662c21 -r 5aae57dc5ad6 src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Thu May 21 08:38:56 2015 -0400 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Thu May 21 22:03:10 2015 -0400 @@ -41,7 +41,7 @@ struct fts_filter_normalizer_icu *np; pool_t pp; unsigned int i; - const char *id = "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC"; + const char *id = "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC; [\\x20] Remove"; for (i = 0; settings[i] != NULL; i += 2) { const char *key = settings[i], *value = settings[i+1]; diff -r 191eaf662c21 -r 5aae57dc5ad6 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Thu May 21 08:38:56 2015 -0400 +++ b/src/lib-fts/test-fts-filter.c Thu May 21 22:03:10 2015 -0400 @@ -372,7 +372,7 @@ "vem", "a", "aao", - "vem kan segla forutan vind?\naaooaa" + "vemkanseglaforutanvind?\naaooaa" }; const char *error = NULL; const char *token = NULL; @@ -446,12 +446,13 @@ { /* test just a couple of these */ static const char *empty_tokens[] = { + "\xC2\xAF", /* U+00AF */ "\xCC\x80", /* U+0300 */ "\xF3\xA0\x87\xAF", /* U+E01EF */ "\xCC\x80\xF3\xA0\x87\xAF" /* U+0300 U+E01EF */ }; const char * const settings[] = - {"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove", NULL}; + {"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; [\\x20] Remove", NULL}; struct fts_filter *norm; const char *error; unsigned int i; From dovecot at dovecot.org Fri May 22 02:22:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 May 2015 02:22:22 +0000 Subject: dovecot-2.2: fts: Added fts_enforced setting to disable fallback... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b29b48376fb4 changeset: 18736:b29b48376fb4 user: Timo Sirainen date: Thu May 21 22:20:10 2015 -0400 description: fts: Added fts_enforced setting to disable fallbacking to slow search on errors. diffstat: src/plugins/fts/fts-storage.c | 17 +++++++++++++++-- src/plugins/fts/fts-storage.h | 1 + 2 files changed, 16 insertions(+), 2 deletions(-) diffs (59 lines): diff -r 5aae57dc5ad6 -r b29b48376fb4 src/plugins/fts/fts-storage.c --- a/src/plugins/fts/fts-storage.c Thu May 21 22:03:10 2015 -0400 +++ b/src/plugins/fts/fts-storage.c Thu May 21 22:20:10 2015 -0400 @@ -215,6 +215,9 @@ fctx->orig_matches = buffer_create_dynamic(default_pool, 64); fctx->virtual_mailbox = strcmp(t->box->storage->name, VIRTUAL_STORAGE_NAME) == 0; + fctx->enforced = + mail_user_plugin_getenv(t->box->storage->user, + "fts_enforced") != NULL; i_array_init(&fctx->levels, 8); fctx->scores = i_new(struct fts_scores, 1); fctx->scores->refcount = 1; @@ -234,7 +237,7 @@ ft->scores = fctx->scores; ft->scores->refcount++; - if (fts_want_build_args(args->args)) + if (fctx->enforced || fts_want_build_args(args->args)) fts_try_build_init(ctx, fctx); else fts_search_lookup(fctx); @@ -277,7 +280,11 @@ struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box); struct fts_search_context *fctx = FTS_CONTEXT(ctx); - if (fctx != NULL && fctx->indexer_ctx != NULL) { + if (fctx == NULL) { + /* no fts */ + } else if (!fctx->fts_lookup_success && fctx->enforced) { + return FALSE; + } else if (fctx->indexer_ctx != NULL) { /* this command is still building the indexes */ if (!fts_mailbox_build_continue(ctx)) { *tryagain_r = TRUE; @@ -362,6 +369,12 @@ } if (fctx->indexing_timed_out) ret = -1; + if (!fctx->fts_lookup_success && fctx->enforced) { + /* FTS lookup failed and we didn't want to fallback to + opening all the mails and searching manually */ + mail_storage_set_internal_error(ctx->transaction->box->storage); + ret = -1; + } buffer_free(&fctx->orig_matches); array_free(&fctx->levels); diff -r 5aae57dc5ad6 -r b29b48376fb4 src/plugins/fts/fts-storage.h --- a/src/plugins/fts/fts-storage.h Thu May 21 22:03:10 2015 -0400 +++ b/src/plugins/fts/fts-storage.h Thu May 21 22:20:10 2015 -0400 @@ -38,6 +38,7 @@ unsigned int virtual_mailbox:1; unsigned int fts_lookup_success:1; unsigned int indexing_timed_out:1; + unsigned int enforced:1; }; /* Figure out if we want to use full text search indexes and update From dovecot at dovecot.org Fri May 22 02:43:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 May 2015 02:43:55 +0000 Subject: dovecot-2.2: fts: Fixed fts_enforced=yes when it has to wait for... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8906101589f9 changeset: 18737:8906101589f9 user: Timo Sirainen date: Thu May 21 22:41:43 2015 -0400 description: fts: Fixed fts_enforced=yes when it has to wait for indexing to finish diffstat: src/plugins/fts/fts-storage.c | 8 +++----- 1 files changed, 3 insertions(+), 5 deletions(-) diffs (25 lines): diff -r b29b48376fb4 -r 8906101589f9 src/plugins/fts/fts-storage.c --- a/src/plugins/fts/fts-storage.c Thu May 21 22:20:10 2015 -0400 +++ b/src/plugins/fts/fts-storage.c Thu May 21 22:41:43 2015 -0400 @@ -280,11 +280,7 @@ struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box); struct fts_search_context *fctx = FTS_CONTEXT(ctx); - if (fctx == NULL) { - /* no fts */ - } else if (!fctx->fts_lookup_success && fctx->enforced) { - return FALSE; - } else if (fctx->indexer_ctx != NULL) { + if (fctx != NULL && fctx->indexer_ctx != NULL) { /* this command is still building the indexes */ if (!fts_mailbox_build_continue(ctx)) { *tryagain_r = TRUE; @@ -295,6 +291,8 @@ return FALSE; } } + if (fctx != NULL && !fctx->fts_lookup_success && fctx->enforced) + return FALSE; return fbox->module_ctx.super. search_next_nonblock(ctx, mail_r, tryagain_r); From pigeonhole at rename-it.nl Fri May 22 19:14:30 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 22 May 2015 21:14:30 +0200 Subject: dovecot-2.2-pigeonhole: managesieve-login: proxy: Fixed use of S... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/330e55055d61 changeset: 2077:330e55055d61 user: Stephan Bosch date: Fri May 22 21:14:21 2015 +0200 description: managesieve-login: proxy: Fixed use of SASL mechanisms with multiple challenge-response cycles. Previous change had several problems. diffstat: src/managesieve-login/managesieve-proxy.c | 26 +++++++++++++++++--------- 1 files changed, 17 insertions(+), 9 deletions(-) diffs (57 lines): diff -r 3df7e50f986d -r 330e55055d61 src/managesieve-login/managesieve-proxy.c --- a/src/managesieve-login/managesieve-proxy.c Fri May 22 02:11:03 2015 +0200 +++ b/src/managesieve-login/managesieve-proxy.c Fri May 22 21:14:21 2015 +0200 @@ -55,6 +55,19 @@ client->common.proxy_ttl - 1); } +static void proxy_write_auth_data +(const unsigned char *data, unsigned int data_len, + string_t *str) +{ + if (data_len == 0) + str_append(str, "\"\""); + else { + string_t *data_str = t_str_new(128); + base64_encode(data, data_len, data_str); + managesieve_quote_append_string(str, str_c(data_str), FALSE); + } +} + static int proxy_write_auth (struct managesieve_client *client, string_t *str) { @@ -93,13 +106,9 @@ mech_name, error)); return -1; } - str_append_c(str, ' '); - if (len == 0) - str_append(str, "\"\""); - else { - string_t *resp = t_str_new(128); - base64_encode(output, len, resp); - managesieve_quote_append_string(str, str_c(resp), FALSE); + if (len > 0) { + str_append_c(str, ' '); + proxy_write_auth_data(output, len, str); } str_append(str, "\r\n"); proxy_free_password(&client->common); @@ -175,7 +184,6 @@ const char *error; int ret; - str = t_str_new(256); if (base64_decode(challenge, strlen(challenge), NULL, str) < 0) { client_log_err(&client->common, "proxy: Server sent invalid base64 data in AUTHENTICATE response"); @@ -196,7 +204,7 @@ i_assert(ret == 0); str_truncate(str, 0); - base64_encode(data, data_len, str); + proxy_write_auth_data(data, data_len, str); str_append(str, "\r\n"); return 0; } From dovecot at dovecot.org Fri May 22 20:39:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 May 2015 20:39:19 +0000 Subject: dovecot-2.2: lib-index: Added asserts to make sure invalid exten... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fd8edab94849 changeset: 18738:fd8edab94849 user: Timo Sirainen date: Fri May 22 16:37:07 2015 -0400 description: lib-index: Added asserts to make sure invalid extension records aren't written to log. diffstat: src/lib-index/mail-index-transaction-export.c | 1 + src/lib-index/mail-index-transaction-update.c | 1 + 2 files changed, 2 insertions(+), 0 deletions(-) diffs (22 lines): diff -r 8906101589f9 -r fd8edab94849 src/lib-index/mail-index-transaction-export.c --- a/src/lib-index/mail-index-transaction-export.c Thu May 21 22:41:43 2015 -0400 +++ b/src/lib-index/mail-index-transaction-export.c Fri May 22 16:37:07 2015 -0400 @@ -164,6 +164,7 @@ intro->hdr_size = hdr_update_size; } } + i_assert(intro->record_size != 0 || intro->hdr_size != 0); if (reset_id != 0) { /* we're going to reset this extension in this transaction */ intro->reset_id = reset_id; diff -r 8906101589f9 -r fd8edab94849 src/lib-index/mail-index-transaction-update.c --- a/src/lib-index/mail-index-transaction-update.c Thu May 21 22:41:43 2015 -0400 +++ b/src/lib-index/mail-index-transaction-update.c Fri May 22 16:37:07 2015 -0400 @@ -979,6 +979,7 @@ rext = array_idx(&index->extensions, ext_id); record_size = rext->record_size; } + i_assert(record_size > 0); if (!array_is_created(&t->ext_rec_updates)) i_array_init(&t->ext_rec_updates, ext_id + 2); From pigeonhole at rename-it.nl Fri May 22 21:36:13 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 22 May 2015 23:36:13 +0200 Subject: dovecot-2.2-pigeonhole: Fixed duplicate const specifier warning ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/aca6b7806a47 changeset: 2078:aca6b7806a47 user: Stephan Bosch date: Fri May 22 23:36:03 2015 +0200 description: Fixed duplicate const specifier warning from Clang. diffstat: src/lib-sieve/plugins/relational/ext-relational-common.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 330e55055d61 -r aca6b7806a47 src/lib-sieve/plugins/relational/ext-relational-common.c --- a/src/lib-sieve/plugins/relational/ext-relational-common.c Fri May 22 21:14:21 2015 +0200 +++ b/src/lib-sieve/plugins/relational/ext-relational-common.c Fri May 22 23:36:03 2015 +0200 @@ -146,7 +146,7 @@ * Relational match-type operand */ -const const struct sieve_match_type_def *rel_match_types[] = { +const struct sieve_match_type_def *rel_match_types[] = { &rel_match_value_gt, &rel_match_value_ge, &rel_match_value_lt, &rel_match_value_le, &rel_match_value_eq, &rel_match_value_ne, &rel_match_count_gt, &rel_match_count_ge, &rel_match_count_lt, From pigeonhole at rename-it.nl Fri May 22 21:39:04 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 22 May 2015 23:39:04 +0200 Subject: dovecot-2.2-pigeonhole: sieve-test: Fixed Clang variable initial... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/add4db882327 changeset: 2079:add4db882327 user: Stephan Bosch date: Fri May 22 23:38:58 2015 +0200 description: sieve-test: Fixed Clang variable initialization warning. diffstat: src/sieve-tools/sieve-test.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r aca6b7806a47 -r add4db882327 src/sieve-tools/sieve-test.c --- a/src/sieve-tools/sieve-test.c Fri May 22 23:36:03 2015 +0200 +++ b/src/sieve-tools/sieve-test.c Fri May 22 23:38:58 2015 +0200 @@ -141,6 +141,7 @@ sieve_tool = sieve_tool_init ("sieve-test", &argc, &argv, "r:a:f:m:d:l:s:eCt:T:DP:x:u:", FALSE); + ehandler = action_ehandler = NULL; t_array_init(&scriptfiles, 16); /* Parse arguments */ From dovecot at dovecot.org Fri May 22 23:10:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 May 2015 23:10:16 +0000 Subject: dovecot-2.2: auth: Don't crash if trying to add password with TA... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b23a19faf304 changeset: 18739:b23a19faf304 user: Timo Sirainen date: Fri May 22 19:07:56 2015 -0400 description: auth: Don't crash if trying to add password with TAB or LF to auth cache. This would happen only if the passwords were stored as plaintext in passdb and the valid password actually contained TAB or LF. diffstat: src/auth/auth-request.c | 6 +----- 1 files changed, 1 insertions(+), 5 deletions(-) diffs (16 lines): diff -r fd8edab94849 -r b23a19faf304 src/auth/auth-request.c --- a/src/auth/auth-request.c Fri May 22 16:37:07 2015 -0400 +++ b/src/auth/auth-request.c Fri May 22 19:07:56 2015 -0400 @@ -457,11 +457,7 @@ str_append(str, passdb->default_pass_scheme); str_append_c(str, '}'); } - if (strchr(request->passdb_password, '\t') != NULL) - i_panic("%s: Password contains TAB", request->user); - if (strchr(request->passdb_password, '\n') != NULL) - i_panic("%s: Password contains LF", request->user); - str_append(str, request->passdb_password); + str_append_tabescaped(str, request->passdb_password); } if (!auth_fields_is_empty(request->extra_fields)) { From dovecot at dovecot.org Sat May 23 20:43:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 23 May 2015 20:43:59 +0000 Subject: dovecot-2.2: lib: Added file_create_locked() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/78bbfe4e4e8e changeset: 18740:78bbfe4e4e8e user: Timo Sirainen date: Sat May 23 16:41:34 2015 -0400 description: lib: Added file_create_locked() diffstat: src/lib/Makefile.am | 2 + src/lib/file-create-locked.c | 126 +++++++++++++++++++++++++++++++++++++++++++ src/lib/file-create-locked.h | 29 +++++++++ 3 files changed, 157 insertions(+), 0 deletions(-) diffs (182 lines): diff -r b23a19faf304 -r 78bbfe4e4e8e src/lib/Makefile.am --- a/src/lib/Makefile.am Fri May 22 19:07:56 2015 -0400 +++ b/src/lib/Makefile.am Sat May 23 16:41:34 2015 -0400 @@ -35,6 +35,7 @@ fdatasync-path.c \ fdpass.c \ file-cache.c \ + file-create-locked.c \ file-copy.c \ file-dotlock.c \ file-lock.c \ @@ -171,6 +172,7 @@ fdatasync-path.h \ fdpass.h \ file-cache.h \ + file-create-locked.h \ file-copy.h \ file-dotlock.h \ file-lock.h \ diff -r b23a19faf304 -r 78bbfe4e4e8e src/lib/file-create-locked.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/file-create-locked.c Sat May 23 16:41:34 2015 -0400 @@ -0,0 +1,126 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "safe-mkstemp.h" +#include "file-lock.h" +#include "file-create-locked.h" + +#include +#include +#include + +#define MAX_RETRY_COUNT 1000 + +static int +try_lock_existing(int fd, const char *path, + const struct file_create_settings *set, + struct file_lock **lock_r, const char **error_r) +{ + struct stat st1, st2; + + if (fstat(fd, &st1) < 0) { + *error_r = t_strdup_printf("fstat(%s) failed: %m", path); + return -1; + } + if (file_wait_lock_error(fd, path, F_WRLCK, FILE_LOCK_METHOD_FCNTL, + set->lock_timeout_secs, lock_r, error_r) <= 0) + return -1; + if (stat(path, &st2) == 0) { + return st1.st_ino == st2.st_ino && + CMP_DEV_T(st1.st_dev, st2.st_dev) ? 1 : 0; + } else if (errno == ENOENT) { + return 0; + } else { + *error_r = t_strdup_printf("stat(%s) failed: %m", path); + return -1; + } +} + +static int +try_create_new(const char *path, const struct file_create_settings *set, + int *fd_r, struct file_lock **lock_r, const char **error_r) +{ + string_t *temp_path = t_str_new(128); + int fd, orig_errno, ret = -1; + int mode = set->mode != 0 ? set->mode : 0600; + uid_t uid = set->uid != 0 ? set->uid : (uid_t)-1; + uid_t gid = set->gid != 0 ? set->gid : (gid_t)-1; + + str_append(temp_path, path); + if (uid != (uid_t)-1) + fd = safe_mkstemp(temp_path, mode, uid, gid); + else + fd = safe_mkstemp_group(temp_path, mode, gid, set->gid_origin); + if (fd == -1) { + *error_r = t_strdup_printf("safe_mkstemp(%s) failed: %m", path); + return -1; + } + if (file_try_lock_error(fd, str_c(temp_path), F_WRLCK, + FILE_LOCK_METHOD_FCNTL, + lock_r, error_r) <= 0) { + } else if (link(str_c(temp_path), path) < 0) { + if (errno == EEXIST) { + /* just created by somebody else */ + ret = 0; + } else if (errno == ENOENT) { + /* our temp file was just deleted by somebody else, + retry creating it. */ + ret = 0; + } else { + *error_r = t_strdup_printf("link(%s, %s) failed: %m", + str_c(temp_path), path); + } + } else { + *fd_r = fd; + return 1; + } + orig_errno = errno; + i_close_fd(&fd); + if (unlink(str_c(temp_path)) < 0) + i_error("unlink(%s) failed: %m", str_c(temp_path)); + errno = orig_errno; + return ret; +} + +int file_create_locked(const char *path, const struct file_create_settings *set, + struct file_lock **lock_r, bool *created_r, + const char **error_r) +{ + unsigned int i; + int fd, ret; + + for (i = 0; i < MAX_RETRY_COUNT; i++) { + fd = open(path, O_RDWR); + if (fd != -1) { + ret = try_lock_existing(fd, path, set, lock_r, error_r); + if (ret < 0) { + i_close_fd(&fd); + return -1; + } + if (ret > 0) { + /* successfully locked an existing file */ + *created_r = FALSE; + return fd; + } + } else if (errno != ENOENT) { + *error_r = t_strdup_printf("open(%s) failed: %m", path); + return -1; + } else { + /* try to create the file */ + ret = try_create_new(path, set, &fd, lock_r, error_r); + if (ret < 0) + return -1; + if (ret > 0) { + /* successfully created a new locked file */ + *created_r = TRUE; + return fd; + } + /* the file was just created - try again opening and + locking it */ + } + } + *error_r = t_strdup_printf("Creating a locked file %s keeps failing", path); + errno = EINVAL; + return -1; +} diff -r b23a19faf304 -r 78bbfe4e4e8e src/lib/file-create-locked.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/file-create-locked.h Sat May 23 16:41:34 2015 -0400 @@ -0,0 +1,29 @@ +#ifndef FILE_CREATE_LOCKED_H +#define FILE_CREATE_LOCKED_H + +struct file_lock; + +struct file_create_settings { + /* 0 = try locking without waiting */ + unsigned int lock_timeout_secs; + + /* 0 = 0600 */ + int mode; + /* 0 = default */ + uid_t uid; + /* 0 = default */ + gid_t gid; + const char *gid_origin; +}; + +/* Either open an existing file and lock it, or create the file locked. + The creation is done by creating a temp file and link()ing it to path. + If link() fails, opening is retried again. Returns fd on success, + -1 on error. errno is preserved for the last failed syscall, so most + importantly ENOENT could mean that the directory doesn't exist and EAGAIN + means locking timed out. */ +int file_create_locked(const char *path, const struct file_create_settings *set, + struct file_lock **lock_r, bool *created_r, + const char **error_r); + +#endif From dovecot at dovecot.org Sat May 23 20:54:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 23 May 2015 20:54:09 +0000 Subject: dovecot-2.2: lib: file_create_locked() wasn't deleting the temp ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/350f11821be2 changeset: 18741:350f11821be2 user: Timo Sirainen date: Sat May 23 16:51:52 2015 -0400 description: lib: file_create_locked() wasn't deleting the temp file after successful link() diffstat: src/lib/file-create-locked.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 78bbfe4e4e8e -r 350f11821be2 src/lib/file-create-locked.c --- a/src/lib/file-create-locked.c Sat May 23 16:41:34 2015 -0400 +++ b/src/lib/file-create-locked.c Sat May 23 16:51:52 2015 -0400 @@ -72,6 +72,8 @@ str_c(temp_path), path); } } else { + if (unlink(str_c(temp_path)) < 0) + i_error("unlink(%s) failed: %m", str_c(temp_path)); *fd_r = fd; return 1; } From dovecot at dovecot.org Sat May 23 23:23:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 23 May 2015 23:23:25 +0000 Subject: dovecot-2.2: lib: file_create_locked() no longer logs an error i... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f98aad82ddda changeset: 18742:f98aad82ddda user: Timo Sirainen date: Sat May 23 19:21:10 2015 -0400 description: lib: file_create_locked() no longer logs an error if temp file is deleted under it. diffstat: src/lib/file-create-locked.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (20 lines): diff -r 350f11821be2 -r f98aad82ddda src/lib/file-create-locked.c --- a/src/lib/file-create-locked.c Sat May 23 16:51:52 2015 -0400 +++ b/src/lib/file-create-locked.c Sat May 23 19:21:10 2015 -0400 @@ -72,14 +72,14 @@ str_c(temp_path), path); } } else { - if (unlink(str_c(temp_path)) < 0) + if (unlink(str_c(temp_path)) < 0 && errno != ENOENT) i_error("unlink(%s) failed: %m", str_c(temp_path)); *fd_r = fd; return 1; } orig_errno = errno; i_close_fd(&fd); - if (unlink(str_c(temp_path)) < 0) + if (unlink(str_c(temp_path)) < 0 && errno != ENOENT) i_error("unlink(%s) failed: %m", str_c(temp_path)); errno = orig_errno; return ret; From dovecot at dovecot.org Sun May 24 17:26:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 May 2015 17:26:35 +0000 Subject: dovecot-2.2: lib: file_create_locked() was leaking fds and locks. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1d088dc567bd changeset: 18743:1d088dc567bd user: Timo Sirainen date: Sun May 24 13:22:56 2015 -0400 description: lib: file_create_locked() was leaking fds and locks. diffstat: src/lib/file-create-locked.c | 19 ++++++++++++------- 1 files changed, 12 insertions(+), 7 deletions(-) diffs (53 lines): diff -r f98aad82ddda -r 1d088dc567bd src/lib/file-create-locked.c --- a/src/lib/file-create-locked.c Sat May 23 19:21:10 2015 -0400 +++ b/src/lib/file-create-locked.c Sun May 24 13:22:56 2015 -0400 @@ -18,6 +18,7 @@ struct file_lock **lock_r, const char **error_r) { struct stat st1, st2; + int ret; if (fstat(fd, &st1) < 0) { *error_r = t_strdup_printf("fstat(%s) failed: %m", path); @@ -27,14 +28,19 @@ set->lock_timeout_secs, lock_r, error_r) <= 0) return -1; if (stat(path, &st2) == 0) { - return st1.st_ino == st2.st_ino && + ret = st1.st_ino == st2.st_ino && CMP_DEV_T(st1.st_dev, st2.st_dev) ? 1 : 0; } else if (errno == ENOENT) { - return 0; + ret = 0; } else { *error_r = t_strdup_printf("stat(%s) failed: %m", path); - return -1; + ret = -1; } + if (ret <= 0) { + /* the fd is closed next - no need to unlock */ + file_lock_free(lock_r); + } + return ret; } static int @@ -96,15 +102,14 @@ fd = open(path, O_RDWR); if (fd != -1) { ret = try_lock_existing(fd, path, set, lock_r, error_r); - if (ret < 0) { - i_close_fd(&fd); - return -1; - } if (ret > 0) { /* successfully locked an existing file */ *created_r = FALSE; return fd; } + i_close_fd(&fd); + if (ret < 0) + return -1; } else if (errno != ENOENT) { *error_r = t_strdup_printf("open(%s) failed: %m", path); return -1; From dovecot at dovecot.org Sun May 24 17:26:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 May 2015 17:26:36 +0000 Subject: dovecot-2.2: lib: file_create_locked() lock method is now config... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/57f8c2a6209b changeset: 18744:57f8c2a6209b user: Timo Sirainen date: Sun May 24 13:24:19 2015 -0400 description: lib: file_create_locked() lock method is now configurable diffstat: src/lib/file-create-locked.c | 5 ++--- src/lib/file-create-locked.h | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diffs (40 lines): diff -r 1d088dc567bd -r 57f8c2a6209b src/lib/file-create-locked.c --- a/src/lib/file-create-locked.c Sun May 24 13:22:56 2015 -0400 +++ b/src/lib/file-create-locked.c Sun May 24 13:24:19 2015 -0400 @@ -24,7 +24,7 @@ *error_r = t_strdup_printf("fstat(%s) failed: %m", path); return -1; } - if (file_wait_lock_error(fd, path, F_WRLCK, FILE_LOCK_METHOD_FCNTL, + if (file_wait_lock_error(fd, path, F_WRLCK, set->lock_method, set->lock_timeout_secs, lock_r, error_r) <= 0) return -1; if (stat(path, &st2) == 0) { @@ -63,8 +63,7 @@ return -1; } if (file_try_lock_error(fd, str_c(temp_path), F_WRLCK, - FILE_LOCK_METHOD_FCNTL, - lock_r, error_r) <= 0) { + set->lock_method, lock_r, error_r) <= 0) { } else if (link(str_c(temp_path), path) < 0) { if (errno == EEXIST) { /* just created by somebody else */ diff -r 1d088dc567bd -r 57f8c2a6209b src/lib/file-create-locked.h --- a/src/lib/file-create-locked.h Sun May 24 13:22:56 2015 -0400 +++ b/src/lib/file-create-locked.h Sun May 24 13:24:19 2015 -0400 @@ -1,12 +1,13 @@ #ifndef FILE_CREATE_LOCKED_H #define FILE_CREATE_LOCKED_H -struct file_lock; +#include "file-lock.h" struct file_create_settings { /* 0 = try locking without waiting */ unsigned int lock_timeout_secs; + enum file_lock_method lock_method; /* 0 = 0600 */ int mode; /* 0 = default */ From dovecot at dovecot.org Sun May 24 17:42:58 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 May 2015 17:42:58 +0000 Subject: dovecot-2.2: lib: file_lock_try_update() comment updated. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c792d026d3fa changeset: 18745:c792d026d3fa user: Timo Sirainen date: Sun May 24 13:40:46 2015 -0400 description: lib: file_lock_try_update() comment updated. It should possibly be removed entirely from the API. diffstat: src/lib/file-lock.h | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 57f8c2a6209b -r c792d026d3fa src/lib/file-lock.h --- a/src/lib/file-lock.h Sun May 24 13:24:19 2015 -0400 +++ b/src/lib/file-lock.h Sun May 24 13:40:46 2015 -0400 @@ -42,7 +42,8 @@ enum file_lock_method lock_method, unsigned int timeout_secs, struct file_lock **lock_r, const char **error_r); -/* Change the lock type. */ +/* Change the lock type. WARNING: This isn't an atomic operation! + The result is the same as file_unlock() + file_try_lock(). */ int file_lock_try_update(struct file_lock *lock, int lock_type); /* Unlock and free the lock. */ From dovecot at dovecot.org Sun May 24 20:16:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 May 2015 20:16:11 +0000 Subject: dovecot-2.2: lib-index: Unit test fix. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2de3c7248922 changeset: 18746:2de3c7248922 user: Timo Sirainen date: Sun May 24 16:13:57 2015 -0400 description: lib-index: Unit test fix. diffstat: src/lib-index/test-mail-index-transaction-update.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r c792d026d3fa -r 2de3c7248922 src/lib-index/test-mail-index-transaction-update.c --- a/src/lib-index/test-mail-index-transaction-update.c Sun May 24 13:40:46 2015 -0400 +++ b/src/lib-index/test-mail-index-transaction-update.c Sun May 24 16:13:57 2015 -0400 @@ -466,7 +466,7 @@ count = array_count(&t->updates); for (i = 0; i < idx; i++) { for (j = idx + 1; j <= count; j++) { - if (!mail_index_transaction_get_flag_update_pos(t, i, j, seq) == idx) { + if (mail_index_transaction_get_flag_update_pos(t, i, j, seq) != idx) { test_assert(FALSE); return FALSE; } From dovecot at dovecot.org Sun May 24 21:43:04 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 May 2015 21:43:04 +0000 Subject: dovecot-2.2: lib-storage: If session_id isn't given, generate a ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5f95cc2bd7aa changeset: 18747:5f95cc2bd7aa user: Timo Sirainen date: Sun May 24 17:40:53 2015 -0400 description: lib-storage: If session_id isn't given, generate a new one. This is useful for tracking logs written by services that aren't directly related to any specific user session. diffstat: src/lib-storage/mail-storage-service.c | 19 +++++++++++++++++++ 1 files changed, 19 insertions(+), 0 deletions(-) diffs (43 lines): diff -r 2de3c7248922 -r 5f95cc2bd7aa src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Sun May 24 16:13:57 2015 -0400 +++ b/src/lib-storage/mail-storage-service.c Sun May 24 17:40:53 2015 -0400 @@ -3,6 +3,7 @@ #include "lib.h" #include "ioloop.h" #include "array.h" +#include "base64.h" #include "hostpid.h" #include "module-dir.h" #include "restrict-access.h" @@ -1024,6 +1025,20 @@ i_set_failure_prefix("%s", str_c(str)); } +static const char *mail_storage_service_generate_session_id(pool_t pool) +{ + guid_128_t guid; + string_t *str = str_new(pool, MAX_BASE64_ENCODED_SIZE(sizeof(guid))); + + guid_128_generate(guid); + base64_encode(guid, sizeof(guid), str); + /* remove the trailing "==" */ + i_assert(str_data(str)[str_len(str)-2] == '='); + str_truncate(str, str_len(str)-2); + return str_c(str); + +} + static int mail_storage_service_lookup_real(struct mail_storage_service_ctx *ctx, const struct mail_storage_service_input *input, @@ -1123,6 +1138,10 @@ user->input.userdb_fields = NULL; user->input.username = p_strdup(user_pool, username); user->input.session_id = p_strdup(user_pool, input->session_id); + if (user->input.session_id == NULL) { + user->input.session_id = + mail_storage_service_generate_session_id(user_pool); + } user->user_info = user_info; user->flags = flags; From dovecot at dovecot.org Sun May 24 21:52:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 May 2015 21:52:46 +0000 Subject: dovecot-2.2: lib-storage: If no namespaces were defined, the aut... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/00ef92771cd9 changeset: 18748:00ef92771cd9 user: Timo Sirainen date: Sun May 24 17:50:26 2015 -0400 description: lib-storage: If no namespaces were defined, the autocreated namespace settings were a bit wrong. diffstat: src/lib-storage/mail-namespace.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 5f95cc2bd7aa -r 00ef92771cd9 src/lib-storage/mail-namespace.c --- a/src/lib-storage/mail-namespace.c Sun May 24 17:40:53 2015 -0400 +++ b/src/lib-storage/mail-namespace.c Sun May 24 17:50:26 2015 -0400 @@ -429,6 +429,9 @@ inbox_set = p_new(user->pool, struct mail_namespace_settings, 1); *inbox_set = mail_namespace_default_settings; inbox_set->inbox = TRUE; + /* enums must be changed */ + inbox_set->type = "private"; + inbox_set->list = "yes"; unexpanded_inbox_set = p_new(user->pool, struct mail_namespace_settings, 1); *unexpanded_inbox_set = *inbox_set; From dovecot at dovecot.org Sun May 24 21:57:47 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 May 2015 21:57:47 +0000 Subject: dovecot-2.2: doveadm user: Don't change privileges when looking ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6fcaaaf13171 changeset: 18749:6fcaaaf13171 user: Timo Sirainen date: Sun May 24 17:55:33 2015 -0400 description: doveadm user: Don't change privileges when looking up users. We don't need to and it might fail. diffstat: src/doveadm/doveadm-auth.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 00ef92771cd9 -r 6fcaaaf13171 src/doveadm/doveadm-auth.c --- a/src/doveadm/doveadm-auth.c Sun May 24 17:50:26 2015 -0400 +++ b/src/doveadm/doveadm-auth.c Sun May 24 17:55:33 2015 -0400 @@ -483,7 +483,7 @@ MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT | MAIL_STORAGE_SERVICE_FLAG_NO_PLUGINS | MAIL_STORAGE_SERVICE_FLAG_NO_NAMESPACES | - MAIL_STORAGE_SERVICE_FLAG_TEMP_PRIV_DROP); + MAIL_STORAGE_SERVICE_FLAG_NO_RESTRICT_ACCESS); mail_storage_service_set_auth_conn(storage_service, conn); conn = NULL; if (show_field == NULL) { From dovecot at dovecot.org Mon May 25 00:30:47 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 25 May 2015 00:30:47 +0000 Subject: dovecot-2.2: systemd: Small improvements to the unit configurati... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8dc79a437858 changeset: 18750:8dc79a437858 user: Pascal Volk date: Sun May 17 11:54:42 2015 +0000 description: systemd: Small improvements to the unit configuration file: * Added Documentation URIs * Added ExecReload, to reload the config instead of restarting * Added a comment for modifying/extending the unit file. diffstat: dovecot.service.in | 23 ++++++++++++++++++++++- 1 files changed, 22 insertions(+), 1 deletions(-) diffs (36 lines): diff -r 6fcaaaf13171 -r 8dc79a437858 dovecot.service.in --- a/dovecot.service.in Sun May 24 17:55:33 2015 -0400 +++ b/dovecot.service.in Sun May 17 11:54:42 2015 +0000 @@ -1,10 +1,31 @@ +# This file is part of Dovecot +# +# If you want to pass additionally command line options to the dovecot +# binary, create the file: +# `/etc/systemd/system/dovecot.service.d/service.conf'. +# In this file create a Service section and configure an Environment with +# the variable `OPTIONS'. For example: +# +# [Service] +# Environment='OPTIONS=-p' +# +# In the `Service' section you may also specify various other setting. +# If you have trouble with `Too many open files' you may set: +#LimitCORE=8192 +# +# If you want to allow the Dovecot services to produce core dumps, use: +#LimitCORE='infinity' + [Unit] Description=Dovecot IMAP/POP3 email server +Documentation=man:dovecot(1) +Documentation=http://wiki2.dovecot.org/ After=local-fs.target network.target [Service] Type=simple -ExecStart=@sbindir@/dovecot -F +ExecStart=@sbindir@/dovecot -F $OPTIONS +ExecReload=@sbindir@/dovecot reload NonBlocking=yes [Install] From dovecot at dovecot.org Mon May 25 14:38:42 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 25 May 2015 14:38:42 +0000 Subject: dovecot-2.2: systemd service: Fixed typos in the comment section. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b088511e1cdb changeset: 18751:b088511e1cdb user: Pascal Volk date: Mon May 25 14:27:22 2015 +0000 description: systemd service: Fixed typos in the comment section. The settings for the file descriptor limit is LimitNOFILE. Removed quotes around the value infinity. Otherwise systemd will fail to parse that resource value. diffstat: dovecot.service.in | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 8dc79a437858 -r b088511e1cdb dovecot.service.in --- a/dovecot.service.in Sun May 17 11:54:42 2015 +0000 +++ b/dovecot.service.in Mon May 25 14:27:22 2015 +0000 @@ -11,10 +11,10 @@ # # In the `Service' section you may also specify various other setting. # If you have trouble with `Too many open files' you may set: -#LimitCORE=8192 +#LimitNOFILE=8192 # # If you want to allow the Dovecot services to produce core dumps, use: -#LimitCORE='infinity' +#LimitCORE=infinity [Unit] Description=Dovecot IMAP/POP3 email server From dovecot at dovecot.org Mon May 25 15:38:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 25 May 2015 15:38:07 +0000 Subject: dovecot-2.2: log: read() errors weren't logged. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fdc03f6cc45d changeset: 18752:fdc03f6cc45d user: Timo Sirainen date: Mon May 25 11:35:53 2015 -0400 description: log: read() errors weren't logged. diffstat: src/log/log-connection.c | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-) diffs (17 lines): diff -r b088511e1cdb -r fdc03f6cc45d src/log/log-connection.c --- a/src/log/log-connection.c Mon May 25 14:27:22 2015 +0000 +++ b/src/log/log-connection.c Mon May 25 11:35:53 2015 -0400 @@ -351,10 +351,9 @@ } } - if (log->input->eof) - log_connection_destroy(log); - else if (log->input->stream_errno != 0) { - i_error("read(log %s) failed: %m", log->default_prefix); + if (log->input->eof) { + if (log->input->stream_errno != 0) + i_error("read(log %s) failed: %m", log->default_prefix); log_connection_destroy(log); } else { i_assert(!log->input->closed); From dovecot at dovecot.org Mon May 25 15:53:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 25 May 2015 15:53:09 +0000 Subject: dovecot-2.2: lib: Avoid race conditions in mkdir*() if directory... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dd2015e1362f changeset: 18753:dd2015e1362f user: Timo Sirainen date: Mon May 25 11:50:48 2015 -0400 description: lib: Avoid race conditions in mkdir*() if directory is being deleted at the same time. Mainly this allows the call to return failure silently without logging unnecessary errors. diffstat: src/lib/mkdir-parents.c | 40 +++++++++++++++++++++++++++++----------- 1 files changed, 29 insertions(+), 11 deletions(-) diffs (90 lines): diff -r fdc03f6cc45d -r dd2015e1362f src/lib/mkdir-parents.c --- a/src/lib/mkdir-parents.c Mon May 25 11:35:53 2015 -0400 +++ b/src/lib/mkdir-parents.c Mon May 25 11:50:48 2015 -0400 @@ -7,6 +7,7 @@ #include "ipwd.h" #include +#include #include static int ATTR_NULL(5) @@ -15,11 +16,24 @@ { string_t *str; mode_t old_mask; - int ret, orig_errno; + unsigned int i; + int ret, fd = -1, orig_errno; - old_mask = umask(0); - ret = mkdir(path, mode); - umask(old_mask); + for (i = 0;; i++) { + old_mask = umask(0); + ret = mkdir(path, mode); + umask(old_mask); + if (ret < 0) + break; + fd = open(path, O_RDONLY); + if (fd != -1) + break; + if (errno != ENOENT || i == 3) { + i_error("open(%s) failed: %m", path); + return -1; + } + /* it was just rmdir()ed by someone else? retry */ + } if (ret < 0) { if (errno == EISDIR || errno == ENOSYS) { @@ -29,22 +43,24 @@ ENOSYS check is for NFS mount points. */ errno = EEXIST; } + i_assert(fd == -1); return -1; } - if (chown(path, uid, gid) < 0) { + if (fchown(fd, uid, gid) < 0) { + i_close_fd(&fd); orig_errno = errno; - if (rmdir(path) < 0) + if (rmdir(path) < 0 && errno != ENOENT) i_error("rmdir(%s) failed: %m", path); errno = orig_errno; if (errno == EPERM && uid == (uid_t)-1) { - i_error("%s", eperm_error_get_chgrp("chown", path, gid, + i_error("%s", eperm_error_get_chgrp("fchown", path, gid, gid_origin)); return -1; } str = t_str_new(256); - str_printfa(str, "chown(%s, %ld", path, + str_printfa(str, "fchown(%s, %ld", path, uid == (uid_t)-1 ? -1L : (long)uid); if (uid != (uid_t)-1) { struct passwd pw; @@ -68,15 +84,17 @@ if (gid != (gid_t)-1 && (mode & S_ISGID) == 0) { /* make sure the directory doesn't have setgid bit enabled (in case its parent had) */ - if (chmod(path, mode) < 0) { + if (fchmod(fd, mode) < 0) { orig_errno = errno; - if (rmdir(path) < 0) + if (rmdir(path) < 0 && errno != ENOENT) i_error("rmdir(%s) failed: %m", path); errno = orig_errno; - i_error("chmod(%s) failed: %m", path); + i_error("fchmod(%s) failed: %m", path); + i_close_fd(&fd); return -1; } } + i_close_fd(&fd); return 0; } From dovecot at dovecot.org Fri May 29 02:45:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 29 May 2015 02:45:40 +0000 Subject: dovecot-2.2: fts: If last-uid lookup fails, return "Internal err... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/492d638c52c6 changeset: 18754:492d638c52c6 user: Timo Sirainen date: Fri May 29 05:43:25 2015 +0300 description: fts: If last-uid lookup fails, return "Internal error" instead of "BUG" to client. diffstat: src/plugins/fts/fts-storage.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (15 lines): diff -r dd2015e1362f -r 492d638c52c6 src/plugins/fts/fts-storage.c --- a/src/plugins/fts/fts-storage.c Mon May 25 11:50:48 2015 -0400 +++ b/src/plugins/fts/fts-storage.c Fri May 29 05:43:25 2015 +0300 @@ -74,8 +74,10 @@ struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list); uint32_t seq1, seq2, last_uid; - if (fts_backend_get_last_uid(flist->backend, box, &last_uid) < 0) + if (fts_backend_get_last_uid(flist->backend, box, &last_uid) < 0) { + mail_storage_set_internal_error(box->storage); return -1; + } if (last_uid == 0) *seq_r = 0; From dovecot at dovecot.org Fri May 29 04:05:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 29 May 2015 04:05:33 +0000 Subject: dovecot-2.2: lib-fs: fs_write_stream() now automatically corks t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0c55e484914c changeset: 18755:0c55e484914c user: Timo Sirainen date: Fri May 29 07:03:15 2015 +0300 description: lib-fs: fs_write_stream() now automatically corks the ostream and uncorks at _finish(). diffstat: src/lib-fs/fs-api.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (20 lines): diff -r 492d638c52c6 -r 0c55e484914c src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Fri May 29 05:43:25 2015 +0300 +++ b/src/lib-fs/fs-api.c Fri May 29 07:03:15 2015 +0300 @@ -549,6 +549,7 @@ file->fs->v.write_stream(file); } T_END; i_assert(file->output != NULL); + o_stream_cork(file->output); return file->output; } @@ -559,6 +560,8 @@ i_assert(*output == file->output || *output == NULL); *output = NULL; + if (file->output != NULL) + o_stream_uncork(file->output); T_BEGIN { ret = file->fs->v.write_stream_finish(file, TRUE); } T_END; From dovecot at dovecot.org Fri May 29 07:45:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 29 May 2015 07:45:55 +0000 Subject: dovecot-2.2: dovecot.m4: If NOVALGRIND environment is set, don't... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2dd8308f5d57 changeset: 18756:2dd8308f5d57 user: Timo Sirainen date: Fri May 29 10:43:38 2015 +0300 description: dovecot.m4: If NOVALGRIND environment is set, don't run tests via valgrind in "make check" diffstat: dovecot.m4 | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diffs (23 lines): diff -r 0c55e484914c -r 2dd8308f5d57 dovecot.m4 --- a/dovecot.m4 Fri May 29 07:03:15 2015 +0300 +++ b/dovecot.m4 Fri May 29 10:43:38 2015 +0300 @@ -6,7 +6,7 @@ # unlimited permission to copy and/or distribute it, with or without # modifications, as long as this notice is preserved. -# serial 14 +# serial 15 AC_DEFUN([DC_DOVECOT_MODULEDIR],[ AC_ARG_WITH(moduledir, @@ -39,6 +39,10 @@ top_srcdir=\$[1] shift +if test "\$NOVALGRIND" != ""; then + exec \$[*] +fi + trap "rm -f test.out.\$\$" 0 1 2 3 15 supp_path="\$top_srcdir/run-test-valgrind.supp" if test -r "\$supp_path"; then From dovecot at dovecot.org Fri May 29 08:10:48 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 29 May 2015 08:10:48 +0000 Subject: dovecot-2.2: dovecot.m4: If run-test.sh fails, print the failing... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5e523f0bbdf8 changeset: 18757:5e523f0bbdf8 user: Timo Sirainen date: Fri May 29 11:08:29 2015 +0300 description: dovecot.m4: If run-test.sh fails, print the failing command to stderr. Currently otherwise the way Dovecot tests are run it's not clearly visible. diffstat: dovecot.m4 | 32 ++++++++++++++++++-------------- 1 files changed, 18 insertions(+), 14 deletions(-) diffs (50 lines): diff -r 2dd8308f5d57 -r 5e523f0bbdf8 dovecot.m4 --- a/dovecot.m4 Fri May 29 10:43:38 2015 +0300 +++ b/dovecot.m4 Fri May 29 11:08:29 2015 +0300 @@ -6,7 +6,7 @@ # unlimited permission to copy and/or distribute it, with or without # modifications, as long as this notice is preserved. -# serial 15 +# serial 16 AC_DEFUN([DC_DOVECOT_MODULEDIR],[ AC_ARG_WITH(moduledir, @@ -40,20 +40,24 @@ shift if test "\$NOVALGRIND" != ""; then - exec \$[*] + \$[*] + ret=\$? +else + trap "rm -f test.out.\$\$" 0 1 2 3 15 + supp_path="\$top_srcdir/run-test-valgrind.supp" + if test -r "\$supp_path"; then + valgrind -q --suppressions="\$supp_path" --log-file=test.out.\$\$ \$[*] + else + valgrind -q --log-file=test.out.\$\$ \$[*] + fi + ret=\$? + if test -s test.out.\$\$; then + cat test.out.\$\$ + ret=1 + fi fi - -trap "rm -f test.out.\$\$" 0 1 2 3 15 -supp_path="\$top_srcdir/run-test-valgrind.supp" -if test -r "\$supp_path"; then - valgrind -q --suppressions="\$supp_path" --log-file=test.out.\$\$ \$[*] -else - valgrind -q --log-file=test.out.\$\$ \$[*] -fi -ret=\$? -if test -s test.out.\$\$; then - cat test.out.\$\$ - exit 1 +if test \$ret != 0; then + echo "Failed to run: \$[*]" >&2 fi exit \$ret EOF From dovecot at dovecot.org Fri May 29 08:52:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 29 May 2015 08:52:35 +0000 Subject: dovecot-2.2: fts: If precaching fails, stop precaching the rest ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/485ed5de2473 changeset: 18758:485ed5de2473 user: Timo Sirainen date: Fri May 29 11:50:18 2015 +0300 description: fts: If precaching fails, stop precaching the rest of the mails. If there are a lot of mails to be precached, this could mean that the precaching is attempted for a long time and every one of them fails the same way. diffstat: src/plugins/fts/fts-storage.c | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-) diffs (27 lines): diff -r 5e523f0bbdf8 -r 485ed5de2473 src/plugins/fts/fts-storage.c --- a/src/plugins/fts/fts-storage.c Fri May 29 11:08:29 2015 +0300 +++ b/src/plugins/fts/fts-storage.c Fri May 29 11:50:18 2015 +0300 @@ -281,6 +281,13 @@ { struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box); struct fts_search_context *fctx = FTS_CONTEXT(ctx); + struct fts_transaction_context *ft = FTS_CONTEXT(ctx->transaction); + + if (fctx == NULL && ft->failed) { + /* precaching already failed - stop now instead of potentially + going through the same failure for all the mails */ + return FALSE; + } if (fctx != NULL && fctx->indexer_ctx != NULL) { /* this command is still building the indexes */ @@ -381,6 +388,9 @@ pool_unref(&fctx->result_pool); fts_scores_unref(&fctx->scores); i_free(fctx); + } else { + if (ft->failed) + ret = -1; } if (fbox->module_ctx.super.search_deinit(ctx) < 0) ret = -1; From dovecot at dovecot.org Fri May 29 14:53:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 29 May 2015 14:53:37 +0000 Subject: dovecot-2.2: lib-fs: fs_write_stream_abort() now ignores missing... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1a52baa9bf27 changeset: 18759:1a52baa9bf27 user: Timo Sirainen date: Fri May 29 17:51:21 2015 +0300 description: lib-fs: fs_write_stream_abort() now ignores missing o_stream_nfinish() call diffstat: src/lib-fs/fs-api.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 485ed5de2473 -r 1a52baa9bf27 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Fri May 29 11:50:18 2015 +0300 +++ b/src/lib-fs/fs-api.c Fri May 29 17:51:21 2015 +0300 @@ -587,6 +587,8 @@ i_assert(*output == file->output); *output = NULL; + if (file->output != NULL) + o_stream_ignore_last_errors(file->output); T_BEGIN { (void)file->fs->v.write_stream_finish(file, FALSE); } T_END; From dovecot at dovecot.org Fri May 29 15:24:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 29 May 2015 15:24:35 +0000 Subject: dovecot-2.2: lib-fs: Updated fs_write_stream_*() comments relate... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/83f1b9956788 changeset: 18760:83f1b9956788 user: Timo Sirainen date: Fri May 29 18:22:17 2015 +0300 description: lib-fs: Updated fs_write_stream_*() comments related to ostream corking and nfinish. diffstat: src/lib-fs/fs-api.h | 10 +++++++--- 1 files changed, 7 insertions(+), 3 deletions(-) diffs (26 lines): diff -r 1a52baa9bf27 -r 83f1b9956788 src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Fri May 29 17:51:21 2015 +0300 +++ b/src/lib-fs/fs-api.h Fri May 29 18:22:17 2015 +0300 @@ -231,15 +231,19 @@ int fs_write(struct fs_file *file, const void *data, size_t size); /* Write to file via output stream. The stream will be destroyed by - fs_write_stream_finish/abort. */ + fs_write_stream_finish/abort. The returned ostream is already corked and + it doesn't need to be uncorked. */ struct ostream *fs_write_stream(struct fs_file *file); -/* Finish writing via stream. The file will be created/replaced/appended only +/* Finish writing via stream, calling also o_stream_nfinish() on the stream and + handling any pending errors. The file will be created/replaced/appended only after this call, same as with fs_write(). Anything written to the stream won't be visible earlier. Returns 1 if ok, 0 if async write isn't finished yet (retry calling fs_write_stream_finish_async()), -1 if error */ int fs_write_stream_finish(struct fs_file *file, struct ostream **output); int fs_write_stream_finish_async(struct fs_file *file); -/* Abort writing via stream. Anything written to the stream is discarded. */ +/* Abort writing via stream. Anything written to the stream is discarded. + o_stream_ignore_last_errors() is called on the output stream so the caller + doesn't need to do it. */ void fs_write_stream_abort(struct fs_file *file, struct ostream **output); /* Set a hash to the following write. The storage can then verify that the From dovecot at dovecot.org Fri May 29 16:14:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 29 May 2015 16:14:07 +0000 Subject: dovecot-2.2: cassandra: Added log_level parameter to connect_str... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/315adb25e6b4 changeset: 18761:315adb25e6b4 user: Timo Sirainen date: Fri May 29 19:11:50 2015 +0300 description: cassandra: Added log_level parameter to connect_string. Available values are critical, error, warn (default), info, debug and trace. diffstat: src/lib-sql/driver-cassandra.c | 32 ++++++++++++++++++++++++++++++++ 1 files changed, 32 insertions(+), 0 deletions(-) diffs (77 lines): diff -r 83f1b9956788 -r 315adb25e6b4 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Fri May 29 18:22:17 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Fri May 29 19:11:50 2015 +0300 @@ -30,6 +30,7 @@ char *hosts, *keyspace; CassConsistency consistency; + CassLogLevel log_level; CassCluster *cluster; CassSession *session; @@ -102,6 +103,18 @@ { CASS_CONSISTENCY_LOCAL_ONE, "local-one" } }; +static struct { + CassLogLevel log_level; + const char *name; +} cass_log_level_names[] = { + { CASS_LOG_CRITICAL, "critical" }, + { CASS_LOG_ERROR, "error" }, + { CASS_LOG_WARN, "warn" }, + { CASS_LOG_INFO, "info" }, + { CASS_LOG_DEBUG, "debug" }, + { CASS_LOG_TRACE, "trace" } +}; + static void result_finish(struct cassandra_result *result); static int consistency_parse(const char *str, CassConsistency *consistency_r) @@ -117,6 +130,19 @@ return -1; } +static int log_level_parse(const char *str, CassLogLevel *log_level_r) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(cass_log_level_names); i++) { + if (strcmp(cass_log_level_names[i].name, str) == 0) { + *log_level_r = cass_log_level_names[i].log_level; + return 0; + } + } + return -1; +} + static void driver_cassandra_set_state(struct cassandra_db *db, enum sql_db_state state) { i_assert(state == SQL_DB_STATE_BUSY || db->cur_result == NULL); @@ -304,6 +330,8 @@ const char *const *args, *key, *value; string_t *hosts = t_str_new(64); + db->log_level = CASS_LOG_WARN; + args = t_strsplit_spaces(connect_string, " "); for (; *args != NULL; args++) { value = strchr(*args, '='); @@ -325,6 +353,9 @@ if (consistency_parse(value, &db->consistency) < 0) i_fatal("cassandra: Unknown consistency: %s", value); db->set_consistency = TRUE; + } else if (strcmp(key, "log_level") == 0) { + if (log_level_parse(value, &db->log_level) < 0) + i_fatal("cassandra: Unknown log_level: %s", value); } else { i_fatal("cassandra: Unknown connect string: %s", key); } @@ -348,6 +379,7 @@ T_BEGIN { driver_cassandra_parse_connect_string(db, connect_string); } T_END; + cass_log_set_level(db->log_level); db->cluster = cass_cluster_new(); cass_cluster_set_connect_timeout(db->cluster, SQL_CONNECT_TIMEOUT_SECS * 1000); From dovecot at dovecot.org Fri May 29 17:58:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 29 May 2015 17:58:30 +0000 Subject: dovecot-2.2: auth: Moved var_expand() related code to its own file. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cb66b25060b7 changeset: 18762:cb66b25060b7 user: Timo Sirainen date: Fri May 29 19:55:47 2015 +0300 description: auth: Moved var_expand() related code to its own file. diffstat: src/auth/Makefile.am | 2 + src/auth/auth-request-var-expand.c | 162 +++++++++++++++++++++++++++++++++++++ src/auth/auth-request-var-expand.h | 26 +++++ src/auth/auth-request.c | 157 ----------------------------------- src/auth/auth-request.h | 22 +---- 5 files changed, 191 insertions(+), 178 deletions(-) diffs (truncated from 435 to 300 lines): diff -r 315adb25e6b4 -r cb66b25060b7 src/auth/Makefile.am --- a/src/auth/Makefile.am Fri May 29 19:11:50 2015 +0300 +++ b/src/auth/Makefile.am Fri May 29 19:55:47 2015 +0300 @@ -70,6 +70,7 @@ auth-penalty.c \ auth-request.c \ auth-request-handler.c \ + auth-request-var-expand.c \ auth-settings.c \ auth-fields.c \ auth-token.c \ @@ -138,6 +139,7 @@ auth-penalty.h \ auth-request.h \ auth-request-handler.h \ + auth-request-var-expand.h \ auth-settings.h \ auth-fields.h \ auth-token.h \ diff -r 315adb25e6b4 -r cb66b25060b7 src/auth/auth-request-var-expand.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/auth/auth-request-var-expand.c Fri May 29 19:55:47 2015 +0300 @@ -0,0 +1,162 @@ +/* Copyright (c) 2002-2015 Dovecot authors, see the included COPYING file */ + +#include "auth-common.h" +#include "strescape.h" +#include "auth-request.h" + +const struct var_expand_table +auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT+1] = { + { 'u', NULL, "user" }, + { 'n', NULL, "username" }, + { 'd', NULL, "domain" }, + { 's', NULL, "service" }, + { 'h', NULL, "home" }, + { 'l', NULL, "lip" }, + { 'r', NULL, "rip" }, + { 'p', NULL, "pid" }, + { 'w', NULL, "password" }, + { '!', NULL, NULL }, + { 'm', NULL, "mech" }, + { 'c', NULL, "secured" }, + { 'a', NULL, "lport" }, + { 'b', NULL, "rport" }, + { 'k', NULL, "cert" }, + { '\0', NULL, "login_user" }, + { '\0', NULL, "login_username" }, + { '\0', NULL, "login_domain" }, + { '\0', NULL, "session" }, + { '\0', NULL, "real_lip" }, + { '\0', NULL, "real_rip" }, + { '\0', NULL, "real_lport" }, + { '\0', NULL, "real_rport" }, + { '\0', NULL, "domain_first" }, + { '\0', NULL, "domain_last" }, + { '\0', NULL, "master_user" }, + { '\0', NULL, "session_pid" }, + { '\0', NULL, "orig_user" }, + { '\0', NULL, "orig_username" }, + { '\0', NULL, "orig_domain" }, + /* be sure to update AUTH_REQUEST_VAR_TAB_COUNT */ + { '\0', NULL, NULL } +}; + +static const char * +escape_none(const char *string, + const struct auth_request *request ATTR_UNUSED) +{ + return string; +} + +const char * +auth_request_str_escape(const char *string, + const struct auth_request *request ATTR_UNUSED) +{ + return str_escape(string); +} + +struct var_expand_table * +auth_request_get_var_expand_table_full(const struct auth_request *auth_request, + auth_request_escape_func_t *escape_func, + unsigned int *count) +{ + const unsigned int auth_count = + N_ELEMENTS(auth_request_var_expand_static_tab); + struct var_expand_table *tab, *ret_tab; + const char *orig_user; + + if (escape_func == NULL) + escape_func = escape_none; + + /* keep the extra fields at the beginning. the last static_tab field + contains the ending NULL-fields. */ + tab = ret_tab = t_malloc((*count + auth_count) * sizeof(*tab)); + memset(tab, 0, *count * sizeof(*tab)); + tab += *count; + *count += auth_count; + + memcpy(tab, auth_request_var_expand_static_tab, + auth_count * sizeof(*tab)); + + tab[0].value = escape_func(auth_request->user, auth_request); + tab[1].value = escape_func(t_strcut(auth_request->user, '@'), + auth_request); + tab[2].value = strchr(auth_request->user, '@'); + if (tab[2].value != NULL) + tab[2].value = escape_func(tab[2].value+1, auth_request); + tab[3].value = auth_request->service; + /* tab[4] = we have no home dir */ + if (auth_request->local_ip.family != 0) + tab[5].value = net_ip2addr(&auth_request->local_ip); + if (auth_request->remote_ip.family != 0) + tab[6].value = net_ip2addr(&auth_request->remote_ip); + tab[7].value = dec2str(auth_request->client_pid); + if (auth_request->mech_password != NULL) { + tab[8].value = escape_func(auth_request->mech_password, + auth_request); + } + if (auth_request->userdb_lookup) { + tab[9].value = auth_request->userdb == NULL ? "" : + dec2str(auth_request->userdb->userdb->id); + } else { + tab[9].value = auth_request->passdb == NULL ? "" : + dec2str(auth_request->passdb->passdb->id); + } + tab[10].value = auth_request->mech_name == NULL ? "" : + auth_request->mech_name; + tab[11].value = auth_request->secured ? "secured" : ""; + tab[12].value = dec2str(auth_request->local_port); + tab[13].value = dec2str(auth_request->remote_port); + tab[14].value = auth_request->valid_client_cert ? "valid" : ""; + + if (auth_request->requested_login_user != NULL) { + const char *login_user = auth_request->requested_login_user; + + tab[15].value = escape_func(login_user, auth_request); + tab[16].value = escape_func(t_strcut(login_user, '@'), + auth_request); + tab[17].value = strchr(login_user, '@'); + if (tab[17].value != NULL) { + tab[17].value = escape_func(tab[17].value+1, + auth_request); + } + } + tab[18].value = auth_request->session_id == NULL ? NULL : + escape_func(auth_request->session_id, auth_request); + if (auth_request->real_local_ip.family != 0) + tab[19].value = net_ip2addr(&auth_request->real_local_ip); + if (auth_request->real_remote_ip.family != 0) + tab[20].value = net_ip2addr(&auth_request->real_remote_ip); + tab[21].value = dec2str(auth_request->real_local_port); + tab[22].value = dec2str(auth_request->real_remote_port); + tab[23].value = strchr(auth_request->user, '@'); + if (tab[23].value != NULL) { + tab[23].value = escape_func(t_strcut(tab[23].value+1, '@'), + auth_request); + } + tab[24].value = strrchr(auth_request->user, '@'); + if (tab[24].value != NULL) + tab[24].value = escape_func(tab[24].value+1, auth_request); + tab[25].value = auth_request->master_user == NULL ? NULL : + escape_func(auth_request->master_user, auth_request); + tab[26].value = auth_request->session_pid == (pid_t)-1 ? NULL : + dec2str(auth_request->session_pid); + + orig_user = auth_request->original_username != NULL ? + auth_request->original_username : auth_request->user; + tab[27].value = escape_func(orig_user, auth_request); + tab[28].value = escape_func(t_strcut(orig_user, '@'), auth_request); + tab[29].value = strchr(orig_user, '@'); + if (tab[29].value != NULL) + tab[29].value = escape_func(tab[29].value+1, auth_request); + return ret_tab; +} + +const struct var_expand_table * +auth_request_get_var_expand_table(const struct auth_request *auth_request, + auth_request_escape_func_t *escape_func) +{ + unsigned int count = 0; + + return auth_request_get_var_expand_table_full(auth_request, escape_func, + &count); +} diff -r 315adb25e6b4 -r cb66b25060b7 src/auth/auth-request-var-expand.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/auth/auth-request-var-expand.h Fri May 29 19:55:47 2015 +0300 @@ -0,0 +1,26 @@ +#ifndef AUTH_REQUEST_VAR_EXPAND_H +#define AUTH_REQUEST_VAR_EXPAND_H + +typedef const char * +auth_request_escape_func_t(const char *string, + const struct auth_request *auth_request); + +#define AUTH_REQUEST_VAR_TAB_USER_IDX 0 +#define AUTH_REQUEST_VAR_TAB_USERNAME_IDX 1 +#define AUTH_REQUEST_VAR_TAB_DOMAIN_IDX 2 +#define AUTH_REQUEST_VAR_TAB_COUNT 30 +extern const struct var_expand_table +auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT+1]; + +const struct var_expand_table * +auth_request_get_var_expand_table(const struct auth_request *auth_request, + auth_request_escape_func_t *escape_func) + ATTR_NULL(2); +struct var_expand_table * +auth_request_get_var_expand_table_full(const struct auth_request *auth_request, + auth_request_escape_func_t *escape_func, + unsigned int *count) ATTR_NULL(2); +const char *auth_request_str_escape(const char *string, + const struct auth_request *request); + +#endif diff -r 315adb25e6b4 -r cb66b25060b7 src/auth/auth-request.c --- a/src/auth/auth-request.c Fri May 29 19:11:50 2015 +0300 +++ b/src/auth/auth-request.c Fri May 29 19:55:47 2015 +0300 @@ -2034,163 +2034,6 @@ return ret; } -static const char * -escape_none(const char *string, - const struct auth_request *request ATTR_UNUSED) -{ - return string; -} - -const char * -auth_request_str_escape(const char *string, - const struct auth_request *request ATTR_UNUSED) -{ - return str_escape(string); -} - -const struct var_expand_table -auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT+1] = { - { 'u', NULL, "user" }, - { 'n', NULL, "username" }, - { 'd', NULL, "domain" }, - { 's', NULL, "service" }, - { 'h', NULL, "home" }, - { 'l', NULL, "lip" }, - { 'r', NULL, "rip" }, - { 'p', NULL, "pid" }, - { 'w', NULL, "password" }, - { '!', NULL, NULL }, - { 'm', NULL, "mech" }, - { 'c', NULL, "secured" }, - { 'a', NULL, "lport" }, - { 'b', NULL, "rport" }, - { 'k', NULL, "cert" }, - { '\0', NULL, "login_user" }, - { '\0', NULL, "login_username" }, - { '\0', NULL, "login_domain" }, - { '\0', NULL, "session" }, - { '\0', NULL, "real_lip" }, - { '\0', NULL, "real_rip" }, - { '\0', NULL, "real_lport" }, - { '\0', NULL, "real_rport" }, - { '\0', NULL, "domain_first" }, - { '\0', NULL, "domain_last" }, - { '\0', NULL, "master_user" }, - { '\0', NULL, "session_pid" }, - { '\0', NULL, "orig_user" }, - { '\0', NULL, "orig_username" }, - { '\0', NULL, "orig_domain" }, - /* be sure to update AUTH_REQUEST_VAR_TAB_COUNT */ - { '\0', NULL, NULL } -}; - -struct var_expand_table * -auth_request_get_var_expand_table_full(const struct auth_request *auth_request, - auth_request_escape_func_t *escape_func, - unsigned int *count) -{ - const unsigned int auth_count = - N_ELEMENTS(auth_request_var_expand_static_tab); - struct var_expand_table *tab, *ret_tab; - const char *orig_user; - - if (escape_func == NULL) - escape_func = escape_none; - - /* keep the extra fields at the beginning. the last static_tab field - contains the ending NULL-fields. */ - tab = ret_tab = t_malloc((*count + auth_count) * sizeof(*tab)); - memset(tab, 0, *count * sizeof(*tab)); - tab += *count; - *count += auth_count; - - memcpy(tab, auth_request_var_expand_static_tab, - auth_count * sizeof(*tab)); - - tab[0].value = escape_func(auth_request->user, auth_request); - tab[1].value = escape_func(t_strcut(auth_request->user, '@'), - auth_request); - tab[2].value = strchr(auth_request->user, '@'); - if (tab[2].value != NULL) From dovecot at dovecot.org Fri May 29 17:58:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 29 May 2015 17:58:30 +0000 Subject: dovecot-2.2: auth: Make sure %{mech} and %{session} is escaped i... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f59b5a599cae changeset: 18763:f59b5a599cae user: Timo Sirainen date: Fri May 29 20:55:58 2015 +0300 description: auth: Make sure %{mech} and %{session} is escaped in %var expansion. %{mech} is already very trusted and %{session} should be only from trusted sources as well, so this doesn't fix any actual security holes. They are also unlikely to have ever even been used in anything that requires escaping. diffstat: src/auth/auth-request-var-expand.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r cb66b25060b7 -r f59b5a599cae src/auth/auth-request-var-expand.c --- a/src/auth/auth-request-var-expand.c Fri May 29 19:55:47 2015 +0300 +++ b/src/auth/auth-request-var-expand.c Fri May 29 20:55:58 2015 +0300 @@ -83,7 +83,7 @@ tab[2].value = strchr(auth_request->user, '@'); if (tab[2].value != NULL) tab[2].value = escape_func(tab[2].value+1, auth_request); - tab[3].value = auth_request->service; + tab[3].value = escape_func(auth_request->service, auth_request); /* tab[4] = we have no home dir */ if (auth_request->local_ip.family != 0) tab[5].value = net_ip2addr(&auth_request->local_ip); @@ -102,7 +102,7 @@ dec2str(auth_request->passdb->passdb->id); } tab[10].value = auth_request->mech_name == NULL ? "" : - auth_request->mech_name; + escape_func(auth_request->mech_name, auth_request); tab[11].value = auth_request->secured ? "secured" : ""; tab[12].value = dec2str(auth_request->local_port); tab[13].value = dec2str(auth_request->remote_port); From dovecot at dovecot.org Fri May 29 17:58:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 29 May 2015 17:58:30 +0000 Subject: dovecot-2.2: auth: Added unit test for auth %variable expansion. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fa891f697005 changeset: 18764:fa891f697005 user: Timo Sirainen date: Fri May 29 20:56:13 2015 +0300 description: auth: Added unit test for auth %variable expansion. diffstat: src/auth/Makefile.am | 5 + src/auth/test-auth-request-var-expand.c | 201 ++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+), 0 deletions(-) diffs (227 lines): diff -r f59b5a599cae -r fa891f697005 src/auth/Makefile.am --- a/src/auth/Makefile.am Fri May 29 20:55:58 2015 +0300 +++ b/src/auth/Makefile.am Fri May 29 20:56:13 2015 +0300 @@ -197,6 +197,7 @@ test_programs = \ test-auth-cache \ + test-auth-request-var-expand \ test-db-dict noinst_PROGRAMS = $(test_programs) @@ -209,6 +210,10 @@ test_auth_cache_LDADD = auth-cache.o $(test_libs) test_auth_cache_DEPENDENCIES = $(pkglibexec_PROGRAMS) $(test_libs) +test_auth_request_var_expand_SOURCES = test-auth-request-var-expand.c +test_auth_request_var_expand_LDADD = auth-request-var-expand.o $(test_libs) +test_auth_request_var_expand_DEPENDENCIES = $(pkglibexec_PROGRAMS) $(test_libs) + test_db_dict_SOURCES = test-db-dict.c test_db_dict_LDADD = db-dict-cache-key.o $(test_libs) test_db_dict_DEPENDENCIES = $(pkglibexec_PROGRAMS) $(test_libs) diff -r f59b5a599cae -r fa891f697005 src/auth/test-auth-request-var-expand.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/auth/test-auth-request-var-expand.c Fri May 29 20:56:13 2015 +0300 @@ -0,0 +1,201 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "auth.h" +#include "passdb.h" +#include "userdb.h" +#include "auth-request.h" +#include "test-common.h" + +static struct passdb_module test_passdb = { + .id = 40 +}; +static struct userdb_module test_userdb = { + .id = 41 +}; + +static struct auth_passdb test_auth_passdb = { + .passdb = &test_passdb +}; +static struct auth_userdb test_auth_userdb = { + .userdb = &test_userdb +}; + +static const struct auth_request default_test_request = { + .user = "-user at +domain1@+domain2", + .service = "-service", + + .local_ip = { .family = AF_INET, .u.ip4.s_addr = 123456789 }, + .remote_ip = { .family = AF_INET, .u.ip4.s_addr = 1234567890 }, + .client_pid = 54321, + .mech_password = "-password", + .mech_name = "-mech", + .secured = TRUE, + .local_port = 21, + .remote_port = 210, + .valid_client_cert = TRUE, + + .requested_login_user = "-loginuser at +logindomain1@+logindomain2", + .session_id = "-session", + .real_local_ip = { .family = AF_INET, .u.ip4.s_addr = 223456788 }, + .real_remote_ip = { .family = AF_INET, .u.ip4.s_addr = 223456789 }, + .real_local_port = 200, + .real_remote_port = 201, + .master_user = "-masteruser at -masterdomain1@-masterdomain2", + .session_pid = 5000, + .original_username = "-origuser at -origdomain1@-origdomain2", + + .passdb = &test_auth_passdb, + .userdb = &test_auth_userdb +}; + +static struct auth_request test_request; +static struct auth_request empty_test_request = { .user = "" }; + +static const char * +test_escape(const char *string, const struct auth_request *request) +{ + char *dest; + unsigned int i; + + test_assert(request == &test_request); + + dest = t_strdup_noconst(string); + for (i = 0; dest[i] != '\0'; i++) { + if (dest[i] == '-') + dest[i] = '+'; + } + return dest; +} + +static bool test_empty_request(string_t *str, const char *input) +{ + str_truncate(str, 0); + var_expand(str, input, + auth_request_get_var_expand_table(&empty_test_request, NULL)); + return strspn(str_c(str), "\n0") == str_len(str); +} + +static void test_auth_request_var_expand_shortlong(void) +{ + static const char *test_input_short = + "%u\n%n\n%d\n%s\n%h\n%l\n%r\n%p\n%w\n%m\n%c\n%a\n%b\n%k\n"; + static const char *test_input_long = + "%{user}\n%{username}\n%{domain}\n%{service}\n%{home}\n" + "%{lip}\n%{rip}\n%{pid}\n%{password}\n%{mech}\n%{secured}\n" + "%{lport}\n%{rport}\n%{cert}\n"; + static const char *test_output = + /* %{home} is intentionally always expanding to empty */ + "+user at +domain1@+domain2\n+user\n+domain1 at +domain2\n+service\n\n" + "21.205.91.7\n210.2.150.73\n54321\n+password\n+mech\nsecured\n" + "21\n210\nvalid\n"; + string_t *str = t_str_new(256); + + test_begin("auth request var expand short and long"); + + var_expand(str, test_input_short, + auth_request_get_var_expand_table(&test_request, test_escape)); + test_assert(strcmp(str_c(str), test_output) == 0); + + str_truncate(str, 0); + var_expand(str, test_input_long, + auth_request_get_var_expand_table(&test_request, test_escape)); + test_assert(strcmp(str_c(str), test_output) == 0); + + /* test with empty input that it won't crash */ + test_assert(test_empty_request(str, test_input_short)); + test_assert(test_empty_request(str, test_input_long)); + + test_end(); +} + +static void test_auth_request_var_expand_flags(void) +{ + static const char *test_input = "%!\n%{secured}\n%{cert}\n"; + string_t *str = t_str_new(10); + + test_begin("auth request var expand flags"); + + test_request.userdb_lookup = FALSE; + test_request.secured = FALSE; + test_request.valid_client_cert = FALSE; + var_expand(str, test_input, + auth_request_get_var_expand_table(&test_request, test_escape)); + test_assert(strcmp(str_c(str), "40\n\n\n") == 0); + + test_request.userdb_lookup = TRUE; + test_request.secured = TRUE; + test_request.valid_client_cert = TRUE; + + str_truncate(str, 0); + var_expand(str, test_input, + auth_request_get_var_expand_table(&test_request, test_escape)); + test_assert(strcmp(str_c(str), "41\nsecured\nvalid\n") == 0); + + test_assert(test_empty_request(str, test_input)); + test_end(); +} + +static void test_auth_request_var_expand_long(void) +{ + static const char *test_input = + "%{login_user}\n%{login_username}\n%{login_domain}\n%{session}\n" + "%{real_lip}\n%{real_rip}\n%{real_lport}\n%{real_rport}\n" + "%{master_user}\n%{session_pid}\n" + "%{orig_user}\n%{orig_username}\n%{orig_domain}\n"; + static const char *test_output = + "+loginuser at +logindomain1@+logindomain2\n+loginuser\n+logindomain1 at +logindomain2\n+session\n" + "20.174.81.13\n21.174.81.13\n200\n201\n" + "+masteruser at +masterdomain1@+masterdomain2\n5000\n" + "+origuser at +origdomain1@+origdomain2\n+origuser\n+origdomain1 at +origdomain2\n"; + string_t *str = t_str_new(256); + + test_begin("auth request var expand long-only"); + + var_expand(str, test_input, + auth_request_get_var_expand_table(&test_request, test_escape)); + test_assert(strcmp(str_c(str), test_output) == 0); + + test_assert(test_empty_request(str, test_input)); + test_end(); +} + +static void test_auth_request_var_expand_usernames(void) +{ + static const struct { + const char *username, *output; + } tests[] = { + { "-foo", "+foo\n\n\n\n+foo" }, + { "-foo at -domain", "+foo\n+domain\n+domain\n+domain\n+foo at +domain" }, + { "-foo at -domain1@-domain2", "+foo\n+domain1 at +domain2\n+domain1\n+domain2\n+foo at +domain1@+domain2" } + }; + static const char *test_input = + "%{username}\n%{domain}\n%{domain_first}\n%{domain_last}\n%{user}"; + string_t *str = t_str_new(64); + unsigned int i; + + test_begin("auth request var expand usernames"); + for (i = 0; i < N_ELEMENTS(tests); i++) { + test_request.user = t_strdup_noconst(tests[i].username); + str_truncate(str, 0); + var_expand(str, test_input, + auth_request_get_var_expand_table(&test_request, test_escape)); + test_assert_idx(strcmp(str_c(str), tests[i].output) == 0, i); + } + test_request.user = default_test_request.user; + test_end(); +} + +int main(void) +{ + static void (*test_functions[])(void) = { + test_auth_request_var_expand_shortlong, + test_auth_request_var_expand_flags, + test_auth_request_var_expand_long, + test_auth_request_var_expand_usernames, + NULL + }; + test_request = default_test_request; + return test_run(test_functions); +} From dovecot at dovecot.org Fri May 29 18:41:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 29 May 2015 18:41:50 +0000 Subject: dovecot-2.2: auth: Added %{passdb:field} and %{userdb:field} var... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bb1522e10108 changeset: 18765:bb1522e10108 user: Timo Sirainen date: Fri May 29 21:39:33 2015 +0300 description: auth: Added %{passdb:field} and %{userdb:field} variables The field expands to either the passdb or userdb extra field. You can also use %{passdb:field:defaultvalue} where if field doesn't exist, it's expanded to defaultvalue. Note that an empty value means that the field still exists and it's not expanded to defaultvalue. diffstat: src/auth/Makefile.am | 2 +- src/auth/auth-request-var-expand.c | 83 +++++++++++++++++++++++++++++++++ src/auth/auth-request-var-expand.h | 13 +++++ src/auth/auth-request.c | 7 +- src/auth/db-checkpassword.c | 4 +- src/auth/db-passwd-file.c | 10 +-- src/auth/passdb-imap.c | 7 +- src/auth/passdb-ldap.c | 19 ++---- src/auth/passdb-pam.c | 7 +-- src/auth/passdb-passwd-file.c | 4 +- src/auth/passdb-sql.c | 26 +++------ src/auth/passdb-static.c | 10 +--- src/auth/passdb-template.c | 4 +- src/auth/test-auth-request-var-expand.c | 38 +++++++++++++++ src/auth/userdb-dict.c | 6 +- src/auth/userdb-ldap.c | 16 +---- src/auth/userdb-passwd-file.c | 4 +- src/auth/userdb-sql.c | 27 +++------ src/auth/userdb-template.c | 4 +- 19 files changed, 188 insertions(+), 103 deletions(-) diffs (truncated from 760 to 300 lines): diff -r fa891f697005 -r bb1522e10108 src/auth/Makefile.am --- a/src/auth/Makefile.am Fri May 29 20:56:13 2015 +0300 +++ b/src/auth/Makefile.am Fri May 29 21:39:33 2015 +0300 @@ -211,7 +211,7 @@ test_auth_cache_DEPENDENCIES = $(pkglibexec_PROGRAMS) $(test_libs) test_auth_request_var_expand_SOURCES = test-auth-request-var-expand.c -test_auth_request_var_expand_LDADD = auth-request-var-expand.o $(test_libs) +test_auth_request_var_expand_LDADD = auth-request-var-expand.o auth-fields.o $(test_libs) test_auth_request_var_expand_DEPENDENCIES = $(pkglibexec_PROGRAMS) $(test_libs) test_db_dict_SOURCES = test-db-dict.c diff -r fa891f697005 -r bb1522e10108 src/auth/auth-request-var-expand.c --- a/src/auth/auth-request-var-expand.c Fri May 29 20:56:13 2015 +0300 +++ b/src/auth/auth-request-var-expand.c Fri May 29 21:39:33 2015 +0300 @@ -1,9 +1,15 @@ /* Copyright (c) 2002-2015 Dovecot authors, see the included COPYING file */ #include "auth-common.h" +#include "str.h" #include "strescape.h" #include "auth-request.h" +struct auth_request_var_expand_ctx { + struct auth_request *auth_request; + auth_request_escape_func_t *escape_func; +}; + const struct var_expand_table auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT+1] = { { 'u', NULL, "user" }, @@ -160,3 +166,80 @@ return auth_request_get_var_expand_table_full(auth_request, escape_func, &count); } + +static const char *field_get_default(const char *data) +{ + const char *p; + + p = strchr(data, ':'); + if (p == NULL) + return ""; + else { + /* default value given */ + return p+1; + } +} + +static const char * +auth_request_var_expand_func_passdb(const char *data, void *context) +{ + struct auth_request_var_expand_ctx *ctx = context; + const char *field_name = t_strcut(data, ':'); + const char *value; + + value = auth_fields_find(ctx->auth_request->extra_fields, field_name); + return ctx->escape_func(value != NULL ? value : field_get_default(data), + ctx->auth_request); +} + +static const char * +auth_request_var_expand_func_userdb(const char *data, void *context) +{ + struct auth_request_var_expand_ctx *ctx = context; + const char *field_name = t_strcut(data, ':'); + const char *value; + + value = ctx->auth_request->userdb_reply == NULL ? NULL : + auth_fields_find(ctx->auth_request->userdb_reply, field_name); + return ctx->escape_func(value != NULL ? value : field_get_default(data), + ctx->auth_request); +} + +const struct var_expand_func_table auth_request_var_funcs_table[] = { + { "passdb", auth_request_var_expand_func_passdb }, + { "userdb", auth_request_var_expand_func_userdb }, + { NULL, NULL } +}; + +void auth_request_var_expand(string_t *dest, const char *str, + struct auth_request *auth_request, + auth_request_escape_func_t *escape_func) +{ + auth_request_var_expand_with_table(dest, str, auth_request, + auth_request_get_var_expand_table(auth_request, escape_func), + escape_func); +} + +void auth_request_var_expand_with_table(string_t *dest, const char *str, + struct auth_request *auth_request, + const struct var_expand_table *table, + auth_request_escape_func_t *escape_func) +{ + struct auth_request_var_expand_ctx ctx; + + memset(&ctx, 0, sizeof(ctx)); + ctx.auth_request = auth_request; + ctx.escape_func = escape_func; + var_expand_with_funcs(dest, str, table, + auth_request_var_funcs_table, &ctx); +} + +const char * +t_auth_request_var_expand(const char *str, + struct auth_request *auth_request, + auth_request_escape_func_t *escape_func) +{ + string_t *dest = t_str_new(128); + auth_request_var_expand(dest, str, auth_request, escape_func); + return str_c(dest); +} diff -r fa891f697005 -r bb1522e10108 src/auth/auth-request-var-expand.h --- a/src/auth/auth-request-var-expand.h Fri May 29 20:56:13 2015 +0300 +++ b/src/auth/auth-request-var-expand.h Fri May 29 21:39:33 2015 +0300 @@ -20,6 +20,19 @@ auth_request_get_var_expand_table_full(const struct auth_request *auth_request, auth_request_escape_func_t *escape_func, unsigned int *count) ATTR_NULL(2); + +void auth_request_var_expand(string_t *dest, const char *str, + struct auth_request *auth_request, + auth_request_escape_func_t *escape_func); +void auth_request_var_expand_with_table(string_t *dest, const char *str, + struct auth_request *auth_request, + const struct var_expand_table *table, + auth_request_escape_func_t *escape_func); +const char * +t_auth_request_var_expand(const char *str, + struct auth_request *auth_request, + auth_request_escape_func_t *escape_func); + const char *auth_request_str_escape(const char *string, const struct auth_request *request); diff -r fa891f697005 -r bb1522e10108 src/auth/auth-request.c --- a/src/auth/auth-request.c Fri May 29 20:56:13 2015 +0300 +++ b/src/auth/auth-request.c Fri May 29 21:39:33 2015 +0300 @@ -1201,7 +1201,6 @@ /* username format given, put it through variable expansion. we'll have to temporarily replace request->user to get %u to be the wanted username */ - const struct var_expand_table *table; char *old_username; string_t *dest; @@ -1209,8 +1208,7 @@ request->user = user; dest = t_str_new(256); - table = auth_request_get_var_expand_table(request, NULL); - var_expand(dest, set->username_format, table); + auth_request_var_expand(dest, set->username_format, request, NULL); user = p_strdup(request->pool, str_c(dest)); request->user = old_username; @@ -1569,8 +1567,7 @@ struct stat st; path = t_str_new(256); - var_expand(path, path_template, - auth_request_get_var_expand_table(request, NULL)); + auth_request_var_expand(path, path_template, request, NULL); if (stat(str_c(path), &st) < 0) { auth_request_log_error(request, AUTH_SUBSYS_DB, "stat(%s) failed: %m", str_c(path)); diff -r fa891f697005 -r bb1522e10108 src/auth/db-checkpassword.c --- a/src/auth/db-checkpassword.c Fri May 29 20:56:13 2015 +0300 +++ b/src/auth/db-checkpassword.c Fri May 29 21:39:33 2015 +0300 @@ -15,7 +15,6 @@ #include "safe-memset.h" #include "strescape.h" #include "child-wait.h" -#include "var-expand.h" #include "db-checkpassword.h" #include @@ -298,8 +297,7 @@ string_t *str; str = t_str_new(256); - var_expand(str, args, - auth_request_get_var_expand_table(request, NULL)); + auth_request_var_expand(str, args, request, NULL); return t_strconcat(str_c(str), " ", checkpassword_reply_path, NULL); } diff -r fa891f697005 -r bb1522e10108 src/auth/db-passwd-file.c --- a/src/auth/db-passwd-file.c Fri May 29 20:56:13 2015 +0300 +++ b/src/auth/db-passwd-file.c Fri May 29 21:39:33 2015 +0300 @@ -13,7 +13,6 @@ #include "hash.h" #include "str.h" #include "eacces-error.h" -#include "var-expand.h" #include #include @@ -431,15 +430,13 @@ { struct passwd_file *pw; struct passwd_user *pu; - const struct var_expand_table *table; string_t *username, *dest; if (!db->vars) pw = db->default_file; else { - table = auth_request_get_var_expand_table(request, path_fix); dest = t_str_new(256); - var_expand(dest, db->path, table); + auth_request_var_expand(dest, db->path, request, path_fix); pw = hash_table_lookup(db->files, str_c(dest)); if (pw == NULL) { @@ -454,9 +451,8 @@ } username = t_str_new(256); - table = auth_request_get_var_expand_table(request, - auth_request_str_escape); - var_expand(username, username_format, table); + auth_request_var_expand(username, username_format, request, + auth_request_str_escape); auth_request_log_debug(request, AUTH_SUBSYS_DB, "lookup: user=%s file=%s", diff -r fa891f697005 -r bb1522e10108 src/auth/passdb-imap.c --- a/src/auth/passdb-imap.c Fri May 29 20:56:13 2015 +0300 +++ b/src/auth/passdb-imap.c Fri May 29 21:39:33 2015 +0300 @@ -3,7 +3,6 @@ #include "auth-common.h" #include "passdb.h" #include "str.h" -#include "var-expand.h" #include "imap-resp-code.h" #include "imapc-client.h" @@ -77,7 +76,6 @@ (struct imap_passdb_module *)_module; struct imap_auth_request *request; struct imapc_client_settings set; - const struct var_expand_table *table; string_t *str; set = module->set; @@ -90,12 +88,11 @@ if (module->set_have_vars) { str = t_str_new(128); - table = auth_request_get_var_expand_table(auth_request, NULL); - var_expand(str, set.username, table); + auth_request_var_expand(str, set.username, auth_request, NULL); set.username = t_strdup(str_c(str)); str_truncate(str, 0); - var_expand(str, set.host, table); + auth_request_var_expand(str, set.host, auth_request, NULL); set.host = t_strdup(str_c(str)); } auth_request_log_debug(auth_request, AUTH_SUBSYS_DB, diff -r fa891f697005 -r bb1522e10108 src/auth/passdb-ldap.c --- a/src/auth/passdb-ldap.c Fri May 29 20:56:13 2015 +0300 +++ b/src/auth/passdb-ldap.c Fri May 29 21:39:33 2015 +0300 @@ -8,7 +8,6 @@ #include "ioloop.h" #include "array.h" #include "str.h" -#include "var-expand.h" #include "password-scheme.h" #include "auth-cache.h" #include "db-ldap.h" @@ -289,20 +288,19 @@ (struct ldap_passdb_module *)_module; struct ldap_connection *conn = module->conn; struct ldap_request_search *srequest = &request->request.search; - const struct var_expand_table *vars; const char **attr_names = (const char **)conn->pass_attr_names; string_t *str; request->require_password = require_password; srequest->request.type = LDAP_REQUEST_TYPE_SEARCH; - vars = auth_request_get_var_expand_table(auth_request, ldap_escape); str = t_str_new(512); - var_expand(str, conn->set.base, vars); + auth_request_var_expand(str, conn->set.base, auth_request, ldap_escape); srequest->base = p_strdup(auth_request->pool, str_c(str)); str_truncate(str, 0); - var_expand(str, conn->set.pass_filter, vars); + auth_request_var_expand(str, conn->set.pass_filter, + auth_request, ldap_escape); srequest->filter = p_strdup(auth_request->pool, str_c(str)); srequest->attr_map = &conn->pass_attr_map; srequest->attributes = conn->pass_attr_names; @@ -325,18 +323,17 @@ (struct ldap_passdb_module *)_module; struct ldap_connection *conn = module->conn; struct ldap_request_search *srequest = &request->request.search;