From pigeonhole at rename-it.nl Mon Nov 2 17:56:00 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 02 Nov 2015 18:56:00 +0100 Subject: dovecot-2.2-pigeonhole: doveadm sieve plugin: Added proper handl... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/8ac7c7069b53 changeset: 2125:8ac7c7069b53 user: Stephan Bosch date: Mon Nov 02 18:55:14 2015 +0100 description: doveadm sieve plugin: Added proper handling of Sieve storage initialization failure occurring when sieve_enabled=no diffstat: src/plugins/doveadm-sieve/doveadm-sieve-cmd.c | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diffs (19 lines): diff -r 96d06e6a9127 -r 8ac7c7069b53 src/plugins/doveadm-sieve/doveadm-sieve-cmd.c --- a/src/plugins/doveadm-sieve/doveadm-sieve-cmd.c Thu Oct 29 22:14:32 2015 +0100 +++ b/src/plugins/doveadm-sieve/doveadm-sieve-cmd.c Mon Nov 02 18:55:14 2015 +0100 @@ -131,8 +131,14 @@ (ctx->svinst, user, SIEVE_STORAGE_FLAG_READWRITE, &error); if ( ctx->storage == NULL ) { switch ( error ) { + case SIEVE_ERROR_NOT_POSSIBLE: + error = SIEVE_ERROR_NOT_FOUND; + i_error("Failed to open Sieve storage: " + "Sieve is disabled for this user"); + break; case SIEVE_ERROR_NOT_FOUND: - i_error("Failed to open Sieve storage: Sieve disabled for user"); + i_error("Failed to open Sieve storage: " + "User cannot manage personal Sieve scripts."); break; default: i_error("Failed to open Sieve storage."); From dovecot at dovecot.org Wed Nov 4 09:40:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 04 Nov 2015 09:40:52 +0000 Subject: dovecot-2.2: lib: test-istream-crlf - fix expected output string... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5717e416f31f changeset: 19342:5717e416f31f user: Phil Carmody date: Wed Nov 04 11:39:58 2015 +0200 description: lib: test-istream-crlf - fix expected output strings, add more coverage Fix an out-by-one which was missing trailing carriage returns in _lf mode. Debug: input = [19]:``...........N.N....R'' Debug: output= [19]:``...........N.N....R'' Debug: data = [18]:``...........N.N....'' test-istream-crlf.c:78: Assert(#20) failed: size == str_len(output) Add some randomised tests to get better coverage of all possibilities. Signed-off-by: Phil Carmody diffstat: src/lib/test-istream-crlf.c | 27 ++++++++++++++++++++++++--- 1 files changed, 24 insertions(+), 3 deletions(-) diffs (50 lines): diff -r 1e7fe403dd50 -r 5717e416f31f src/lib/test-istream-crlf.c --- a/src/lib/test-istream-crlf.c Tue Oct 27 21:54:26 2015 -0600 +++ b/src/lib/test-istream-crlf.c Wed Nov 04 11:39:58 2015 +0200 @@ -24,7 +24,7 @@ crlf_istream = i_stream_create_lf(istream); for (i = 0; i < input_len; i++) { if (input[i] == '\r' && - (i == input_len || input[i+1] == '\n')) + (i == input_len-1 || input[i+1] == '\n')) ; else str_append_c(output, input[i]); @@ -66,9 +66,9 @@ test_assert(pos + (unsigned int)ret1 == size); pos += ret1; } - test_assert(memcmp(data, str_data(output), size) == 0); + test_assert_idx(memcmp(data, str_data(output), size) == 0, j*10000+i); } - test_assert(size == str_len(output)); + test_assert_idx(size == str_len(output), j*10000+i); i_stream_unref(&crlf_istream); i_stream_unref(&istream); } @@ -89,4 +89,25 @@ for (i = 0; i < N_ELEMENTS(input); i++) test_istream_crlf_input(input[i]); test_end(); + +#define ISTREAM_CRLF_TEST_REPS 1000 + test_begin("istream crlf(random)"); + for (i = 0; i < ISTREAM_CRLF_TEST_REPS; i++) T_BEGIN { + char buf[100]; + size_t len = 0; + while (len < sizeof(buf) - 1) { + switch(rand()%16) { + case 0: goto outahere; + case 1: buf[len] = '\r'; break; + case 2: buf[len] = '\n'; break; + default: buf[len]= '.'; break; + } + len++; + } + outahere: + buf[len] = '\0'; + if (len > 0) + test_istream_crlf_input(buf); + } T_END; + test_end(); } From dovecot at dovecot.org Thu Nov 5 09:42:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 05 Nov 2015 09:42:15 +0000 Subject: dovecot-2.2: auth: Fixed crash when using %{passdb:} or %{userdb... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c129d92e2735 changeset: 19343:c129d92e2735 user: Timo Sirainen date: Thu Nov 05 11:41:52 2015 +0200 description: auth: Fixed crash when using %{passdb:} or %{userdb:} functions when escape_func was NULL Patch by Michael Slusarz. diffstat: src/auth/auth-request-var-expand.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 5717e416f31f -r c129d92e2735 src/auth/auth-request-var-expand.c --- a/src/auth/auth-request-var-expand.c Wed Nov 04 11:39:58 2015 +0200 +++ b/src/auth/auth-request-var-expand.c Thu Nov 05 11:41:52 2015 +0200 @@ -229,7 +229,7 @@ memset(&ctx, 0, sizeof(ctx)); ctx.auth_request = auth_request; - ctx.escape_func = escape_func; + ctx.escape_func = escape_func == NULL ? escape_none : escape_func; var_expand_with_funcs(dest, str, table, auth_request_var_funcs_table, &ctx); } From pigeonhole at rename-it.nl Sat Nov 7 11:43:07 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 07 Nov 2015 12:43:07 +0100 Subject: dovecot-2.2-pigeonhole: Added pigeonhole.m4 to installation. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/8442c6d04ce8 changeset: 2126:8442c6d04ce8 user: Stephan Bosch date: Mon Nov 02 18:56:39 2015 +0100 description: Added pigeonhole.m4 to installation. diffstat: Makefile.am | 4 +++ pigeonhole.m4 | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 0 deletions(-) diffs (96 lines): diff -r 8ac7c7069b53 -r 8442c6d04ce8 Makefile.am --- a/Makefile.am Mon Nov 02 18:55:14 2015 +0100 +++ b/Makefile.am Mon Nov 02 18:56:39 2015 +0100 @@ -1,3 +1,5 @@ +aclocaldir = $(datadir)/aclocal + if BUILD_DOCS DOCS = doc endif @@ -31,6 +33,8 @@ hg log --style=changelog > ChangeLog endif +aclocal_DATA = pigeonhole.m4 + pigeonhole-version.h: noop $(SHELL) $(top_srcdir)/update-version.sh $(top_srcdir) $(top_builddir) diff -r 8ac7c7069b53 -r 8442c6d04ce8 pigeonhole.m4 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pigeonhole.m4 Mon Nov 02 18:56:39 2015 +0100 @@ -0,0 +1,74 @@ +# pigeonhole.m4 - Check presence of pigeonhole -*-Autoconf-*- +#. + +# serial 3 + +AC_DEFUN([DC_PIGEONHOLE],[ + AC_ARG_WITH(pigeonhole, + [ --with-pigeonhole=DIR Pigeonhole base directory], + pigeonholedir="$withval", + [ + pg_prefix=$prefix + test "x$pg_prefix" = xNONE && pg_prefix=$ac_default_prefix + pigeonholedir="$pg_prefix/include/dovecot/sieve" + ] + ) + + AC_MSG_CHECKING([for pigeonhole in "$pigeonholedir"]) + + top=`pwd` + cd $pigeonholedir + pigeonholedir=`pwd` + cd $top + AC_SUBST(pigeonholedir) + + PIGEONHOLE_TESTSUITE= + if test -f "$pigeonholedir/src/lib-sieve/sieve.h"; then + AC_MSG_RESULT([found]) + pigeonhole_incdir="$pigeonholedir" + LIBSIEVE_INCLUDE='\ + -I$(pigeonhole_incdir) \ + -I$(pigeonhole_incdir)/src/lib-sieve \ + -I$(pigeonhole_incdir)/src/lib-sieve/util \ + -I$(pigeonhole_incdir)/src/lib-sieve/plugins/copy \ + -I$(pigeonhole_incdir)/src/lib-sieve/plugins/enotify \ + -I$(pigeonhole_incdir)/src/lib-sieve/plugins/imap4flags \ + -I$(pigeonhole_incdir)/src/lib-sieve/plugins/mailbox \ + -I$(pigeonhole_incdir)/src/lib-sieve/plugins/variables' + if test -f "$pigeonholedir/src/testsuite/testsuite"; then + PIGEONHOLE_TESTSUITE="${pigeonholedir}/src/testsuite/testsuite" + fi + elif test -f "$pigeonholedir/sieve.h"; then + AC_MSG_RESULT([found]) + pigeonhole_incdir="$pigeonholedir" + LIBSIEVE_INCLUDE='-I$(pigeonhole_incdir)' + else + AC_MSG_RESULT([not found]) + AC_MSG_NOTICE([ + Pigeonhole Sieve headers not found from $pigeonholedir and they + are not installed in the Dovecot include path, use --with-pigeonhole=PATH + to give path to Pigeonhole sources or installed headers.]) + AC_MSG_ERROR([pigeonhole not found]) + fi + + DISTCHECK_CONFIGURE_FLAGS="$DISTCHECK_CONFIGURE_FLAGS --with-pigeonhole=$pigeonholedir" + + AM_CONDITIONAL(PIGEONHOLE_TESTSUITE_AVAILABLE, ! test -z "$PIGEONHOLE_TESTSUITE") + + pigeonhole_incdir="$pigeonholedir" + + AC_ARG_ENABLE(valgrind, + [AC_HELP_STRING([--enable-valgrind], [Enable Valgrind memory leak checks in testsuite [default=no]])], + if test x$enableval = xno || test x$enableval = xauto; then + want_valgrind=$enableval + else + want_valgrind=yes + fi, + want_valgrind=no) + AM_CONDITIONAL(PIGEONHOLE_TESTSUITE_VALGRIND, test "$want_valgrind" = "yes") + + AC_SUBST(pigeonhole_incdir) + + AC_SUBST(LIBSIEVE_INCLUDE) + AC_SUBST(PIGEONHOLE_TESTSUITE) +]) From pigeonhole at rename-it.nl Sat Nov 7 11:43:07 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 07 Nov 2015 12:43:07 +0100 Subject: dovecot-2.2-pigeonhole: testsuite: Allow setting configuration o... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/a146b20160f6 changeset: 2128:a146b20160f6 user: Stephan Bosch date: Mon Nov 02 18:56:39 2015 +0100 description: testsuite: Allow setting configuration options from command line. These will influence some of the performed tests. diffstat: src/testsuite/testsuite-mailstore.c | 2 ++ src/testsuite/testsuite-settings.c | 17 +++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diffs (52 lines): diff -r 62fb7272c675 -r a146b20160f6 src/testsuite/testsuite-mailstore.c --- a/src/testsuite/testsuite-mailstore.c Mon Nov 02 18:56:39 2015 +0100 +++ b/src/testsuite/testsuite-mailstore.c Mon Nov 02 18:56:39 2015 +0100 @@ -125,6 +125,8 @@ struct mail_user *testsuite_mailstore_get_user(void) { + if (testsuite_mailstore_user == NULL) + return sieve_tool_get_mail_user(sieve_tool); return testsuite_mailstore_user; } diff -r 62fb7272c675 -r a146b20160f6 src/testsuite/testsuite-settings.c --- a/src/testsuite/testsuite-settings.c Mon Nov 02 18:56:39 2015 +0100 +++ b/src/testsuite/testsuite-settings.c Mon Nov 02 18:56:39 2015 +0100 @@ -5,10 +5,12 @@ #include "hash.h" #include "imem.h" #include "strfuncs.h" +#include "mail-user.h" #include "sieve-common.h" #include "testsuite-common.h" +#include "testsuite-mailstore.h" #include "testsuite-settings.h" struct testsuite_setting { @@ -49,14 +51,17 @@ static const char *testsuite_setting_get (void *context ATTR_UNUSED, const char *identifier) { - struct testsuite_setting *setting = - hash_table_lookup(settings, identifier); + struct testsuite_setting *setting; + struct mail_user *user; - if ( setting == NULL ) { + setting = hash_table_lookup(settings, identifier); + if ( setting != NULL ) + return setting->value; + + user = testsuite_mailstore_get_user(); + if ( user == NULL ) return NULL; - } - - return setting->value; + return mail_user_plugin_getenv(user, identifier); } void testsuite_setting_set(const char *identifier, const char *value) From pigeonhole at rename-it.nl Sat Nov 7 11:43:07 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 07 Nov 2015 12:43:07 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Implemented means to override... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/62fb7272c675 changeset: 2127:62fb7272c675 user: Stephan Bosch date: Mon Nov 02 18:56:39 2015 +0100 description: lib-sieve: Implemented means to override existing (standard) extensions. diffstat: src/lib-sieve/sieve-extensions.c | 101 ++++++++++++++++++++++++++++---------- src/lib-sieve/sieve-extensions.h | 5 + 2 files changed, 79 insertions(+), 27 deletions(-) diffs (170 lines): diff -r 8442c6d04ce8 -r 62fb7272c675 src/lib-sieve/sieve-extensions.c --- a/src/lib-sieve/sieve-extensions.c Mon Nov 02 18:56:39 2015 +0100 +++ b/src/lib-sieve/sieve-extensions.c Mon Nov 02 18:56:39 2015 +0100 @@ -361,34 +361,64 @@ return FALSE; } +static struct sieve_extension *sieve_extension_lookup +(struct sieve_instance *svinst, const char *name) +{ + struct sieve_extension_registry *ext_reg = svinst->ext_reg; + + return hash_table_lookup(ext_reg->extension_index, name); +} + +static void sieve_extension_insert +(struct sieve_instance *svinst, const char *name, + struct sieve_extension *ext) +{ + struct sieve_extension_registry *ext_reg = svinst->ext_reg; + + hash_table_insert(ext_reg->extension_index, name, ext); +} + +static struct sieve_extension *sieve_extension_alloc +(struct sieve_instance *svinst, + const struct sieve_extension_def *extdef) +{ + struct sieve_extension_registry *ext_reg = svinst->ext_reg; + struct sieve_extension *ext, **extr; + int ext_id; + + ext_id = (int)array_count(&ext_reg->extensions); + + /* Add extension to the registry */ + extr = array_append_space(&ext_reg->extensions); + *extr = ext = p_new(svinst->pool, struct sieve_extension, 1); + ext->id = ext_id; + ext->def = extdef; + ext->svinst = svinst; + return ext; +} + static struct sieve_extension *_sieve_extension_register (struct sieve_instance *svinst, const struct sieve_extension_def *extdef, bool load, bool required) { - struct sieve_extension_registry *ext_reg = svinst->ext_reg; - struct sieve_extension *ext = - hash_table_lookup(ext_reg->extension_index, extdef->name); + struct sieve_extension *ext; + + ext = sieve_extension_lookup(svinst, extdef->name); /* Register extension if it is not registered already */ if ( ext == NULL ) { - struct sieve_extension **extr; + ext = sieve_extension_alloc(svinst, extdef); + sieve_extension_insert(svinst, extdef->name, ext); - int ext_id = (int)array_count(&ext_reg->extensions); + } else if ( ext->overridden ) { + /* Create a dummy */ + ext = sieve_extension_alloc(svinst, extdef); - /* Add extension to the registry */ - - extr = array_append_space(&ext_reg->extensions); - *extr = ext = p_new(svinst->pool, struct sieve_extension, 1); - ext->id = ext_id; - ext->def = extdef; - ext->svinst = svinst; - - hash_table_insert(ext_reg->extension_index, extdef->name, ext); - - /* Re-register it if it were previously unregistered - * (not going to happen) - */ - } else if ( ext->def == NULL ) { + } else { + /* Re-register it if it were previously unregistered + * (not going to happen) + */ + i_assert( ext->def == NULL || ext->def == extdef ); ext->def = extdef; } @@ -426,11 +456,11 @@ if ( ext_id >= 0 && ext_id < (int) array_count(&ext_reg->extensions) ) { mod_ext = array_idx(&ext_reg->extensions, ext_id); - sieve_extension_capabilities_unregister(*mod_ext); - _sieve_extension_unload(*mod_ext); - (*mod_ext)->loaded = FALSE; - (*mod_ext)->enabled = FALSE; - (*mod_ext)->def = NULL; + sieve_extension_capabilities_unregister(*mod_ext); + _sieve_extension_unload(*mod_ext); + (*mod_ext)->loaded = FALSE; + (*mod_ext)->enabled = FALSE; + (*mod_ext)->def = NULL; } } @@ -441,6 +471,25 @@ return _sieve_extension_register(svinst, extdef, load, TRUE); } +void sieve_extension_override +(struct sieve_instance *svinst, const char *name, + const struct sieve_extension *ext) +{ + struct sieve_extension_registry *ext_reg = ext->svinst->ext_reg; + struct sieve_extension * const *mod_ext; + struct sieve_extension *old_ext; + + i_assert( ext->id >= 0 && + ext->id < (int) array_count(&ext_reg->extensions) ); + mod_ext = array_idx(&ext_reg->extensions, ext->id); + + old_ext = sieve_extension_lookup(svinst, name); + i_assert( old_ext == NULL || !old_ext->overridden ); + + sieve_extension_insert(svinst, name, *mod_ext); + (*mod_ext)->overridden = TRUE; +} + int sieve_extensions_get_count(struct sieve_instance *svinst) { struct sieve_extension_registry *ext_reg = svinst->ext_reg; @@ -467,7 +516,6 @@ const struct sieve_extension *sieve_extension_get_by_name (struct sieve_instance *svinst, const char *name) { - struct sieve_extension_registry *ext_reg = svinst->ext_reg; const struct sieve_extension *ext; if ( *name == '@' ) @@ -476,8 +524,7 @@ if ( strlen(name) > 128 ) return NULL; - ext = hash_table_lookup(ext_reg->extension_index, name); - + ext = sieve_extension_lookup(svinst, name); if ( ext == NULL || ext->def == NULL || (!ext->enabled && !ext->required)) return NULL; diff -r 8442c6d04ce8 -r 62fb7272c675 src/lib-sieve/sieve-extensions.h --- a/src/lib-sieve/sieve-extensions.h Mon Nov 02 18:56:39 2015 +0100 +++ b/src/lib-sieve/sieve-extensions.h Mon Nov 02 18:56:39 2015 +0100 @@ -95,6 +95,7 @@ unsigned int enabled:1; unsigned int dummy:1; unsigned int global:1; + unsigned int overridden:1; }; #define sieve_extension_is(ext, definition) \ @@ -137,6 +138,10 @@ void sieve_extension_unregister(const struct sieve_extension *ext); +void sieve_extension_override + (struct sieve_instance *svinst, const char *name, + const struct sieve_extension *ext); + int sieve_extensions_get_count(struct sieve_instance *svinst); const struct sieve_extension *sieve_extension_get_by_id From pigeonhole at rename-it.nl Sat Nov 7 11:43:07 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 07 Nov 2015 12:43:07 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: imap4flags extension: Made fl... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/334f0ba2fd9b changeset: 2129:334f0ba2fd9b user: Stephan Bosch date: Mon Nov 02 18:56:39 2015 +0100 description: lib-sieve: imap4flags extension: Made flag manipulation API available to other extensions. diffstat: src/lib-sieve/plugins/imap4flags/Makefile.am | 8 ++- src/lib-sieve/plugins/imap4flags/cmd-flag.c | 6 +- src/lib-sieve/plugins/imap4flags/ext-imap4flags-common.c | 35 +++++++---- src/lib-sieve/plugins/imap4flags/ext-imap4flags-common.h | 21 +---- src/lib-sieve/plugins/imap4flags/sieve-ext-imap4flags.h | 50 ++++++++++++++++ src/lib-sieve/plugins/imap4flags/tst-hasflag.c | 4 +- 6 files changed, 90 insertions(+), 34 deletions(-) diffs (229 lines): diff -r a146b20160f6 -r 334f0ba2fd9b src/lib-sieve/plugins/imap4flags/Makefile.am --- a/src/lib-sieve/plugins/imap4flags/Makefile.am Mon Nov 02 18:56:39 2015 +0100 +++ b/src/lib-sieve/plugins/imap4flags/Makefile.am Mon Nov 02 18:56:39 2015 +0100 @@ -22,6 +22,12 @@ ext-imap4flags.c \ ext-imapflags.c +public_headers = \ + sieve-ext-imap4flags.h -noinst_HEADERS = \ +headers = \ ext-imap4flags-common.h + +pkginc_libdir=$(dovecot_pkgincludedir)/sieve +pkginc_lib_HEADERS = $(public_headers) +noinst_HEADERS = $(headers) diff -r a146b20160f6 -r 334f0ba2fd9b src/lib-sieve/plugins/imap4flags/cmd-flag.c --- a/src/lib-sieve/plugins/imap4flags/cmd-flag.c Mon Nov 02 18:56:39 2015 +0100 +++ b/src/lib-sieve/plugins/imap4flags/cmd-flag.c Mon Nov 02 18:56:39 2015 +0100 @@ -230,13 +230,13 @@ if ( sieve_operation_is(op, setflag_operation) ) { sieve_runtime_trace(renv, SIEVE_TRLVL_COMMANDS, "setflag command"); - flag_op = ext_imap4flags_set_flags; + flag_op = sieve_ext_imap4flags_set_flags; } else if ( sieve_operation_is(op, addflag_operation) ) { sieve_runtime_trace(renv, SIEVE_TRLVL_COMMANDS, "addflag command"); - flag_op = ext_imap4flags_add_flags; + flag_op = sieve_ext_imap4flags_add_flags; } else if ( sieve_operation_is(op, removeflag_operation) ) { sieve_runtime_trace(renv, SIEVE_TRLVL_COMMANDS, "removeflag command"); - flag_op = ext_imap4flags_remove_flags; + flag_op = sieve_ext_imap4flags_remove_flags; } else { i_unreached(); } diff -r a146b20160f6 -r 334f0ba2fd9b src/lib-sieve/plugins/imap4flags/ext-imap4flags-common.c --- a/src/lib-sieve/plugins/imap4flags/ext-imap4flags-common.c Mon Nov 02 18:56:39 2015 +0100 +++ b/src/lib-sieve/plugins/imap4flags/ext-imap4flags-common.c Mon Nov 02 18:56:39 2015 +0100 @@ -404,6 +404,11 @@ /* Flag operations */ +static string_t *ext_imap4flags_get_flag_variable +(const struct sieve_runtime_env *renv, + struct sieve_variable_storage *storage, unsigned int var_index) + ATTR_NULL(2); + static bool flags_list_flag_exists (string_t *flags_list, const char *flag) { @@ -479,8 +484,8 @@ } static string_t *ext_imap4flags_get_flag_variable -(const struct sieve_runtime_env *renv, struct sieve_variable_storage *storage, - unsigned int var_index) +(const struct sieve_runtime_env *renv, + struct sieve_variable_storage *storage, unsigned int var_index) { string_t *flags; @@ -504,9 +509,10 @@ return flags; } -int ext_imap4flags_set_flags -(const struct sieve_runtime_env *renv, struct sieve_variable_storage *storage, - unsigned int var_index, struct sieve_stringlist *flags) +int sieve_ext_imap4flags_set_flags +(const struct sieve_runtime_env *renv, + struct sieve_variable_storage *storage, unsigned int var_index, + struct sieve_stringlist *flags) { string_t *cur_flags = ext_imap4flags_get_flag_variable (renv, storage, var_index); @@ -531,9 +537,10 @@ return SIEVE_EXEC_BIN_CORRUPT; } -int ext_imap4flags_add_flags -(const struct sieve_runtime_env *renv, struct sieve_variable_storage *storage, - unsigned int var_index, struct sieve_stringlist *flags) +int sieve_ext_imap4flags_add_flags +(const struct sieve_runtime_env *renv, + struct sieve_variable_storage *storage, unsigned int var_index, + struct sieve_stringlist *flags) { string_t *cur_flags = ext_imap4flags_get_flag_variable (renv, storage, var_index); @@ -557,9 +564,10 @@ return SIEVE_EXEC_BIN_CORRUPT; } -int ext_imap4flags_remove_flags -(const struct sieve_runtime_env *renv, struct sieve_variable_storage *storage, - unsigned int var_index, struct sieve_stringlist *flags) +int sieve_ext_imap4flags_remove_flags +(const struct sieve_runtime_env *renv, + struct sieve_variable_storage *storage, unsigned int var_index, + struct sieve_stringlist *flags) { string_t *cur_flags = ext_imap4flags_get_flag_variable (renv, storage, var_index); @@ -690,8 +698,9 @@ /* Flag access */ -struct sieve_stringlist *ext_imap4flags_get_flags -(const struct sieve_runtime_env *renv, struct sieve_stringlist *flags_list) +struct sieve_stringlist *sieve_ext_imap4flags_get_flags +(const struct sieve_runtime_env *renv, + struct sieve_stringlist *flags_list) { if ( flags_list == NULL ) return ext_imap4flags_stringlist_create_single diff -r a146b20160f6 -r 334f0ba2fd9b src/lib-sieve/plugins/imap4flags/ext-imap4flags-common.h --- a/src/lib-sieve/plugins/imap4flags/ext-imap4flags-common.h Mon Nov 02 18:56:39 2015 +0100 +++ b/src/lib-sieve/plugins/imap4flags/ext-imap4flags-common.h Mon Nov 02 18:56:39 2015 +0100 @@ -9,6 +9,8 @@ #include "sieve-common.h" #include "sieve-ext-variables.h" +#include "sieve-ext-imap4flags.h" + /* * Extension */ @@ -89,24 +91,13 @@ /* Flag operations */ typedef int (*ext_imapflag_flag_operation_t) - (const struct sieve_runtime_env *renv, struct sieve_variable_storage *storage, - unsigned int var_index, struct sieve_stringlist *flags); - -int ext_imap4flags_set_flags - (const struct sieve_runtime_env *renv, struct sieve_variable_storage *storage, - unsigned int var_index, struct sieve_stringlist *flags); -int ext_imap4flags_add_flags - (const struct sieve_runtime_env *renv, struct sieve_variable_storage *storage, - unsigned int var_index, struct sieve_stringlist *flags); -int ext_imap4flags_remove_flags - (const struct sieve_runtime_env *renv, struct sieve_variable_storage *storage, - unsigned int var_index, struct sieve_stringlist *flags); + (const struct sieve_runtime_env *renv, + struct sieve_variable_storage *storage, + unsigned int var_index, struct sieve_stringlist *flags) + ATTR_NULL(2); /* Flags access */ -struct sieve_stringlist *ext_imap4flags_get_flags - (const struct sieve_runtime_env *renv, struct sieve_stringlist *flags_list); - void ext_imap4flags_get_implicit_flags_init (struct ext_imap4flags_iter *iter, const struct sieve_extension *this_ext, struct sieve_result *result); diff -r a146b20160f6 -r 334f0ba2fd9b src/lib-sieve/plugins/imap4flags/sieve-ext-imap4flags.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/imap4flags/sieve-ext-imap4flags.h Mon Nov 02 18:56:39 2015 +0100 @@ -0,0 +1,50 @@ +/* Copyright (c) 2002-2015 Pigeonhole authors, see the included COPYING file + */ + +#ifndef __SIEVE_EXT_IMAP4FLAGS_H +#define __SIEVE_EXT_IMAP4FLAGS_H + +struct sieve_variable_storage; + +/* + * Imap4flags extension + */ + +/* FIXME: this is not suitable for future plugin support */ + +extern const struct sieve_extension_def imap4flags_extension; + +static inline const struct sieve_extension * +sieve_ext_imap4flags_require_extension +(struct sieve_instance *svinst) +{ + return sieve_extension_require + (svinst, &imap4flags_extension, TRUE); +} + +/* + * Flag manipulation + */ + +int sieve_ext_imap4flags_set_flags +(const struct sieve_runtime_env *renv, + struct sieve_variable_storage *storage, unsigned int var_index, + struct sieve_stringlist *flags) ATTR_NULL(2); +int sieve_ext_imap4flags_add_flags +(const struct sieve_runtime_env *renv, + struct sieve_variable_storage *storage, unsigned int var_index, + struct sieve_stringlist *flags) ATTR_NULL(2); +int sieve_ext_imap4flags_remove_flags +(const struct sieve_runtime_env *renv, + struct sieve_variable_storage *storage, unsigned int var_index, + struct sieve_stringlist *flags) ATTR_NULL(2); + +/* + * Flag retrieval + */ + +struct sieve_stringlist *sieve_ext_imap4flags_get_flags +(const struct sieve_runtime_env *renv, + struct sieve_stringlist *flags_list); + +#endif diff -r a146b20160f6 -r 334f0ba2fd9b src/lib-sieve/plugins/imap4flags/tst-hasflag.c --- a/src/lib-sieve/plugins/imap4flags/tst-hasflag.c Mon Nov 02 18:56:39 2015 +0100 +++ b/src/lib-sieve/plugins/imap4flags/tst-hasflag.c Mon Nov 02 18:56:39 2015 +0100 @@ -225,11 +225,11 @@ sieve_runtime_trace(renv, SIEVE_TRLVL_TESTS, "hasflag test"); - value_list = ext_imap4flags_get_flags(renv, variables_list); + value_list = sieve_ext_imap4flags_get_flags(renv, variables_list); if ( sieve_match_type_is(&mcht, is_match_type) || sieve_match_type_is(&mcht, contains_match_type) ) - key_list = ext_imap4flags_get_flags(renv, flag_list); + key_list = sieve_ext_imap4flags_get_flags(renv, flag_list); else key_list = flag_list; From dovecot at dovecot.org Mon Nov 9 07:32:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 09 Nov 2015 07:32:17 +0000 Subject: dovecot-2.2: replicator: Send initial status notification to mas... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f5e997d20461 changeset: 19344:f5e997d20461 user: Timo Sirainen date: Mon Nov 09 09:31:48 2015 +0200 description: replicator: Send initial status notification to master before listing all users To avoid master process from killing us after 30 seconds if the user listing takes longer than that. diffstat: src/replication/replicator/replicator.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (16 lines): diff -r c129d92e2735 -r f5e997d20461 src/replication/replicator/replicator.c --- a/src/replication/replicator/replicator.c Thu Nov 05 11:41:52 2015 +0200 +++ b/src/replication/replicator/replicator.c Mon Nov 09 09:31:48 2015 +0200 @@ -104,9 +104,11 @@ restrict_access_by_env(NULL, FALSE); restrict_access_allow_coredumps(TRUE); + /* finish init before we get list of users from auth, because that + can take long enough for master process to kill us otherwise. */ + master_service_init_finish(master_service); main_init(); - master_service_init_finish(master_service); master_service_run(master_service, client_connected); main_deinit(); From dovecot at dovecot.org Mon Nov 9 11:03:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 09 Nov 2015 11:03:31 +0000 Subject: dovecot-2.2: lib: Removed unnecessary includes from bits.h Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5d58bdcafc7d changeset: 19345:5d58bdcafc7d user: Timo Sirainen date: Mon Nov 09 13:02:56 2015 +0200 description: lib: Removed unnecessary includes from bits.h All of them are already in lib.h, and bits.h gets included from lib.h. This also solves a compiling problem for systems where stdint.h doesn't exist. diffstat: src/lib/bits.h | 11 ----------- 1 files changed, 0 insertions(+), 11 deletions(-) diffs (21 lines): diff -r f5e997d20461 -r 5d58bdcafc7d src/lib/bits.h --- a/src/lib/bits.h Mon Nov 09 09:31:48 2015 +0200 +++ b/src/lib/bits.h Mon Nov 09 13:02:56 2015 +0200 @@ -1,17 +1,6 @@ #ifndef BITS_H #define BITS_H -/* default lib includes */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "macros.h" - -#include -#include -#include - #define UINT64_SUM_OVERFLOWS(a, b) \ (a > (uint64_t)-1 - b) From dovecot at dovecot.org Mon Nov 9 11:17:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 09 Nov 2015 11:17:16 +0000 Subject: dovecot-2.2: lib-storage: Support %{userdb:*} variables in mail_... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2f2d78bedeed changeset: 19346:2f2d78bedeed user: Timo Sirainen date: Mon Nov 09 13:16:50 2015 +0200 description: lib-storage: Support %{userdb:*} variables in mail_home and mail_chroot settings. diffstat: src/lib-storage/mail-storage-service.c | 16 +++++++++++----- src/lib-storage/mail-user.c | 4 +++- 2 files changed, 14 insertions(+), 6 deletions(-) diffs (62 lines): diff -r 5d58bdcafc7d -r 2f2d78bedeed src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Mon Nov 09 13:02:56 2015 +0200 +++ b/src/lib-storage/mail-storage-service.c Mon Nov 09 13:16:50 2015 +0200 @@ -88,6 +88,13 @@ struct module *mail_storage_service_modules = NULL; +static void +mail_storage_service_var_expand(struct mail_storage_service_ctx *ctx, + string_t *str, const char *format, + struct mail_storage_service_user *user, + const struct mail_storage_service_input *input, + const struct mail_storage_service_privileges *priv); + static bool mail_user_set_get_mail_debug(const struct setting_parser_info *user_info, const struct mail_user_settings *user_set) @@ -443,7 +450,7 @@ } static const char * -user_expand_varstr(struct master_service *service, +user_expand_varstr(struct mail_storage_service_ctx *ctx, struct mail_storage_service_user *user, struct mail_storage_service_privileges *priv, const char *str) @@ -456,8 +463,7 @@ i_assert(*str == SETTING_STRVAR_UNEXPANDED[0]); ret = t_str_new(256); - var_expand(ret, str + 1, - get_var_expand_table(service, user, &user->input, priv)); + mail_storage_service_var_expand(ctx, ret, str + 1, user, &user->input, priv); return str_c(ret); } @@ -512,9 +518,9 @@ /* variable strings are expanded in mail_user_init(), but we need the home and chroot sooner so do them separately here. */ - priv_r->home = user_expand_varstr(ctx->service, user, priv_r, + priv_r->home = user_expand_varstr(ctx, user, priv_r, user->user_set->mail_home); - priv_r->chroot = user_expand_varstr(ctx->service, user, priv_r, + priv_r->chroot = user_expand_varstr(ctx, user, priv_r, user->user_set->mail_chroot); return 0; } diff -r 5d58bdcafc7d -r 2f2d78bedeed src/lib-storage/mail-user.c --- a/src/lib-storage/mail-user.c Mon Nov 09 13:02:56 2015 +0200 +++ b/src/lib-storage/mail-user.c Mon Nov 09 13:16:50 2015 +0200 @@ -377,7 +377,9 @@ return; str = t_str_new(128); - var_expand(str, home, mail_user_var_expand_table(user)); + var_expand_with_funcs(str, home, + mail_user_var_expand_table(user), + mail_user_var_expand_func_table, user); user->_home = p_strdup(user->pool, str_c(str)); } From dovecot at dovecot.org Mon Nov 9 12:11:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 09 Nov 2015 12:11:39 +0000 Subject: dovecot-2.2: auth: Don't crash when trying to use CRYPT scheme w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/61eb9ac0d29e changeset: 19347:61eb9ac0d29e user: Timo Sirainen date: Mon Nov 09 14:11:12 2015 +0200 description: auth: Don't crash when trying to use CRYPT scheme when crypt() doesn't support DES diffstat: src/auth/password-scheme-crypt.c | 16 ++++++++++++++++ src/auth/password-scheme.c | 14 -------------- 2 files changed, 16 insertions(+), 14 deletions(-) diffs (71 lines): diff -r 2f2d78bedeed -r 61eb9ac0d29e src/auth/password-scheme-crypt.c --- a/src/auth/password-scheme-crypt.c Mon Nov 09 13:16:50 2015 +0200 +++ b/src/auth/password-scheme-crypt.c Mon Nov 09 14:11:12 2015 +0200 @@ -24,6 +24,19 @@ } static void +crypt_generate_des(const char *plaintext, const char *user ATTR_UNUSED, + const unsigned char **raw_password_r, size_t *size_r) +{ +#define CRYPT_SALT_LEN 2 + const char *password, *salt; + + salt = password_generate_salt(CRYPT_SALT_LEN); + password = t_strdup(mycrypt(plaintext, salt)); + *raw_password_r = (const unsigned char *)password; + *size_r = strlen(password); +} + +static void crypt_generate_blowfisch(const char *plaintext, const char *user ATTR_UNUSED, const unsigned char **raw_password_r, size_t *size_r) { @@ -98,6 +111,7 @@ const char *salt; const char *expected; } sample[] = { + { "08/15!test~4711", "JB", "JBOZ0DgmtucwE" }, { "08/15!test~4711", "$2a$04$0123456789abcdefABCDEF", "$2a$04$0123456789abcdefABCDE.N.drYX5yIAL1LkTaaZotW3yI0hQhZru" }, { "08/15!test~4711", "$5$rounds=1000$0123456789abcdef", @@ -110,6 +124,8 @@ /* keep in sync with the sample struct above */ static const struct password_scheme crypt_schemes[] = { + { "CRYPT", PW_ENCODING_NONE, 0, crypt_verify, + crypt_generate_des }, { "BLF-CRYPT", PW_ENCODING_NONE, 0, crypt_verify, crypt_generate_blowfisch }, { "SHA256-CRYPT", PW_ENCODING_NONE, 0, crypt_verify, diff -r 2f2d78bedeed -r 61eb9ac0d29e src/auth/password-scheme.c --- a/src/auth/password-scheme.c Mon Nov 09 13:16:50 2015 +0200 +++ b/src/auth/password-scheme.c Mon Nov 09 14:11:12 2015 +0200 @@ -341,19 +341,6 @@ return strcmp(crypted, password) == 0 ? 1 : 0; } -static void -crypt_generate(const char *plaintext, const char *user ATTR_UNUSED, - const unsigned char **raw_password_r, size_t *size_r) -{ -#define CRYPT_SALT_LEN 2 - const char *password, *salt; - - salt = password_generate_salt(CRYPT_SALT_LEN); - password = t_strdup(mycrypt(plaintext, salt)); - *raw_password_r = (const unsigned char *)password; - *size_r = strlen(password); -} - static int md5_verify(const char *plaintext, const char *user, const unsigned char *raw_password, size_t size, const char **error_r) @@ -803,7 +790,6 @@ } static const struct password_scheme builtin_schemes[] = { - { "CRYPT", PW_ENCODING_NONE, 0, crypt_verify, crypt_generate }, { "MD5", PW_ENCODING_NONE, 0, md5_verify, md5_crypt_generate }, { "MD5-CRYPT", PW_ENCODING_NONE, 0, md5_crypt_verify, md5_crypt_generate }, From dovecot at dovecot.org Mon Nov 9 12:49:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 09 Nov 2015 12:49:52 +0000 Subject: dovecot-2.2: virtual: Include mailbox name in "Backend mailbox a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a2824031f6a8 changeset: 19348:a2824031f6a8 user: Timo Sirainen date: Mon Nov 09 14:49:26 2015 +0200 description: virtual: Include mailbox name in "Backend mailbox added by another session." error. diffstat: src/plugins/virtual/virtual-sync.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (16 lines): diff -r 61eb9ac0d29e -r a2824031f6a8 src/plugins/virtual/virtual-sync.c --- a/src/plugins/virtual/virtual-sync.c Mon Nov 09 14:11:12 2015 +0200 +++ b/src/plugins/virtual/virtual-sync.c Mon Nov 09 14:49:26 2015 +0200 @@ -134,9 +134,9 @@ /* another process just added a new mailbox. we can't handle this currently. */ mbox->inconsistent = TRUE; - mail_storage_set_error(mbox->box.storage, MAIL_ERROR_TEMP, - "Backend mailbox added by another session. " - "Reopen the virtual mailbox."); + mail_storage_set_error(mbox->box.storage, MAIL_ERROR_TEMP, t_strdup_printf( + "Backend mailbox '%s' added by another session. " + "Reopen the virtual mailbox.", name)); return -1; } From dovecot at dovecot.org Tue Nov 10 08:00:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 10 Nov 2015 08:00:40 +0000 Subject: dovecot-2.2: push-notification: Removed unused code. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0b1c73b01a5a changeset: 19349:0b1c73b01a5a user: Timo Sirainen date: Tue Nov 10 10:00:11 2015 +0200 description: push-notification: Removed unused code. diffstat: src/plugins/push-notification/push-notification-events.h | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diffs (12 lines): diff -r a2824031f6a8 -r 0b1c73b01a5a src/plugins/push-notification/push-notification-events.h --- a/src/plugins/push-notification/push-notification-events.h Mon Nov 09 14:49:26 2015 +0200 +++ b/src/plugins/push-notification/push-notification-events.h Tue Nov 10 10:00:11 2015 +0200 @@ -14,8 +14,6 @@ struct push_notification_txn_mbox; struct push_notification_txn_msg; -HASH_TABLE_DEFINE_TYPE(push_notification_mq_data, const char *, const char *); - struct push_notification_event_vfuncs_init { /* Return the default config for an event (or NULL if config is * required). */ From dovecot at dovecot.org Wed Nov 11 11:16:06 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 11 Nov 2015 11:16:06 +0000 Subject: dovecot-2.2: auth: Added hardcoded 5 second timeout to LDAP conn... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ca91d540fd87 changeset: 19350:ca91d540fd87 user: Timo Sirainen date: Wed Nov 11 13:15:30 2015 +0200 description: auth: Added hardcoded 5 second timeout to LDAP connect() Although it doesn't look like the timeout is exactly 5 seconds always due to OpenSSL's internal workings, but this should be good enough. diffstat: src/auth/db-ldap.c | 12 ++++++++++++ src/auth/db-ldap.h | 2 ++ 2 files changed, 14 insertions(+), 0 deletions(-) diffs (34 lines): diff -r 0b1c73b01a5a -r ca91d540fd87 src/auth/db-ldap.c --- a/src/auth/db-ldap.c Tue Nov 10 10:00:11 2015 +0200 +++ b/src/auth/db-ldap.c Wed Nov 11 13:15:30 2015 +0200 @@ -1096,6 +1096,18 @@ unsigned int ldap_version; int value; +#ifdef LDAP_OPT_NETWORK_TIMEOUT + struct timeval tv; + int ret; + + tv.tv_sec = DB_LDAP_CONNECT_TIMEOUT_SECS; tv.tv_usec = 0; + ret = ldap_set_option(conn->ld, LDAP_OPT_NETWORK_TIMEOUT, &tv); + if (ret != LDAP_SUCCESS) { + i_fatal("LDAP %s: Can't set network-timeout: %s", + conn->config_path, ldap_err2string(ret)); + } +#endif + db_ldap_set_opt(conn, conn->ld, LDAP_OPT_DEREF, &conn->set.ldap_deref, "deref", conn->set.deref); #ifdef LDAP_OPT_DEBUG_LEVEL diff -r 0b1c73b01a5a -r ca91d540fd87 src/auth/db-ldap.h --- a/src/auth/db-ldap.h Tue Nov 10 10:00:11 2015 +0200 +++ b/src/auth/db-ldap.h Wed Nov 11 13:15:30 2015 +0200 @@ -7,6 +7,8 @@ /* Maximum number of pending requests before delaying new requests. */ #define DB_LDAP_MAX_PENDING_REQUESTS 8 +/* connect() timeout to LDAP */ +#define DB_LDAP_CONNECT_TIMEOUT_SECS 5 /* If LDAP connection is down, fail requests after waiting for this long. */ #define DB_LDAP_REQUEST_DISCONNECT_TIMEOUT_SECS 4 /* If request is still in queue after this many seconds and other requests From dovecot at dovecot.org Wed Nov 11 15:35:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 11 Nov 2015 15:35:43 +0000 Subject: dovecot-2.2: login proxy: Include some extra information in disc... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9dd14c376418 changeset: 19351:9dd14c376418 user: Timo Sirainen date: Wed Nov 11 17:35:10 2015 +0200 description: login proxy: Include some extra information in disconnect log lines. diffstat: src/login-common/login-proxy.c | 28 +++++++++++++++++++++------- 1 files changed, 21 insertions(+), 7 deletions(-) diffs (43 lines): diff -r ca91d540fd87 -r 9dd14c376418 src/login-common/login-proxy.c --- a/src/login-common/login-proxy.c Wed Nov 11 13:15:30 2015 +0200 +++ b/src/login-common/login-proxy.c Wed Nov 11 17:35:10 2015 +0200 @@ -79,18 +79,32 @@ login_proxy_free_delayed(struct login_proxy **_proxy, const char *reason) ATTR_NULL(2); -static void login_proxy_free_errno(struct login_proxy **proxy, +static void login_proxy_free_errno(struct login_proxy **_proxy, int err, bool server) { - const char *reason, *who = server ? "server" : "client"; + struct login_proxy *proxy = *_proxy; + string_t *reason = t_str_new(128); - reason = err == 0 || err == EPIPE ? - t_strdup_printf("Disconnected by %s", who) : - t_strdup_printf("Disconnected by %s: %s", who, strerror(errno)); + str_printfa(reason, "Disconnected by %s", server ? "server" : "client"); + if (err != 0 && err != EPIPE) + str_printfa(reason, ": %s", strerror(errno)); + + str_printfa(reason, "(%ds idle, in=%"PRIuUOFF_T", out=%"PRIuUOFF_T, + (int)(ioloop_time - proxy->last_io), + proxy->server_output->offset, proxy->client_output->offset); + if (o_stream_get_buffer_used_size(proxy->client_output) > 0) { + str_printfa(reason, "+%"PRIuSIZE_T, + o_stream_get_buffer_used_size(proxy->client_output)); + } + if (proxy->server_io == NULL) + str_append(reason, ", client output blocked"); + if (proxy->client_io == NULL) + str_append(reason, ", server output blocked"); + str_append_c(reason, ')'); if (server) - login_proxy_free_delayed(proxy, reason); + login_proxy_free_delayed(_proxy, str_c(reason)); else - login_proxy_free_reason(proxy, reason); + login_proxy_free_reason(_proxy, str_c(reason)); } static void server_input(struct login_proxy *proxy) From dovecot at dovecot.org Wed Nov 11 15:40:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 11 Nov 2015 15:40:56 +0000 Subject: dovecot-2.2: login proxy: If writing to ostream fails, log the o... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e7f71caad9c5 changeset: 19352:e7f71caad9c5 user: Timo Sirainen date: Wed Nov 11 17:40:23 2015 +0200 description: login proxy: If writing to ostream fails, log the ostream's error string. Although for now it's always just based on the errno anyway. diffstat: src/login-common/login-proxy.c | 46 +++++++++++++++++++++++++++-------------- 1 files changed, 30 insertions(+), 16 deletions(-) diffs (96 lines): diff -r 9dd14c376418 -r e7f71caad9c5 src/login-common/login-proxy.c --- a/src/login-common/login-proxy.c Wed Nov 11 17:35:10 2015 +0200 +++ b/src/login-common/login-proxy.c Wed Nov 11 17:40:23 2015 +0200 @@ -79,15 +79,15 @@ login_proxy_free_delayed(struct login_proxy **_proxy, const char *reason) ATTR_NULL(2); -static void login_proxy_free_errno(struct login_proxy **_proxy, - int err, bool server) +static void login_proxy_free_errstr(struct login_proxy **_proxy, + const char *errstr, bool server) { struct login_proxy *proxy = *_proxy; string_t *reason = t_str_new(128); str_printfa(reason, "Disconnected by %s", server ? "server" : "client"); - if (err != 0 && err != EPIPE) - str_printfa(reason, ": %s", strerror(errno)); + if (errstr[0] != '\0') + str_printfa(reason, ": %s", errstr); str_printfa(reason, "(%ds idle, in=%"PRIuUOFF_T", out=%"PRIuUOFF_T, (int)(ioloop_time - proxy->last_io), @@ -107,6 +107,26 @@ login_proxy_free_reason(_proxy, str_c(reason)); } +static void login_proxy_free_errno(struct login_proxy **_proxy, + int err, bool server) +{ + const char *errstr; + + errstr = err == 0 || err == EPIPE ? "" : strerror(err); + login_proxy_free_errstr(_proxy, errstr, server); +} + +static void login_proxy_free_ostream(struct login_proxy **_proxy, + struct ostream *output, bool server) +{ + const char *errstr; + + errstr = output->stream_errno == 0 || + output->stream_errno == EPIPE ? "" : + o_stream_get_error(output); + login_proxy_free_errstr(_proxy, errstr, server); +} + static void server_input(struct login_proxy *proxy) { unsigned char buf[OUTBUF_THRESHOLD]; @@ -129,10 +149,8 @@ o_stream_cork(proxy->client_output); ret2 = o_stream_send(proxy->client_output, buf, ret); o_stream_uncork(proxy->client_output); - if (ret2 != ret) { - login_proxy_free_errno(&proxy, - proxy->client_output->stream_errno, FALSE); - } + if (ret2 != ret) + login_proxy_free_ostream(&proxy, proxy->client_output, FALSE); } static void proxy_client_input(struct login_proxy *proxy) @@ -157,10 +175,8 @@ o_stream_cork(proxy->server_output); ret2 = o_stream_send(proxy->server_output, buf, ret); o_stream_uncork(proxy->server_output); - if (ret2 != ret) { - login_proxy_free_errno(&proxy, - proxy->server_output->stream_errno, TRUE); - } + if (ret2 != ret) + login_proxy_free_ostream(&proxy, proxy->server_output, TRUE); } static void proxy_client_disconnected_input(struct login_proxy *proxy) @@ -177,8 +193,7 @@ { proxy->last_io = ioloop_time; if (o_stream_flush(proxy->server_output) < 0) { - login_proxy_free_errno(&proxy, - proxy->server_output->stream_errno, TRUE); + login_proxy_free_ostream(&proxy, proxy->server_output, TRUE); return 1; } @@ -197,8 +212,7 @@ { proxy->last_io = ioloop_time; if (o_stream_flush(proxy->client_output) < 0) { - login_proxy_free_errno(&proxy, - proxy->client_output->stream_errno, FALSE); + login_proxy_free_ostream(&proxy, proxy->client_output, FALSE); return 1; } From dovecot at dovecot.org Thu Nov 12 09:30:01 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 12 Nov 2015 09:30:01 +0000 Subject: dovecot-2.2: quota: When recalculating quota, don't try to acces... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/53f4464235b3 changeset: 19353:53f4464235b3 user: Timo Sirainen date: Thu Nov 12 11:29:29 2015 +0200 description: quota: When recalculating quota, don't try to access nonexistent autocreate-mailboxes. There's no need to physically create them, since they're empty anyway. diffstat: src/plugins/quota/quota-count.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r e7f71caad9c5 -r 53f4464235b3 src/plugins/quota/quota-count.c --- a/src/plugins/quota/quota-count.c Wed Nov 11 17:40:23 2015 +0200 +++ b/src/plugins/quota/quota-count.c Thu Nov 12 11:29:29 2015 +0200 @@ -106,7 +106,8 @@ iter->ns = namespaces[iter->ns_idx++]; iter->iter = mailbox_list_iter_init(iter->ns->list, "*", MAILBOX_LIST_ITER_SKIP_ALIASES | - MAILBOX_LIST_ITER_RETURN_NO_FLAGS); + MAILBOX_LIST_ITER_RETURN_NO_FLAGS | + MAILBOX_LIST_ITER_NO_AUTO_BOXES); } while ((info = mailbox_list_iter_next(iter->iter)) != NULL) { if ((info->flags & (MAILBOX_NONEXISTENT | From dovecot at dovecot.org Thu Nov 12 10:18:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 12 Nov 2015 10:18:36 +0000 Subject: dovecot-2.2: imap: When client disconnects during a running comm... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5aba08de8b2f changeset: 19355:5aba08de8b2f user: Timo Sirainen date: Thu Nov 12 12:17:04 2015 +0200 description: imap: When client disconnects during a running command, log the IO input/output wait status. This is mainly useful for debugging to make sure that the hang isn't happening because of a bug (missing io) in Dovecot. diffstat: src/imap/imap-client.c | 16 ++++++++++++++-- 1 files changed, 14 insertions(+), 2 deletions(-) diffs (37 lines): diff -r 2f3b58470752 -r 5aba08de8b2f src/imap/imap-client.c --- a/src/imap/imap-client.c Thu Nov 12 12:14:20 2015 +0200 +++ b/src/imap/imap-client.c Thu Nov 12 12:17:04 2015 +0200 @@ -261,6 +261,8 @@ uint64_t running_usecs = 0, ioloop_wait_usecs; unsigned long long bytes_in = 0, bytes_out = 0; string_t *str; + enum io_condition cond; + const char *cond_str; if (client->command_queue == NULL) return ""; @@ -276,12 +278,22 @@ bytes_out += cmd->bytes_out; } + cond = io_loop_find_fd_conditions(current_ioloop, client->fd_out); + if ((cond & (IO_READ | IO_WRITE)) != 0) + cond_str = "input/output"; + else if ((cond & IO_READ) != 0) + cond_str = "input"; + else if ((cond & IO_WRITE) != 0) + cond_str = "output"; + else + cond_str = "nothing"; + ioloop_wait_usecs = io_loop_get_wait_usecs(current_ioloop); msecs_in_ioloop = (ioloop_wait_usecs - client->command_queue->start_ioloop_wait_usecs + 999) / 1000; - str_printfa(str, " running for %d.%03d + waiting for %d.%03d secs", + str_printfa(str, " running for %d.%03d + waiting %s for %d.%03d secs", (int)((running_usecs+999)/1000 / 1000), - (int)((running_usecs+999)/1000 % 1000), + (int)((running_usecs+999)/1000 % 1000), cond_str, msecs_in_ioloop / 1000, msecs_in_ioloop % 1000); str_printfa(str, ", %llu B in + %llu+%"PRIuSIZE_T" B out)", bytes_in, bytes_out, From dovecot at dovecot.org Thu Nov 12 10:18:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 12 Nov 2015 10:18:35 +0000 Subject: dovecot-2.2: lib: Added io_loop_find_fd_conditions() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2f3b58470752 changeset: 19354:2f3b58470752 user: Timo Sirainen date: Thu Nov 12 12:14:20 2015 +0200 description: lib: Added io_loop_find_fd_conditions() diffstat: src/lib/ioloop.c | 14 ++++++++++++++ src/lib/ioloop.h | 3 +++ src/lib/test-ioloop.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 0 deletions(-) diffs (100 lines): diff -r 53f4464235b3 -r 2f3b58470752 src/lib/ioloop.c --- a/src/lib/ioloop.c Thu Nov 12 11:29:29 2015 +0200 +++ b/src/lib/ioloop.c Thu Nov 12 12:14:20 2015 +0200 @@ -948,3 +948,17 @@ { return ioloop->ioloop_wait_usecs; } + +enum io_condition io_loop_find_fd_conditions(struct ioloop *ioloop, int fd) +{ + enum io_condition conditions = 0; + struct io_file *io; + + i_assert(fd >= 0); + + for (io = ioloop->io_files; io != NULL; io = io->next) { + if (io->fd == fd) + conditions |= io->io.condition; + } + return conditions; +} diff -r 53f4464235b3 -r 2f3b58470752 src/lib/ioloop.h --- a/src/lib/ioloop.h Thu Nov 12 11:29:29 2015 +0200 +++ b/src/lib/ioloop.h Thu Nov 12 12:14:20 2015 +0200 @@ -187,5 +187,8 @@ bool io_loop_have_immediate_timeouts(struct ioloop *ioloop); /* Returns number of microseconds spent on the ioloop waiting itself. */ uint64_t io_loop_get_wait_usecs(struct ioloop *ioloop); +/* Return all io conditions added for the given fd. This needs to scan through + all the file ios in the ioloop. */ +enum io_condition io_loop_find_fd_conditions(struct ioloop *ioloop, int fd); #endif diff -r 53f4464235b3 -r 2f3b58470752 src/lib/test-ioloop.c --- a/src/lib/test-ioloop.c Thu Nov 12 11:29:29 2015 +0200 +++ b/src/lib/test-ioloop.c Thu Nov 12 12:14:20 2015 +0200 @@ -1,6 +1,7 @@ /* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ #include "test-lib.h" +#include "net.h" #include "time-util.h" #include "ioloop.h" @@ -50,7 +51,55 @@ test_end(); } +static void io_callback(void *context ATTR_UNUSED) +{ +} + +static void test_ioloop_find_fd_conditions(void) +{ + struct { + enum io_condition condition; + int fd[2]; + struct io *io; + } tests[] = { + { IO_ERROR, { -1, -1 }, NULL }, + { IO_READ, { -1, -1 }, NULL }, + { IO_WRITE, { -1, -1 }, NULL }, + { IO_READ | IO_WRITE, { -1, -1 }, NULL }, + { IO_READ, { -1, -1 }, NULL } /* read+write as separate ios */ + }; + struct ioloop *ioloop; + struct io *io; + unsigned int i; + + test_begin("ioloop find fd conditions"); + + ioloop = io_loop_create(); + + for (i = 0; i < N_ELEMENTS(tests); i++) { + if (socketpair(AF_UNIX, SOCK_STREAM, 0, tests[i].fd) < 0) + i_fatal("socketpair() failed: %m"); + tests[i].io = io_add(tests[i].fd[0], tests[i].condition, io_callback, NULL); + } + io = io_add(tests[i-1].fd[0], IO_WRITE, io_callback, NULL); + tests[i-1].condition |= IO_WRITE; + + for (i = 0; i < N_ELEMENTS(tests); i++) + test_assert_idx(io_loop_find_fd_conditions(ioloop, tests[i].fd[0]) == tests[i].condition, i); + + io_remove(&io); + for (i = 0; i < N_ELEMENTS(tests); i++) { + io_remove(&tests[i].io); + i_close_fd(&tests[i].fd[0]); + i_close_fd(&tests[i].fd[1]); + } + io_loop_destroy(&ioloop); + + test_end(); +} + void test_ioloop(void) { test_ioloop_timeout(); + test_ioloop_find_fd_conditions(); } From dovecot at dovecot.org Thu Nov 12 10:29:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 12 Nov 2015 10:29:30 +0000 Subject: dovecot-2.2: imap: If IDLE or FETCH notices a disconnection, log... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4c8f7316a659 changeset: 19356:4c8f7316a659 user: Timo Sirainen date: Thu Nov 12 12:28:58 2015 +0200 description: imap: If IDLE or FETCH notices a disconnection, log the running command statistics. diffstat: src/imap/cmd-fetch.c | 2 +- src/imap/cmd-idle.c | 2 +- src/imap/imap-client.c | 24 ++++++++++++++---------- 3 files changed, 16 insertions(+), 12 deletions(-) diffs (77 lines): diff -r 5aba08de8b2f -r 4c8f7316a659 src/imap/cmd-fetch.c --- a/src/imap/cmd-fetch.c Thu Nov 12 12:17:04 2015 +0200 +++ b/src/imap/cmd-fetch.c Thu Nov 12 12:28:58 2015 +0200 @@ -192,7 +192,7 @@ const char *errstr; if (cmd->client->output->closed) { - client_disconnect(cmd->client, "Disconnected"); + client_disconnect(cmd->client, NULL); return TRUE; } diff -r 5aba08de8b2f -r 4c8f7316a659 src/imap/cmd-idle.c --- a/src/imap/cmd-idle.c Thu Nov 12 12:17:04 2015 +0200 +++ b/src/imap/cmd-idle.c Thu Nov 12 12:28:58 2015 +0200 @@ -82,7 +82,7 @@ switch (i_stream_read(client->input)) { case -1: /* disconnected */ - client_disconnect(client, "Disconnected in IDLE"); + client_disconnect(client, NULL); return; case -2: client->input_skip_line = TRUE; diff -r 5aba08de8b2f -r 4c8f7316a659 src/imap/imap-client.c --- a/src/imap/imap-client.c Thu Nov 12 12:17:04 2015 +0200 +++ b/src/imap/imap-client.c Thu Nov 12 12:28:58 2015 +0200 @@ -301,22 +301,28 @@ return str_c(str); } +static void client_log_disconnect(struct client *client, const char *reason) +{ + const char *cmd_status = ""; + + if (reason == NULL) { + reason = io_stream_get_disconnect_reason(client->input, + client->output); + cmd_status = client_get_commands_status(client); + } + i_info("%s%s %s", reason, cmd_status, client_stats(client)); +} + static void client_default_destroy(struct client *client, const char *reason) { struct client_command_context *cmd; - const char *cmd_status = ""; i_assert(!client->destroyed); client->destroyed = TRUE; if (!client->disconnected) { client->disconnected = TRUE; - if (reason == NULL) { - reason = io_stream_get_disconnect_reason(client->input, - client->output); - cmd_status = client_get_commands_status(client); - } - i_info("%s%s %s", reason, cmd_status, client_stats(client)); + client_log_disconnect(client, reason); } i_stream_close(client->input); @@ -391,12 +397,10 @@ void client_disconnect(struct client *client, const char *reason) { - i_assert(reason != NULL); - if (client->disconnected) return; - i_info("Disconnected: %s %s", reason, client_stats(client)); + client_log_disconnect(client, reason); client->disconnected = TRUE; o_stream_nflush(client->output); o_stream_uncork(client->output); From dovecot at dovecot.org Thu Nov 12 16:28:26 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 12 Nov 2015 16:28:26 +0000 Subject: dovecot-2.2: imap: When client disconnects during a running comm... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8dae5dc991ce changeset: 19357:8dae5dc991ce user: Timo Sirainen date: Thu Nov 12 18:27:54 2015 +0200 description: imap: When client disconnects during a running command, log the command wait state. diffstat: src/imap/imap-client.c | 14 ++++++++++++-- 1 files changed, 12 insertions(+), 2 deletions(-) diffs (32 lines): diff -r 4c8f7316a659 -r 8dae5dc991ce src/imap/imap-client.c --- a/src/imap/imap-client.c Thu Nov 12 12:28:58 2015 +0200 +++ b/src/imap/imap-client.c Thu Nov 12 18:27:54 2015 +0200 @@ -33,6 +33,15 @@ struct client *imap_clients = NULL; unsigned int imap_client_count = 0; +static const char *client_command_state_names[CLIENT_COMMAND_STATE_DONE+1] = { + "wait-input", + "wait-output", + "wait-external", + "wait-unambiguity", + "wait-sync", + "done" +}; + static void client_idle_timeout(struct client *client) { if (client->output_cmd_lock == NULL) @@ -295,9 +304,10 @@ (int)((running_usecs+999)/1000 / 1000), (int)((running_usecs+999)/1000 % 1000), cond_str, msecs_in_ioloop / 1000, msecs_in_ioloop % 1000); - str_printfa(str, ", %llu B in + %llu+%"PRIuSIZE_T" B out)", + str_printfa(str, ", %llu B in + %llu+%"PRIuSIZE_T" B out, state=%s)", bytes_in, bytes_out, - o_stream_get_buffer_used_size(client->output)); + o_stream_get_buffer_used_size(client->output), + client_command_state_names[client->command_queue->state]); return str_c(str); } From pigeonhole at rename-it.nl Fri Nov 13 23:30:39 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 14 Nov 2015 00:30:39 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Moved message body parsing co... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/d1e5a06fc9d7 changeset: 2130:d1e5a06fc9d7 user: Stephan Bosch date: Sat Nov 14 00:30:30 2015 +0100 description: lib-sieve: Moved message body parsing code from body extension to Sieve core. This makes this available for other extensions. diffstat: src/lib-sieve/plugins/body/ext-body-common.c | 582 +-------------------------- src/lib-sieve/sieve-message.c | 568 ++++++++++++++++++++++++++- src/lib-sieve/sieve-message.h | 23 +- 3 files changed, 594 insertions(+), 579 deletions(-) diffs (truncated from 1277 to 300 lines): diff -r 334f0ba2fd9b -r d1e5a06fc9d7 src/lib-sieve/plugins/body/ext-body-common.c --- a/src/lib-sieve/plugins/body/ext-body-common.c Mon Nov 02 18:56:39 2015 +0100 +++ b/src/lib-sieve/plugins/body/ext-body-common.c Sat Nov 14 00:30:30 2015 +0100 @@ -7,11 +7,6 @@ #include "array.h" #include "str.h" #include "istream.h" -#include "rfc822-parser.h" -#include "message-date.h" -#include "message-parser.h" -#include "message-decoder.h" -#include "mail-html2text.h" #include "mail-storage.h" #include "sieve-common.h" @@ -22,571 +17,6 @@ #include "ext-body-common.h" -struct ext_body_part { - const char *content; - unsigned long size; -}; - -struct ext_body_part_cached { - const char *content_type; - - const char *decoded_body; - const char *text_body; - size_t decoded_body_size; - size_t text_body_size; - - bool have_body; /* there's the empty end-of-headers line */ -}; - -struct ext_body_message_context { - pool_t pool; - ARRAY(struct ext_body_part_cached) cached_body_parts; - ARRAY(struct ext_body_part) return_body_parts; - buffer_t *tmp_buffer; - buffer_t *raw_body; -}; - -static bool _is_wanted_content_type -(const char * const *wanted_types, const char *content_type) -{ - const char *subtype = strchr(content_type, '/'); - size_t type_len; - - type_len = ( subtype == NULL ? strlen(content_type) : - (size_t)(subtype - content_type) ); - - i_assert( wanted_types != NULL ); - - for (; *wanted_types != NULL; wanted_types++) { - const char *wanted_subtype; - - if (**wanted_types == '\0') { - /* empty string matches everything */ - return TRUE; - } - - wanted_subtype = strchr(*wanted_types, '/'); - if (wanted_subtype == NULL) { - /* match only main type */ - if (strlen(*wanted_types) == type_len && - strncasecmp(*wanted_types, content_type, type_len) == 0) - return TRUE; - } else { - /* match whole type/subtype */ - if (strcasecmp(*wanted_types, content_type) == 0) - return TRUE; - } - } - return FALSE; -} - -static bool _want_multipart_content_type -(const char * const *wanted_types) -{ - for (; *wanted_types != NULL; wanted_types++) { - if (**wanted_types == '\0') { - /* empty string matches everything */ - return TRUE; - } - - /* match only main type */ - if ( strncasecmp(*wanted_types, "multipart", 9) == 0 && - ( strlen(*wanted_types) == 9 || *(*wanted_types+9) == '/' ) ) - return TRUE; - } - - return FALSE; -} - - -static bool ext_body_get_return_parts -(struct ext_body_message_context *ctx, const char * const *wanted_types, - bool extract_text) -{ - const struct ext_body_part_cached *body_parts; - unsigned int i, count; - struct ext_body_part *return_part; - - /* Check whether any body parts are cached already */ - body_parts = array_get(&ctx->cached_body_parts, &count); - if ( count == 0 ) - return FALSE; - - /* Clear result array */ - array_clear(&ctx->return_body_parts); - - /* Fill result array with requested content_types */ - for (i = 0; i < count; i++) { - if (!body_parts[i].have_body) { - /* Part has no body; according to RFC this MUST not match to anything and - * therefore it is not included in the result. - */ - continue; - } - - /* Skip content types that are not requested */ - if (!_is_wanted_content_type(wanted_types, body_parts[i].content_type)) - continue; - - /* Add new item to the result */ - return_part = array_append_space(&ctx->return_body_parts); - - /* Depending on whether a decoded body part is requested, the appropriate - * cache item is read. If it is missing, this function fails and the cache - * needs to be completed by ext_body_parts_add_missing(). - */ - if (extract_text) { - if (body_parts[i].text_body == NULL) - return FALSE; - return_part->content = body_parts[i].text_body; - return_part->size = body_parts[i].text_body_size; - } else { - if (body_parts[i].decoded_body == NULL) - return FALSE; - return_part->content = body_parts[i].decoded_body; - return_part->size = body_parts[i].decoded_body_size; - } - } - - return TRUE; -} - -static void ext_body_part_save -(struct ext_body_message_context *ctx, - struct ext_body_part_cached *body_part, bool extract_text) -{ - buffer_t *buf = ctx->tmp_buffer; - buffer_t *text_buf = NULL; - char *part_data; - size_t part_size; - - /* Add terminating NUL to the body part buffer */ - buffer_append_c(buf, '\0'); - - if ( extract_text ) { - if ( mail_html2text_content_type_match - (body_part->content_type) ) { - struct mail_html2text *html2text; - - text_buf = buffer_create_dynamic(default_pool, 4096); - - /* Remove HTML markup */ - html2text = mail_html2text_init(0); - mail_html2text_more(html2text, buf->data, buf->used, text_buf); - mail_html2text_deinit(&html2text); - - buf = text_buf; - } - } - - part_data = p_malloc(ctx->pool, buf->used); - memcpy(part_data, buf->data, buf->used); - part_size = buf->used - 1; - - if ( text_buf != NULL) - buffer_free(&text_buf); - - /* Depending on whether the part is processed into text, store message - * body in the appropriate cache location. - */ - if ( !extract_text ) { - body_part->decoded_body = part_data; - body_part->decoded_body_size = part_size; - } else { - body_part->text_body = part_data; - body_part->text_body_size = part_size; - } - - /* Clear buffer */ - buffer_set_used_size(ctx->tmp_buffer, 0); -} - -static const char *_parse_content_type(const struct message_header_line *hdr) -{ - struct rfc822_parser_context parser; - string_t *content_type; - - /* Initialize parsing */ - rfc822_parser_init(&parser, hdr->full_value, hdr->full_value_len, NULL); - (void)rfc822_skip_lwsp(&parser); - - /* Parse content type */ - content_type = t_str_new(64); - if (rfc822_parse_content_type(&parser, content_type) < 0) - return ""; - - /* Content-type value must end here, otherwise it is invalid after all */ - (void)rfc822_skip_lwsp(&parser); - if ( parser.data != parser.end && *parser.data != ';' ) - return ""; - - /* Success */ - return str_c(content_type); -} - -/* ext_body_parts_add_missing(): - * Add requested message body parts to the cache that are missing. - */ -static int ext_body_parts_add_missing -(const struct sieve_runtime_env *renv, - struct ext_body_message_context *ctx, - const char *const *content_types, bool extract_text) -{ - buffer_t *buf = ctx->tmp_buffer; - struct mail *mail = sieve_message_get_mail(renv->msgctx); - struct ext_body_part_cached *body_part = NULL, *header_part = NULL; - struct message_parser_ctx *parser; - struct message_decoder_context *decoder; - struct message_block block, decoded; - struct message_part *parts, *prev_part = NULL; - ARRAY(struct message_part *) part_index; - struct istream *input; - unsigned int idx = 0; - bool save_body = FALSE, want_multipart, have_all; - int ret; - - /* First check whether any are missing */ - if (ext_body_get_return_parts(ctx, content_types, extract_text)) { - /* Cache hit; all are present */ - return SIEVE_EXEC_OK; - } - - /* Get the message stream */ - if ( mail_get_stream(mail, NULL, NULL, &input) < 0 ) { - return sieve_runtime_mail_error(renv, mail, - "body test: failed to read input message"); - } - if (mail_get_parts(mail, &parts) < 0) { - return sieve_runtime_mail_error(renv, mail, - "body test: failed to parse input message"); - } - - if ( (want_multipart=_want_multipart_content_type(content_types)) ) { - t_array_init(&part_index, 8); - } - - buffer_set_used_size(buf, 0); - - /* Initialize body decoder */ - decoder = message_decoder_init(NULL, 0); - - //parser = message_parser_init_from_parts(parts, input, 0, - //MESSAGE_PARSER_FLAG_INCLUDE_MULTIPART_BLOCKS); - parser = message_parser_init(ctx->pool, input, 0, - MESSAGE_PARSER_FLAG_INCLUDE_MULTIPART_BLOCKS); - while ( (ret = message_parser_parse_next_block(parser, &block)) > 0 ) { - - if ( block.part != prev_part ) { - bool message_rfc822 = FALSE; - - /* Save previous body part */ - if ( body_part != NULL ) { - /* Treat message/rfc822 separately; headers become content */ - if ( block.part->parent == prev_part && - strcmp(body_part->content_type, "message/rfc822") == 0 ) { - message_rfc822 = TRUE; - } else { - if ( save_body ) { - ext_body_part_save(ctx, body_part, extract_text); - } - } - } - - /* Start processing next */ - body_part = array_idx_modifiable(&ctx->cached_body_parts, idx); - body_part->content_type = "text/plain"; - - /* Check whether this is the epilogue block of a wanted multipart part */ - if ( want_multipart ) { - array_idx_set(&part_index, idx, &block.part); - - if ( prev_part != NULL && prev_part->next != block.part && - block.part->parent != prev_part ) { - struct message_part *const *iparts; From pigeonhole at rename-it.nl Fri Nov 13 23:36:34 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 14 Nov 2015 00:36:34 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Make message the time the mes... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/e26cb6c0bf07 changeset: 2131:e26cb6c0bf07 user: Stephan Bosch date: Sat Nov 14 00:36:19 2015 +0100 description: lib-sieve: Make message the time the message processing started available to any extension. diffstat: src/lib-sieve/plugins/date/ext-date-common.c | 6 ++++-- src/lib-sieve/sieve-message.c | 10 ++++++++++ src/lib-sieve/sieve-message.h | 2 ++ 3 files changed, 16 insertions(+), 2 deletions(-) diffs (66 lines): diff -r d1e5a06fc9d7 -r e26cb6c0bf07 src/lib-sieve/plugins/date/ext-date-common.c --- a/src/lib-sieve/plugins/date/ext-date-common.c Sat Nov 14 00:30:30 2015 +0100 +++ b/src/lib-sieve/plugins/date/ext-date-common.c Sat Nov 14 00:36:19 2015 +0100 @@ -33,12 +33,14 @@ { struct ext_date_context *dctx; pool_t pool; + struct timeval msg_time; + time_t current_date; struct tm *tm; - time_t current_date; int zone_offset; /* Get current time at instance main script is started */ - time(¤t_date); + sieve_message_context_time(renv->msgctx, &msg_time); + current_date = msg_time.tv_sec; tm = localtime(¤t_date); zone_offset = utc_offset(tm, current_date); diff -r d1e5a06fc9d7 -r e26cb6c0bf07 src/lib-sieve/sieve-message.c --- a/src/lib-sieve/sieve-message.c Sat Nov 14 00:30:30 2015 +0100 +++ b/src/lib-sieve/sieve-message.c Sat Nov 14 00:36:19 2015 +0100 @@ -76,6 +76,7 @@ int refcount; struct sieve_instance *svinst; + struct timeval time; struct mail_user *mail_user; const struct sieve_message_data *msgdata; @@ -163,6 +164,9 @@ msgctx->mail_user = mail_user; msgctx->msgdata = msgdata; + if (gettimeofday(&msgctx->time, NULL) < 0) + i_fatal("gettimeofday(): %m"); + sieve_message_context_reset(msgctx); return msgctx; @@ -247,6 +251,12 @@ return msgctx->context_pool; } +void sieve_message_context_time(struct sieve_message_context *msgctx, + struct timeval *time) +{ + *time = msgctx->time; +} + /* Extension support */ void sieve_message_context_extension_set diff -r d1e5a06fc9d7 -r e26cb6c0bf07 src/lib-sieve/sieve-message.h --- a/src/lib-sieve/sieve-message.h Sat Nov 14 00:30:30 2015 +0100 +++ b/src/lib-sieve/sieve-message.h Sat Nov 14 00:36:19 2015 +0100 @@ -29,6 +29,8 @@ pool_t sieve_message_context_pool (struct sieve_message_context *msgctx) ATTR_PURE; +void sieve_message_context_time(struct sieve_message_context *msgctx, + struct timeval *time); /* Extension support */ From pigeonhole at rename-it.nl Sun Nov 15 21:46:07 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 15 Nov 2015 22:46:07 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Made message part content typ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/f2c5a9f92e0d changeset: 2132:f2c5a9f92e0d user: Stephan Bosch date: Sun Nov 15 22:46:01 2015 +0100 description: lib-sieve: Made message part content type and disposition available to extensions. diffstat: src/lib-sieve/sieve-message.c | 42 +++++++++++++++++++++++++++++++++++++++--- src/lib-sieve/sieve-message.h | 3 +++ 2 files changed, 42 insertions(+), 3 deletions(-) diffs (100 lines): diff -r e26cb6c0bf07 -r f2c5a9f92e0d src/lib-sieve/sieve-message.c --- a/src/lib-sieve/sieve-message.c Sat Nov 14 00:36:19 2015 +0100 +++ b/src/lib-sieve/sieve-message.c Sun Nov 15 22:46:01 2015 +0100 @@ -61,6 +61,7 @@ struct sieve_message_body_part_cached { const char *content_type; + const char *content_disposition; const char *decoded_body; const char *text_body; @@ -892,6 +893,8 @@ /* Add new item to the result */ return_part = array_append_space(&msgctx->return_body_parts); + return_part->content_type = body_parts[i].content_type; + return_part->content_disposition = body_parts[i].content_disposition; /* Depending on whether a decoded body part is requested, the appropriate * cache item is read. If it is missing, this function fails and the cache @@ -990,6 +993,30 @@ return str_c(content_type); } +static const char * +_parse_content_disposition(const struct message_header_line *hdr) +{ + struct rfc822_parser_context parser; + string_t *content_disp; + + /* Initialize parsing */ + rfc822_parser_init(&parser, hdr->full_value, hdr->full_value_len, NULL); + (void)rfc822_skip_lwsp(&parser); + + /* Parse content type */ + content_disp = t_str_new(64); + if (rfc822_parse_mime_token(&parser, content_disp) < 0) + return ""; + + /* Content-type value must end here, otherwise it is invalid after all */ + (void)rfc822_skip_lwsp(&parser); + if ( parser.data != parser.end && *parser.data != ';' ) + return ""; + + /* Success */ + return str_c(content_disp); +} + /* sieve_message_body_parts_add_missing(): * Add requested message body parts to the cache that are missing. */ @@ -1107,6 +1134,8 @@ } if ( block.hdr != NULL || block.size == 0 ) { + bool is_ctype = FALSE; + /* Reading headers */ /* Decode block */ @@ -1150,7 +1179,9 @@ } /* We're interested in only the Content-Type: header */ - if ( strcasecmp(block.hdr->name, "Content-Type" ) != 0 ) + if ( strcasecmp(block.hdr->name, "Content-Type" ) == 0 ) + is_ctype = TRUE; + else if ( strcasecmp(block.hdr->name, "Content-Disposition" ) != 0 ) continue; /* Header can have folding whitespace. Acquire the full value before @@ -1165,8 +1196,13 @@ /* Parse the content type from the Content-type header */ T_BEGIN { - body_part->content_type = - p_strdup(pool, _parse_content_type(block.hdr)); + if ( is_ctype ) { + body_part->content_type = + p_strdup(pool, _parse_content_type(block.hdr)); + } else { + body_part->content_disposition = + p_strdup(pool, _parse_content_disposition(block.hdr)); + } } T_END; continue; diff -r e26cb6c0bf07 -r f2c5a9f92e0d src/lib-sieve/sieve-message.h --- a/src/lib-sieve/sieve-message.h Sat Nov 14 00:36:19 2015 +0100 +++ b/src/lib-sieve/sieve-message.h Sun Nov 15 22:46:01 2015 +0100 @@ -173,6 +173,9 @@ */ struct sieve_message_body_part { + const char *content_type; + const char *content_disposition; + const char *content; unsigned long size; }; From dovecot at dovecot.org Mon Nov 16 10:25:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Nov 2015 10:25:22 +0000 Subject: dovecot-2.2: imapc: Treat literal8 input the same as regular lit... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c0229e018a24 changeset: 19358:c0229e018a24 user: Timo Sirainen date: Mon Nov 16 12:24:53 2015 +0200 description: imapc: Treat literal8 input the same as regular literal. This is mainly for migrating away from broken servers (a patched Cyrus) that send literal8 if a (corrupted) message contains NULs. diffstat: src/lib-imap-client/imapc-connection.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 8dae5dc991ce -r c0229e018a24 src/lib-imap-client/imapc-connection.c --- a/src/lib-imap-client/imapc-connection.c Thu Nov 12 18:27:54 2015 +0200 +++ b/src/lib-imap-client/imapc-connection.c Mon Nov 16 12:24:53 2015 +0200 @@ -548,6 +548,7 @@ ret = imap_parser_read_args(conn->parser, 0, IMAP_PARSE_FLAG_LITERAL_SIZE | IMAP_PARSE_FLAG_ATOM_ALLCHARS | + IMAP_PARSE_FLAG_LITERAL8 | IMAP_PARSE_FLAG_SERVER_TEXT, imap_args_r); if (ret == -2) { /* need more data */ From dovecot at dovecot.org Mon Nov 16 10:30:21 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Nov 2015 10:30:21 +0000 Subject: dovecot-2.2: lib-http: ref/unref input stream in http message pa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fe23c9b18c9d changeset: 19359:fe23c9b18c9d user: Phil Carmody date: Mon Nov 16 12:26:06 2015 +0200 description: lib-http: ref/unref input stream in http message parser To prevent assidental disappearance of the stream while it's in use. Also, the caller can create and forget - we'll do the cleanup later. Signed-off-by: Phil Carmody diffstat: src/lib-http/http-message-parser.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (20 lines): diff -r c0229e018a24 -r fe23c9b18c9d src/lib-http/http-message-parser.c --- a/src/lib-http/http-message-parser.c Mon Nov 16 12:24:53 2015 +0200 +++ b/src/lib-http/http-message-parser.c Mon Nov 16 12:26:06 2015 +0200 @@ -20,6 +20,7 @@ { memset(parser, 0, sizeof(*parser)); parser->input = input; + i_stream_ref(parser->input); if (hdr_limits != NULL) parser->header_limits = *hdr_limits; parser->max_payload_size = max_payload_size; @@ -34,6 +35,8 @@ pool_unref(&parser->msg.pool); if (parser->payload != NULL) i_stream_unref(&parser->payload); + if (parser->input != NULL) + i_stream_unref(&parser->input); } void http_message_parser_restart(struct http_message_parser *parser, From dovecot at dovecot.org Mon Nov 16 10:30:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Nov 2015 10:30:22 +0000 Subject: dovecot-2.2: lib-http: free resources used in the tests Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f049fe9e7799 changeset: 19360:f049fe9e7799 user: Phil Carmody date: Mon Nov 16 12:26:32 2015 +0200 description: lib-http: free resources used in the tests Signed-off-by: Phil Carmody diffstat: src/lib-http/test-http-header-parser.c | 2 ++ src/lib-http/test-http-request-parser.c | 6 ++++++ src/lib-http/test-http-response-parser.c | 5 +++++ src/lib-http/test-http-transfer.c | 4 ++++ 4 files changed, 17 insertions(+), 0 deletions(-) diffs (120 lines): diff -r fe23c9b18c9d -r f049fe9e7799 src/lib-http/test-http-header-parser.c --- a/src/lib-http/test-http-header-parser.c Mon Nov 16 12:26:06 2015 +0200 +++ b/src/lib-http/test-http-header-parser.c Mon Nov 16 12:26:32 2015 +0200 @@ -232,6 +232,7 @@ test_out_reason("parse success", ret > 0, error); test_end(); + i_stream_unref(&input); http_header_parser_deinit(&parser); } T_END; } @@ -359,6 +360,7 @@ test_out_reason("parse failure", ret < 0, error); test_end(); + i_stream_unref(&input); http_header_parser_deinit(&parser); } T_END; } diff -r fe23c9b18c9d -r f049fe9e7799 src/lib-http/test-http-request-parser.c --- a/src/lib-http/test-http-request-parser.c Mon Nov 16 12:26:06 2015 +0200 +++ b/src/lib-http/test-http-request-parser.c Mon Nov 16 12:26:32 2015 +0200 @@ -188,6 +188,8 @@ (parser, NULL, &request, &error_code, &error); } test_istream_set_size(input, request_text_len); + i_stream_unref(&input); + while (ret > 0) { if (request.payload != NULL) { buffer_set_used_size(payload_buffer, 0); @@ -402,6 +404,7 @@ request_text = test->request; input = i_stream_create_from_data(request_text, strlen(request_text)); parser = http_request_parser_init(input, NULL); + i_stream_unref(&input); test_begin(t_strdup_printf("http request invalid [%d]", i)); @@ -423,9 +426,12 @@ input = i_stream_create_from_data(invalid_request_with_nuls, sizeof(invalid_request_with_nuls)-1); parser = http_request_parser_init(input, NULL); + i_stream_unref(&input); + while ((ret=http_request_parse_next (parser, NULL, &request, &error_code, &error)) > 0); test_assert(ret < 0); + http_request_parser_deinit(&parser); test_end(); } diff -r fe23c9b18c9d -r f049fe9e7799 src/lib-http/test-http-response-parser.c --- a/src/lib-http/test-http-response-parser.c Mon Nov 16 12:26:06 2015 +0200 +++ b/src/lib-http/test-http-response-parser.c Mon Nov 16 12:26:32 2015 +0200 @@ -119,6 +119,8 @@ ret = http_response_parse_next(parser, FALSE, &response, &error); } test_istream_set_size(input, response_text_len); + i_stream_unref(&input); + while (ret > 0) { if (response.payload != NULL) { buffer_set_used_size(payload_buffer, 0); @@ -197,6 +199,7 @@ response_text = test; input = i_stream_create_from_data(response_text, strlen(response_text)); parser = http_response_parser_init(input, NULL); + i_stream_unref(&input); test_begin(t_strdup_printf("http response invalid [%d]", i)); @@ -231,6 +234,7 @@ input = i_stream_create_from_data(bad_response_with_nuls, sizeof(bad_response_with_nuls)-1); parser = http_response_parser_init(input, NULL); + i_stream_unref(&input); while ((ret=http_response_parse_next(parser, FALSE, &response, &error)) > 0); test_out("parse success", ret == 0); header = http_response_header_get(&response, "server"); @@ -240,6 +244,7 @@ strcmp(header, "textserver") == 0); } test_end(); + http_response_parser_deinit(&parser); } int main(void) diff -r fe23c9b18c9d -r f049fe9e7799 src/lib-http/test-http-transfer.c --- a/src/lib-http/test-http-transfer.c Mon Nov 16 12:26:06 2015 +0200 +++ b/src/lib-http/test-http-transfer.c Mon Nov 16 12:26:32 2015 +0200 @@ -100,12 +100,14 @@ input = i_stream_create_from_data(in, strlen(in)); chunked = http_transfer_chunked_istream_create(input, 0); + i_stream_unref(&input); buffer_set_used_size(payload_buffer, 0); output = o_stream_create_buffer(payload_buffer); test_out("payload read", o_stream_send_istream(output, chunked) > 0 && chunked->stream_errno == 0); o_stream_destroy(&output); + i_stream_unref(&chunked); stream_out = str_c(payload_buffer); test_out(t_strdup_printf("response->payload = %s", @@ -194,11 +196,13 @@ input = i_stream_create_from_data(in, strlen(in)); chunked = http_transfer_chunked_istream_create(input, 0); + i_stream_unref(&input); buffer_set_used_size(payload_buffer, 0); output = o_stream_create_buffer(payload_buffer); (void)o_stream_send_istream(output, chunked); test_out("payload read failure", chunked->stream_errno != 0); + i_stream_unref(&chunked); o_stream_destroy(&output); test_end(); From dovecot at dovecot.org Mon Nov 16 10:30:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Nov 2015 10:30:22 +0000 Subject: dovecot-2.2: lib-master: stop tests from leaking memory Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/80a47f533fbf changeset: 19361:80a47f533fbf user: Phil Carmody date: Mon Nov 16 12:26:50 2015 +0200 description: lib-master: stop tests from leaking memory This permits Valgrind to run without complaint. Signed-off-by: Phil Carmody diffstat: src/lib-master/test-master-service-settings-cache.c | 11 ++++++++--- 1 files changed, 8 insertions(+), 3 deletions(-) diffs (27 lines): diff -r f049fe9e7799 -r 80a47f533fbf src/lib-master/test-master-service-settings-cache.c --- a/src/lib-master/test-master-service-settings-cache.c Mon Nov 16 12:26:32 2015 +0200 +++ b/src/lib-master/test-master-service-settings-cache.c Mon Nov 16 12:26:50 2015 +0200 @@ -98,15 +98,20 @@ test_master_service_settings_cache, NULL }; + pool_t pool; + int ret; memset(&input, 0, sizeof(input)); input.module = "module"; input.service = "service_name"; set.config_cache_size = 1024*4; + pool = pool_alloconly_create("set pool", 1024); test_master_service.set_parser = - settings_parser_init(pool_alloconly_create("set pool", 1024), - &test_setting_parser_info, 0); + settings_parser_init(pool, &test_setting_parser_info, 0); master_service = &test_master_service; - return test_run(test_functions); + ret = test_run(test_functions); + settings_parser_deinit(&test_master_service.set_parser); + pool_unref(&pool); + return ret; } From dovecot at dovecot.org Mon Nov 16 10:30:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Nov 2015 10:30:23 +0000 Subject: dovecot-2.2: lib: fix memory leaks in failure-at streams Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/28f088e237d9 changeset: 19362:28f088e237d9 user: Phil Carmody date: Mon Nov 16 12:27:03 2015 +0200 description: lib: fix memory leaks in failure-at streams The parent wasn't getting unref'ed, and would hang around for ever. Signed-off-by: Phil Carmody diffstat: src/lib/istream-failure-at.c | 1 + src/lib/ostream-failure-at.c | 1 + 2 files changed, 2 insertions(+), 0 deletions(-) diffs (22 lines): diff -r 80a47f533fbf -r 28f088e237d9 src/lib/istream-failure-at.c --- a/src/lib/istream-failure-at.c Mon Nov 16 12:26:50 2015 +0200 +++ b/src/lib/istream-failure-at.c Mon Nov 16 12:27:03 2015 +0200 @@ -16,6 +16,7 @@ (struct failure_at_istream *)stream; i_free(fstream->error_string); + i_stream_unref(&fstream->istream.parent); } static ssize_t diff -r 80a47f533fbf -r 28f088e237d9 src/lib/ostream-failure-at.c --- a/src/lib/ostream-failure-at.c Mon Nov 16 12:26:50 2015 +0200 +++ b/src/lib/ostream-failure-at.c Mon Nov 16 12:27:03 2015 +0200 @@ -18,6 +18,7 @@ (struct failure_at_ostream *)stream; i_free(fstream->error_string); + o_stream_unref(&fstream->ostream.parent); } static ssize_t From dovecot at dovecot.org Mon Nov 16 10:30:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Nov 2015 10:30:23 +0000 Subject: dovecot-2.2: lib: make test-istream-failure-at not leak memory Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/416399bf1643 changeset: 19363:416399bf1643 user: Phil Carmody date: Mon Nov 16 12:29:09 2015 +0200 description: lib: make test-istream-failure-at not leak memory This permits Valgrind to run without complaint. Signed-off-by: Phil Carmody diffstat: src/lib/test-istream-failure-at.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (19 lines): diff -r 28f088e237d9 -r 416399bf1643 src/lib/test-istream-failure-at.c --- a/src/lib/test-istream-failure-at.c Mon Nov 16 12:27:03 2015 +0200 +++ b/src/lib/test-istream-failure-at.c Mon Nov 16 12:29:09 2015 +0200 @@ -34,6 +34,7 @@ while ((ret = i_stream_read(input)) > 0) i_stream_skip(input, ret); test_assert(ret == -1 && input->stream_errno == 0); + i_stream_destroy(&input); /* fail at EOF */ i_stream_seek(data_input, 0); input = i_stream_create_failure_at_eof(data_input, TEST_ERRMSG); @@ -42,5 +43,7 @@ test_assert_idx(ret == -1 && input->v_offset == TEST_DATA_LENGTH && input->stream_errno == EIO && strcmp(i_stream_get_error(input), TEST_ERRMSG) == 0, i); + i_stream_destroy(&input); + i_stream_destroy(&data_input); test_end(); } From dovecot at dovecot.org Mon Nov 16 10:30:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Nov 2015 10:30:23 +0000 Subject: dovecot-2.2: lib: fix memory leak of iterator in str_table_deinit() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ae5be2c19fce changeset: 19364:ae5be2c19fce user: Phil Carmody date: Mon Nov 16 12:29:21 2015 +0200 description: lib: fix memory leak of iterator in str_table_deinit() The deinit function uses an iterator, but never cleared it up. Found by Valgrind. Signed-off-by: Phil Carmody diffstat: src/lib/str-table.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 416399bf1643 -r ae5be2c19fce src/lib/str-table.c --- a/src/lib/str-table.c Mon Nov 16 12:29:09 2015 +0200 +++ b/src/lib/str-table.c Mon Nov 16 12:29:21 2015 +0200 @@ -29,6 +29,7 @@ iter = hash_table_iterate_init(table->hash); while (hash_table_iterate(iter, table->hash, &key, &value)) i_free(key); + hash_table_iterate_deinit(&iter); hash_table_destroy(&table->hash); i_free(table); } From dovecot at dovecot.org Mon Nov 16 10:30:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Nov 2015 10:30:23 +0000 Subject: dovecot-2.2: various - remove 8-bit characters from literal stri... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/36353046b107 changeset: 19365:36353046b107 user: Phil Carmody date: Mon Nov 16 12:29:51 2015 +0200 description: various - remove 8-bit characters from literal strings in test cases C has a portable way of expressing characters not in the basic character set, namely \xNN escaping. Otherwise, the interpretation of the raw utf-8 is implentation dependent. This has the benefit of making some tests' expected output more obvious, such as "=c3=a4" matching "\xC3\xA4", even if it hinders the readability of some natural-language-based tests. Signed-off-by: Phil Carmody diffstat: src/lib-charset/test-charset.c | 8 ++++++-- src/lib-fts/test-fts-filter.c | 16 ++++++++-------- src/lib-fts/test-fts-tokenizer.c | 22 +++++++++++----------- src/lib-imap/test-imap-utf7.c | 5 +++-- src/lib-mail/test-istream-qp-decoder.c | 10 +++++----- src/lib-mail/test-message-header-decode.c | 14 +++++++------- src/lib-mail/test-message-header-encode.c | 12 ++++++------ 7 files changed, 46 insertions(+), 41 deletions(-) diffs (258 lines): diff -r ae5be2c19fce -r 36353046b107 src/lib-charset/test-charset.c --- a/src/lib-charset/test-charset.c Mon Nov 16 12:29:21 2015 +0200 +++ b/src/lib-charset/test-charset.c Mon Nov 16 12:29:51 2015 +0200 @@ -66,9 +66,13 @@ const char *output; enum charset_result result; } tests[] = { - { "ISO-8859-1", "p\xE4\xE4", "p??", CHARSET_RET_OK }, + { "ISO-8859-1", "p\xE4\xE4", "p\xC3\xA4\xC3\xA4", CHARSET_RET_OK }, { "UTF-7", "+AOQA5AD2AOQA9gDkAPYA5AD2AOQA9gDkAPYA5AD2AOQA9gDkAPYA5AD2AOQA9gDkAPYA5AD2AOQA9gDkAPYA5AD2AOQA9gDk", - "????????????????????????????????????", CHARSET_RET_OK } + "\xC3\xA4\xC3\xA4\xC3\xB6\xC3\xA4\xC3\xB6\xC3\xA4\xC3\xB6\xC3\xA4" + "\xC3\xB6\xC3\xA4\xC3\xB6\xC3\xA4\xC3\xB6\xC3\xA4\xC3\xB6\xC3\xA4" + "\xC3\xB6\xC3\xA4\xC3\xB6\xC3\xA4\xC3\xB6\xC3\xA4\xC3\xB6\xC3\xA4" + "\xC3\xB6\xC3\xA4\xC3\xB6\xC3\xA4\xC3\xB6\xC3\xA4\xC3\xB6\xC3\xA4" + "\xC3\xB6\xC3\xA4\xC3\xB6\xC3\xA4", CHARSET_RET_OK } }; string_t *str = t_str_new(128); struct charset_translation *trans; diff -r ae5be2c19fce -r 36353046b107 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Mon Nov 16 12:29:21 2015 +0200 +++ b/src/lib-fts/test-fts-filter.c Mon Nov 16 12:29:51 2015 +0200 @@ -415,8 +415,8 @@ struct fts_filter *norm = NULL; const char *input[] = { "Vem", - "?", - "???", + "\xC3\x85", + "\xC3\x85\xC3\x84\xC3\x96", "Vem kan segla f\xC3\xB6rutan vind?\n" "\xC3\x85\xC3\x84\xC3\x96\xC3\xB6\xC3\xA4\xC3\xA5" }; @@ -450,8 +450,8 @@ struct fts_filter *norm = NULL; const char *input[] = { "Vem", - "?", - "???", + "\xC3\x85", + "\xC3\x85\xC3\x84\xC3\x96", "Vem kan segla f\xC3\xB6rutan vind?\n" "\xC3\x85\xC3\x84\xC3\x96\xC3\xB6\xC3\xA4\xC3\xA5" }; @@ -662,14 +662,14 @@ "foo'", "foo's", - "foo?'s", + "foo\xC3\xA4's", "foo'S", "foos'S", "foo's's", "foo'ss", "foo\xE2\x80\x99s", - "foo?\xE2\x80\x99s", + "foo\xC3\xA4\xE2\x80\x99s", "foo\xE2\x80\x99S", "foos\xE2\x80\x99S", "foo\xE2\x80\x99s\xE2\x80\x99s", @@ -679,14 +679,14 @@ "foo'", "foo", - "foo?", + "foo\xC3\xA4", "foo", "foos", "foo's", "foo'ss", "foo", - "foo?", + "foo\xC3\xA4", "foo", "foos", "foo\xE2\x80\x99s", diff -r ae5be2c19fce -r 36353046b107 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Mon Nov 16 12:29:21 2015 +0200 +++ b/src/lib-fts/test-fts-tokenizer.c Mon Nov 16 12:29:51 2015 +0200 @@ -20,9 +20,9 @@ "abc at example.com, " "Bar Baz , " "foo at domain " - "1234567890123456789012345678?," - "12345678901234567890123456789?," - "123456789012345678901234567890?," + "1234567890123456789012345678\xC3\xA4," + "12345678901234567890123456789\xC3\xA4," + "123456789012345678901234567890\xC3\xA4," "and longlonglongabcdefghijklmnopqrstuvwxyz more.\n\n " "(\"Hello world\")3.14 3,14 last", @@ -30,8 +30,8 @@ "' ' '' ''' 'quoted text' 'word' 'hlo words' you're bad'''word '''pre post'''", - "'1234567890123456789012345678?," - "123456789012345678901234567x'?," + "'1234567890123456789012345678\xC3\xA4," + "123456789012345678901234567x'\xC3\xA4," "1234567890123456789012345678x're," "1234567890123456789012345678x'," "1234567890123456789012345678x''," @@ -141,7 +141,7 @@ "there", "was", "text", "galor\xC3\xA9", "abc", "example", "com", "Bar", "Baz", "bar", "example", "org", "foo", "domain", - "1234567890123456789012345678?", + "1234567890123456789012345678\xC3\xA4", "12345678901234567890123456789", "123456789012345678901234567890", "and", "longlonglongabcdefghijklmnopqr", @@ -152,7 +152,7 @@ "quoted", "text", "word", "hlo", "words", "you're", "bad", "word", "pre", "post", NULL, - "1234567890123456789012345678?", + "1234567890123456789012345678\xC3\xA4", "123456789012345678901234567x'", "1234567890123456789012345678x'", "1234567890123456789012345678x", @@ -200,7 +200,7 @@ "there", "was", "text", "galor\xC3\xA9", "abc", "example", "com", "Bar", "Baz", "bar", "example", "org", "foo", "domain", - "1234567890123456789012345678?", + "1234567890123456789012345678\xC3\xA4", "12345678901234567890123456789", "123456789012345678901234567890", "and", "longlonglongabcdefghijklmnopqr", @@ -211,7 +211,7 @@ "quoted", "text", "word", "hlo", "words", "you're", "bad", "word", "pre", "post", NULL, - "1234567890123456789012345678?", + "1234567890123456789012345678\xC3\xA4", "123456789012345678901234567x'", "1234567890123456789012345678x'", "1234567890123456789012345678x", @@ -256,7 +256,7 @@ "there", "was", "text", "galor\xC3\xA9", "abc", "example", "com", "Bar", "Baz", "bar", "example", "org", "foo", "domain", - "1234567890123456789012345678?", + "1234567890123456789012345678\xC3\xA4", "12345678901234567890123456789", "123456789012345678901234567890", "and", "longlonglongabcdefghijklmnopqr", @@ -267,7 +267,7 @@ "quoted", "text", "word", "hlo", "words", "you're", "bad", "word", "pre", "post", NULL, - "1234567890123456789012345678?", + "1234567890123456789012345678\xC3\xA4", "123456789012345678901234567x'", "1234567890123456789012345678x'", "1234567890123456789012345678x", diff -r ae5be2c19fce -r 36353046b107 src/lib-imap/test-imap-utf7.c --- a/src/lib-imap/test-imap-utf7.c Mon Nov 16 12:29:21 2015 +0200 +++ b/src/lib-imap/test-imap-utf7.c Mon Nov 16 12:29:51 2015 +0200 @@ -13,8 +13,9 @@ const char *mutf7; } tests[] = { { "&&x&&", "&-&-x&-&-" }, - { "~peter/mail/??/???", "~peter/mail/&U,BTFw-/&ZeVnLIqe-" }, - { "tiet?j?", "tiet&AOQ-j&AOQ-" }, + { "~peter/mail/\xe5\x8f\xb0\xe5\x8c\x97/\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e", + "~peter/mail/&U,BTFw-/&ZeVnLIqe-" }, + { "tiet\xc3\xa4j\xc3\xa4", "tiet&AOQ-j&AOQ-" }, { "p\xe4\xe4", NULL }, { NULL, "&" }, { NULL, "&Jjo" }, diff -r ae5be2c19fce -r 36353046b107 src/lib-mail/test-istream-qp-decoder.c --- a/src/lib-mail/test-istream-qp-decoder.c Mon Nov 16 12:29:21 2015 +0200 +++ b/src/lib-mail/test-istream-qp-decoder.c Mon Nov 16 12:29:51 2015 +0200 @@ -10,15 +10,15 @@ const char *output; int ret; } tests[] = { - { "p=C3=A4=C3=A4t=C3=B6s", "p??t?s", 0 }, - { "p=c3=a4=c3=a4t=c3=b6s= \n", "p??t?s", 0 }, - { "p=c3=a4= \t \n=c3=\r\n=a4t= \r\n=c3=b6s", "p??t?s", 0 }, + { "p=C3=A4=C3=A4t=C3=B6s", "p\xC3\xA4\xC3\xA4t\xC3\xB6s", 0 }, + { "p=c3=a4=c3=a4t=c3=b6s= \n", "p\xC3\xA4\xC3\xA4t\xC3\xB6s", 0 }, + { "p=c3=a4= \t \n=c3=\r\n=a4t= \r\n=c3=b6s", "p\xC3\xA4\xC3\xA4t\xC3\xB6s", 0 }, - { "p=c3=a4\rasdf", "p?", -1 }, + { "p=c3=a4\rasdf", "p\xC3\xA4", -1 }, { "p=c", "p", -1 }, { "p=A", "p", -1 }, { "p=Ax", "p", -1 }, - { "p=c3=a4=c3=a4t=c3=b6s= ", "p??t?s", -1 } + { "p=c3=a4=c3=a4t=c3=b6s= ", "p\xC3\xA4\xC3\xA4t\xC3\xB6s", -1 } }; static void diff -r ae5be2c19fce -r 36353046b107 src/lib-mail/test-message-header-decode.c --- a/src/lib-mail/test-message-header-decode.c Mon Nov 16 12:29:21 2015 +0200 +++ b/src/lib-mail/test-message-header-decode.c Mon Nov 16 12:29:51 2015 +0200 @@ -27,13 +27,13 @@ static void test_message_header_decode(void) { static const char *data[] = { - " \t=?utf-8?q?=c3=a4?= =?utf-8?q?=c3=a4?= b \t\r\n ", "?? b \t\r\n ", - "a =?utf-8?q?=c3=a4?= b", "a ? b", - "a =?utf-8?q?=c3=a4?= b", "a ? b", - "a =?utf-8?q?=c3=a4?=\t\t\r\n =?utf-8?q?=c3=a4?= b", "a ?? b", - "a =?utf-8?q?=c3=a4?= x =?utf-8?q?=c3=a4?= b", "a ? x ? b", - "a =?utf-8?b?w6TDpCDDpA==?= b", "a ?? ? b", - "=?utf-8?b?w6Qgw6Q=?=", "? ?", + " \t=?utf-8?q?=c3=a4?= =?utf-8?q?=c3=a4?= b \t\r\n ", "\xC3\xA4\xC3\xA4 b \t\r\n ", + "a =?utf-8?q?=c3=a4?= b", "a \xC3\xA4 b", + "a =?utf-8?q?=c3=a4?= b", "a \xC3\xA4 b", + "a =?utf-8?q?=c3=a4?=\t\t\r\n =?utf-8?q?=c3=a4?= b", "a \xC3\xA4\xC3\xA4 b", + "a =?utf-8?q?=c3=a4?= x =?utf-8?q?=c3=a4?= b", "a \xC3\xA4 x \xC3\xA4 b", + "a =?utf-8?b?w6TDpCDDpA==?= b", "a \xC3\xA4\xC3\xA4 \xC3\xA4 b", + "=?utf-8?b?w6Qgw6Q=?=", "\xC3\xA4 \xC3\xA4", }; string_t *dest; unsigned int i; diff -r ae5be2c19fce -r 36353046b107 src/lib-mail/test-message-header-encode.c --- a/src/lib-mail/test-message-header-encode.c Mon Nov 16 12:29:21 2015 +0200 +++ b/src/lib-mail/test-message-header-encode.c Mon Nov 16 12:29:51 2015 +0200 @@ -57,7 +57,7 @@ str_append_c(input, 'a'); for (i = 0; i < 40; i++) - str_append(input, "?"); + str_append(input, "\xC3\xA4"); for (i = 0; i < 80; i++) { for (skip = 0; skip < 2; skip++) { str_truncate(str, 0); @@ -144,7 +144,7 @@ str_append_c(input, 'a'); for (i = 0; i < 40; i++) - str_append(input, "?"); + str_append(input, "\xC3\xA4"); for (i = 0; i < 80; i++) { for (skip = 0; skip < 2; skip++) { str_truncate(str, 0); @@ -166,10 +166,10 @@ { const char *data[] = { "a b", "a b", - "a bc?de f", "a =?utf-8?q?bc=C3=A4de?= f", - "a ?? ? b", "a =?utf-8?b?w6TDpCDDpA==?= b", - "? a ?", "=?utf-8?q?=C3=A4_a_=C3=A4?=", - "?? a ?", "=?utf-8?b?w6TDpCBhIMOk?=", + "a bc\xC3\xA4""de f", "a =?utf-8?q?bc=C3=A4de?= f", + "a \xC3\xA4\xC3\xA4 \xC3\xA4 b", "a =?utf-8?b?w6TDpCDDpA==?= b", + "\xC3\xA4 a \xC3\xA4", "=?utf-8?q?=C3=A4_a_=C3=A4?=", + "\xC3\xA4\xC3\xA4 a \xC3\xA4", "=?utf-8?b?w6TDpCBhIMOk?=", "=", "=", "?", "?", "a=?", "a=?", From dovecot at dovecot.org Mon Nov 16 10:40:48 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Nov 2015 10:40:48 +0000 Subject: dovecot-2.2: dovecot.m4: Added --leak-check=full to valgrind Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2a85e0b0dc35 changeset: 19366:2a85e0b0dc35 user: Timo Sirainen date: Mon Nov 16 12:40:22 2015 +0200 description: dovecot.m4: Added --leak-check=full to valgrind diffstat: dovecot.m4 | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (24 lines): diff -r 36353046b107 -r 2a85e0b0dc35 dovecot.m4 --- a/dovecot.m4 Mon Nov 16 12:29:51 2015 +0200 +++ b/dovecot.m4 Mon Nov 16 12:40:22 2015 +0200 @@ -6,7 +6,7 @@ # unlimited permission to copy and/or distribute it, with or without # modifications, as long as this notice is preserved. -# serial 17 +# serial 18 AC_DEFUN([DC_DOVECOT_MODULEDIR],[ AC_ARG_WITH(moduledir, @@ -46,9 +46,9 @@ trap "rm -f test.out.\$\$" 0 1 2 3 15 supp_path="\$top_srcdir/run-test-valgrind.supp" if test -r "\$supp_path"; then - valgrind -q --suppressions="\$supp_path" --log-file=test.out.\$\$ \$[*] + valgrind -q --leak-check=full --suppressions="\$supp_path" --log-file=test.out.\$\$ \$[*] else - valgrind -q --log-file=test.out.\$\$ \$[*] + valgrind -q --leak-check=full --log-file=test.out.\$\$ \$[*] fi ret=\$? if test -s test.out.\$\$; then From dovecot at dovecot.org Mon Nov 16 12:16:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Nov 2015 12:16:28 +0000 Subject: dovecot-2.2: Added Valgrind suppressions file for ignoring any b... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fdfc926202da changeset: 19367:fdfc926202da user: Timo Sirainen date: Mon Nov 16 14:15:02 2015 +0200 description: Added Valgrind suppressions file for ignoring any bash memory leaks. Some of the unit tests are run via shell. diffstat: run-test-valgrind.supp | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (10 lines): diff -r 2a85e0b0dc35 -r fdfc926202da run-test-valgrind.supp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/run-test-valgrind.supp Mon Nov 16 14:15:02 2015 +0200 @@ -0,0 +1,6 @@ +{ + + Memcheck:Leak + fun:malloc + obj:*/bash +} From dovecot at dovecot.org Mon Nov 16 12:16:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Nov 2015 12:16:28 +0000 Subject: dovecot-2.2: dovecot.m4: Run valgrind with --trace-children=yes Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/923935a77ca8 changeset: 19368:923935a77ca8 user: Timo Sirainen date: Mon Nov 16 14:16:00 2015 +0200 description: dovecot.m4: Run valgrind with --trace-children=yes Some of the unit tests are run via shell and we want to check the unit tests instead of just the shell. diffstat: dovecot.m4 | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (24 lines): diff -r fdfc926202da -r 923935a77ca8 dovecot.m4 --- a/dovecot.m4 Mon Nov 16 14:15:02 2015 +0200 +++ b/dovecot.m4 Mon Nov 16 14:16:00 2015 +0200 @@ -6,7 +6,7 @@ # unlimited permission to copy and/or distribute it, with or without # modifications, as long as this notice is preserved. -# serial 18 +# serial 19 AC_DEFUN([DC_DOVECOT_MODULEDIR],[ AC_ARG_WITH(moduledir, @@ -46,9 +46,9 @@ trap "rm -f test.out.\$\$" 0 1 2 3 15 supp_path="\$top_srcdir/run-test-valgrind.supp" if test -r "\$supp_path"; then - valgrind -q --leak-check=full --suppressions="\$supp_path" --log-file=test.out.\$\$ \$[*] + valgrind -q --trace-children=yes --leak-check=full --suppressions="\$supp_path" --log-file=test.out.\$\$ \$[*] else - valgrind -q --leak-check=full --log-file=test.out.\$\$ \$[*] + valgrind -q --trace-children=yes --leak-check=full --log-file=test.out.\$\$ \$[*] fi ret=\$? if test -s test.out.\$\$; then From dovecot at dovecot.org Mon Nov 16 12:48:26 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Nov 2015 12:48:26 +0000 Subject: dovecot-2.2: pop3-migration: Fixed memory leak in unit test. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/caf770735755 changeset: 19370:caf770735755 user: Timo Sirainen date: Mon Nov 16 14:47:18 2015 +0200 description: pop3-migration: Fixed memory leak in unit test. diffstat: src/plugins/pop3-migration/test-pop3-migration-plugin.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 1146e005bd75 -r caf770735755 src/plugins/pop3-migration/test-pop3-migration-plugin.c --- a/src/plugins/pop3-migration/test-pop3-migration-plugin.c Mon Nov 16 14:46:40 2015 +0200 +++ b/src/plugins/pop3-migration/test-pop3-migration-plugin.c Mon Nov 16 14:47:18 2015 +0200 @@ -35,6 +35,7 @@ digest, &have_eoh) == 0, i); test_assert_idx(strcasecmp(binary_to_hex(digest, sizeof(digest)), tests[i].sha1) == 0, i); test_assert_idx(tests[i].have_eoh == have_eoh, i); + i_stream_unref(&input); } test_end(); From dovecot at dovecot.org Mon Nov 16 12:48:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Nov 2015 12:48:25 +0000 Subject: dovecot-2.2: lib-mail: Fixed istream-header-filter when callback... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1146e005bd75 changeset: 19369:1146e005bd75 user: Timo Sirainen date: Mon Nov 16 14:46:40 2015 +0200 description: lib-mail: Fixed istream-header-filter when callback excluded multiple headers. This only affected pop3-migration plugin. diffstat: src/lib-mail/istream-header-filter.c | 3 ++- src/lib-mail/test-istream-header-filter.c | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diffs (38 lines): diff -r 923935a77ca8 -r 1146e005bd75 src/lib-mail/istream-header-filter.c --- a/src/lib-mail/istream-header-filter.c Mon Nov 16 14:16:00 2015 +0200 +++ b/src/lib-mail/istream-header-filter.c Mon Nov 16 14:46:40 2015 +0200 @@ -220,7 +220,8 @@ mstream->context); if (matched != orig_matched && !mstream->headers_edited) { - i_array_init(&mstream->match_change_lines, 8); + if (!array_is_created(&mstream->match_change_lines)) + i_array_init(&mstream->match_change_lines, 8); array_append(&mstream->match_change_lines, &mstream->cur_line, 1); } diff -r 923935a77ca8 -r 1146e005bd75 src/lib-mail/test-istream-header-filter.c --- a/src/lib-mail/test-istream-header-filter.c Mon Nov 16 14:16:00 2015 +0200 +++ b/src/lib-mail/test-istream-header-filter.c Mon Nov 16 14:46:40 2015 +0200 @@ -12,16 +12,17 @@ struct message_header_line *hdr, bool *matched, void *context ATTR_UNUSED) { - if (hdr != NULL && hdr->name_offset == 0) { - /* drop first header */ + if (hdr != NULL && (hdr->name_offset == 0 || + strcmp(hdr->name, "X-Drop") == 0)) { + /* drop 1) first header, 2) X-Drop header */ *matched = TRUE; } } static void test_istream_filter(void) { - static const char *exclude_headers[] = { "Subject", "To", NULL }; - const char *input = "From: foo\nFrom: abc\nTo: bar\nSubject: plop\n\nhello world\n"; + static const char *exclude_headers[] = { "Subject", "To", "X-Drop", NULL }; + const char *input = "From: foo\nFrom: abc\nTo: bar\nSubject: plop\nX-Drop: 1\n\nhello world\n"; const char *output = "From: abc\n\nhello world\n"; struct istream *istream, *filter, *filter2; unsigned int i, input_len = strlen(input); From dovecot at dovecot.org Tue Nov 17 09:56:21 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 09:56:21 +0000 Subject: dovecot-2.2: lib-fts: Add l' contraction to French unit test. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/01bd19136b2c changeset: 19371:01bd19136b2c user: Teemu Huovila date: Tue Nov 17 11:42:59 2015 +0200 description: lib-fts: Add l' contraction to French unit test. diffstat: src/lib-fts/test-fts-filter.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r caf770735755 -r 01bd19136b2c src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Mon Nov 16 14:47:18 2015 +0200 +++ b/src/lib-fts/test-fts-filter.c Tue Nov 17 11:42:59 2015 +0200 @@ -61,7 +61,8 @@ { "d", "d" }, { "qu'", NULL }, { "j'adore", "adore" }, - { "quelqu'un", "quelqu'un" } + { "quelqu'un", "quelqu'un" }, + { "l'esprit", "esprit" } }; struct fts_filter *filter; const char *error; From dovecot at dovecot.org Tue Nov 17 09:56:21 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 09:56:21 +0000 Subject: dovecot-2.2: lib-fts: Add Swedish (sv) to supported languages. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c7f9d3d8f278 changeset: 19372:c7f9d3d8f278 user: Teemu Huovila date: Tue Nov 17 11:43:28 2015 +0200 description: lib-fts: Add Swedish (sv) to supported languages. diffstat: src/lib-fts/Makefile.am | 3 +- src/lib-fts/fts-language.c | 1 + src/lib-fts/stopwords_sv.txt | 131 ++++++++++++++++++++++++++++++++++++++++ src/lib-fts/test-fts-filter.c | 53 ++++++++++++++++ src/lib-fts/test-fts-language.c | 27 ++++++++ 5 files changed, 214 insertions(+), 1 deletions(-) diffs (280 lines): diff -r 01bd19136b2c -r c7f9d3d8f278 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Tue Nov 17 11:42:59 2015 +0200 +++ b/src/lib-fts/Makefile.am Tue Nov 17 11:43:28 2015 +0200 @@ -14,7 +14,8 @@ dist_stopwords_DATA = \ stopwords_en.txt \ stopwords_fi.txt \ - stopwords_fr.txt + stopwords_fr.txt \ + stopwords_sv.txt BUILT_SOURCES = word-boundary-data.c word-break-data.c diff -r 01bd19136b2c -r c7f9d3d8f278 src/lib-fts/fts-language.c --- a/src/lib-fts/fts-language.c Tue Nov 17 11:42:59 2015 +0200 +++ b/src/lib-fts/fts-language.c Tue Nov 17 11:43:28 2015 +0200 @@ -41,6 +41,7 @@ { "pt" }, { "ro" }, { "ru" }, + { "sv" } }; const struct fts_language fts_language_data = { diff -r 01bd19136b2c -r c7f9d3d8f278 src/lib-fts/stopwords_sv.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/stopwords_sv.txt Tue Nov 17 11:43:28 2015 +0200 @@ -0,0 +1,131 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/swedish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + + | A Swedish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + | Swedish stop words occasionally exhibit homonym clashes. For example + | s? = so, but also seed. These are indicated clearly below. + +och | and +det | it, this/that +att | to (with infinitive) +i | in, at +en | a +jag | I +hon | she +som | who, that +han | he +p? | on +den | it, this/that +med | with +var | where, each +sig | him(self) etc +f?r | for +s? | so (also: seed) +till | to +?r | is +men | but +ett | a +om | if; around, about +hade | had +de | they, these/those +av | of +icke | not, no +mig | me +du | you +henne | her +d? | then, when +sin | his +nu | now +har | have +inte | inte n?gon = no one +hans | his +honom | him +skulle | 'sake' +hennes | her +d?r | there +min | my +man | one (pronoun) +ej | nor +vid | at, by, on (also: vast) +kunde | could +n?got | some etc +fr?n | from, off +ut | out +n?r | when +efter | after, behind +upp | up +vi | we +dem | them +vara | be +vad | what +?ver | over +?n | than +dig | you +kan | can +sina | his +h?r | here +ha | have +mot | towards +alla | all +under | under (also: wonder) +n?gon | some etc +eller | or (else) +allt | all +mycket | much +sedan | since +ju | why +denna | this/that +sj?lv | myself, yourself etc +detta | this/that +?t | to +utan | without +varit | was +hur | how +ingen | no +mitt | my +ni | you +bli | to be, become +blev | from bli +oss | us +din | thy +dessa | these/those +n?gra | some etc +deras | their +blir | from bli +mina | my +samma | (the) same +vilken | who, that +er | you, your +s?dan | such a +v?r | our +blivit | from bli +dess | its +inom | within +mellan | between +s?dant | such a +varf?r | why +varje | each +vilka | who, that +ditt | thy +vem | who +vilket | who, that +sitta | his +s?dana | such a +vart | each +dina | thy +vars | whose +v?rt | our +v?ra | our +ert | your +era | your +vilkas | whose + diff -r 01bd19136b2c -r c7f9d3d8f278 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Tue Nov 17 11:42:59 2015 +0200 +++ b/src/lib-fts/test-fts-filter.c Tue Nov 17 11:43:28 2015 +0200 @@ -13,6 +13,7 @@ static const char *const stopword_settings[] = {"stopwords_dir", TEST_STOPWORDS_DIR, NULL}; static struct fts_language english_language = { .name = "en" }; static struct fts_language french_language = { .name = "fr" }; +static struct fts_language swedish_language = { .name = "sv" }; static void test_fts_filter_find(void) { @@ -653,6 +654,57 @@ test_assert(normalizer == NULL); test_end(); } + +static void test_fts_filter_stopwords_normalizer_stemmer_sv(void) +{ + int ret; + struct fts_filter *normalizer; + struct fts_filter *stemmer; + struct fts_filter *filter; + const char *error; + const char *token = NULL; + const char * const tokens[] = { + "En?r", "erk?nnandet", "av", "det", "inneboende", "v?rdet", + "hos", "alla", "medlemmar", "av", "m?nniskosl?ktet", "och", + "av", "deras", "lika", "och", "of?rytterliga", "r?ttigheter", + "?r", "grundvalen", "f?r", "frihet", "r?ttvisa", "och", "fred", + "i", "v?rlden", NULL}; + const char * const bases[] = { + "enar", "erkan", NULL, NULL, "inneboend", "vardet", "hos", NULL, + "medlemm", NULL, "manniskoslaktet", NULL, NULL, NULL, "lik", + NULL, "oforytter", "ratt", NULL, "grundval", NULL, "frihet", + "rattvis", NULL, "fred", NULL, "varld", NULL}; + const char * const *tpp; + const char * const *bpp; + + test_begin("fts filters with stopwords, default normalizer and stemming chained, Swedish"); + + + test_assert(fts_filter_create(fts_filter_stopwords, NULL, &swedish_language, stopword_settings, &filter, &error) == 0); + test_assert(fts_filter_create(fts_filter_normalizer_icu, filter, NULL, NULL, &normalizer, &error) == 0); + test_assert(fts_filter_create(fts_filter_stemmer_snowball, normalizer, &swedish_language, NULL, &stemmer, &error) == 0); + + bpp = bases; + for (tpp = tokens; *tpp != NULL; tpp++) { + token = *tpp; + ret = fts_filter_filter(stemmer, &token, &error); + if (ret <= 0) { + test_assert(ret == 0); + test_assert(*bpp == NULL); + } else { + test_assert(*bpp != NULL); + test_assert(strcmp(*bpp, token) == 0); + } + bpp++; + } + fts_filter_unref(&stemmer); + fts_filter_unref(&normalizer); + fts_filter_unref(&filter); + test_assert(stemmer == NULL); + test_assert(filter == NULL); + test_assert(normalizer == NULL); + test_end(); +} #endif #endif @@ -741,6 +793,7 @@ test_fts_filter_normalizer_invalid_id, #ifdef HAVE_FTS_STEMMER test_fts_filter_normalizer_stopwords_stemmer_eng, + test_fts_filter_stopwords_normalizer_stemmer_sv, #endif #endif test_fts_filter_english_possessive, diff -r 01bd19136b2c -r c7f9d3d8f278 src/lib-fts/test-fts-language.c --- a/src/lib-fts/test-fts-language.c Tue Nov 17 11:42:59 2015 +0200 +++ b/src/lib-fts/test-fts-language.c Tue Nov 17 11:43:28 2015 +0200 @@ -134,6 +134,32 @@ test_end(); } +/* Detect Swedish */ +static void test_fts_language_detect_swedish(void) +{ + struct fts_language_list *lp = NULL; + const struct fts_language *lang_r = NULL; + const unsigned char swedish[] = + "Artikel 1."\ + "Alla m\xC3\xA4nniskor \xC3\xA4ro f\xC3\xB6""dda fria och lika"\ + " i v\xC3\xA4rde och r\xC3\xA4ttigheter. De \xC3\xA4ro "\ + "utrustade med f\xC3\xB6rnuft och samvete och b\xC3\xB6ra "\ + "handla gentemot varandra i en anda av broderskap."; + + + + const char names[] = "fi, de, sv, fr, en"; + const char *unknown, *error; + test_begin("fts language detect Swedish"); + test_assert(fts_language_list_init(settings, &lp, &error) == 0); + test_assert(fts_language_list_add_names(lp, names, &unknown) == TRUE); + test_assert(fts_language_detect(lp, swedish, sizeof(swedish)-1, &lang_r) + == FTS_LANGUAGE_RESULT_OK); + test_assert(strcmp(lang_r->name, "sv") == 0); + fts_language_list_deinit(&lp); + test_end(); +} + /* Detect Finnish as English */ static void test_fts_language_detect_finnish_as_english(void) { @@ -212,6 +238,7 @@ test_fts_language_detect_english, test_fts_language_detect_french, test_fts_language_detect_german, + test_fts_language_detect_swedish, test_fts_language_detect_finnish_as_english, test_fts_language_detect_na, test_fts_language_detect_unknown, From dovecot at dovecot.org Tue Nov 17 09:56:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 09:56:32 +0000 Subject: dovecot-2.2: lib-fts: Add comment to language names. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f31fadf622f2 changeset: 19373:f31fadf622f2 user: Teemu Huovila date: Tue Nov 17 11:43:58 2015 +0200 description: lib-fts: Add comment to language names. Languages are defined by their ISO 639-1 code, which is a two letters. It is possible, that some languages with only a three letter code, ie. a ISO 639-2 code, could be added in the future. diffstat: src/lib-fts/fts-language.c | 25 +++++++++++++------------ 1 files changed, 13 insertions(+), 12 deletions(-) diffs (36 lines): diff -r c7f9d3d8f278 -r f31fadf622f2 src/lib-fts/fts-language.c --- a/src/lib-fts/fts-language.c Tue Nov 17 11:43:28 2015 +0200 +++ b/src/lib-fts/fts-language.c Tue Nov 17 11:43:58 2015 +0200 @@ -29,19 +29,20 @@ bool textcat_failed; }; +/* ISO 639-1 alpha 2 codes for languages */ const struct fts_language fts_languages[] = { - { "da" }, - { "de" }, - { "en" }, - { "es" }, - { "fi" }, - { "fr" }, - { "it" }, - { "nl" }, - { "pt" }, - { "ro" }, - { "ru" }, - { "sv" } + { "da" }, /* Danish */ + { "de" }, /* German */ + { "en" }, /* English */ + { "es" }, /* Spanish */ + { "fi" }, /* Finnish */ + { "fr" }, /* French */ + { "it" }, /* Italian */ + { "nl" }, /* Dutch */ + { "pt" }, /* Portuguese */ + { "ro" }, /* Romanian */ + { "ru" }, /* Russian */ + { "sv" } /* Swedish */ }; const struct fts_language fts_language_data = { From dovecot at dovecot.org Tue Nov 17 09:56:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 09:56:32 +0000 Subject: dovecot-2.2: lib-fts: Add Norwegian. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/301d48ef7398 changeset: 19374:301d48ef7398 user: Teemu Huovila date: Tue Nov 17 11:44:19 2015 +0200 description: lib-fts: Add Norwegian. Norwegian has two main dialects, Bokmal(nb) and Nynorsk(nn). They are detected separately by libexttextcat, but the stemmer only knows Norwegian. Thus they are treated as a single language, Norwegian (no). This might also make more sense in everyday use of mixed writing style Norwegian. Caveat: The default normalizer filter does not modify U+00F8 (Latin Small Letter O with Stroke). In some configurations it might be desirable to rewrite it to e.g. o. Same goes for the upper case version. This can be done by passing a modified "id" setting to the normalizer filter. diffstat: src/lib-fts/Makefile.am | 1 + src/lib-fts/fts-language.c | 5 + src/lib-fts/stopwords_no.txt | 192 ++++++++++++++++++++++++++++++++++++++++ src/lib-fts/test-fts-filter.c | 113 +++++++++++++++++++++++ src/lib-fts/test-fts-language.c | 50 ++++++++++ 5 files changed, 361 insertions(+), 0 deletions(-) diffs (truncated from 447 to 300 lines): diff -r f31fadf622f2 -r 301d48ef7398 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Tue Nov 17 11:43:58 2015 +0200 +++ b/src/lib-fts/Makefile.am Tue Nov 17 11:44:19 2015 +0200 @@ -15,6 +15,7 @@ stopwords_en.txt \ stopwords_fi.txt \ stopwords_fr.txt \ + stopwords_no.txt \ stopwords_sv.txt BUILT_SOURCES = word-boundary-data.c word-break-data.c diff -r f31fadf622f2 -r 301d48ef7398 src/lib-fts/fts-language.c --- a/src/lib-fts/fts-language.c Tue Nov 17 11:43:58 2015 +0200 +++ b/src/lib-fts/fts-language.c Tue Nov 17 11:44:19 2015 +0200 @@ -39,6 +39,7 @@ { "fr" }, /* French */ { "it" }, /* Italian */ { "nl" }, /* Dutch */ + { "no" }, /* Both Bokmal and Nynorsk are detected as Norwegian */ { "pt" }, /* Portuguese */ { "ro" }, /* Romanian */ { "ru" }, /* Russian */ @@ -175,6 +176,10 @@ /* name is -- eg, fi--utf8 or pt-PT-utf8 */ name = t_strcut(candp[i].name, '-'); + + /* For Norwegian we treat both bokmal and nynorsk as "no". */ + if (strcmp(name, "nb") == 0 || strcmp(name, "nn") == 0) + name = "no"; if ((*lang_r = fts_language_list_find(list, name)) != NULL) return TRUE; } diff -r f31fadf622f2 -r 301d48ef7398 src/lib-fts/stopwords_no.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/stopwords_no.txt Tue Nov 17 11:44:19 2015 +0200 @@ -0,0 +1,192 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/norwegian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + + | A Norwegian stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This stop word list is for the dominant bokm?l dialect. Words unique + | to nynorsk are marked *. + + | Revised by Jan Bruusgaard , Jan 2005 + +og | and +i | in +jeg | I +det | it/this/that +at | to (w. inf.) +en | a/an +et | a/an +den | it/this/that +til | to +er | is/am/are +som | who/that +p? | on +de | they / you(formal) +med | with +han | he +av | of +ikke | not +ikkje | not * +der | there +s? | so +var | was/were +meg | me +seg | you +men | but +ett | one +har | have +om | about +vi | we +min | my +mitt | my +ha | have +hadde | had +hun | she +n? | now +over | over +da | when/as +ved | by/know +fra | from +du | you +ut | out +sin | your +dem | them +oss | us +opp | up +man | you/one +kan | can +hans | his +hvor | where +eller | or +hva | what +skal | shall/must +selv | self (reflective) +sj?l | self (reflective) +her | here +alle | all +vil | will +bli | become +ble | became +blei | became * +blitt | have become +kunne | could +inn | in +n?r | when +v?re | be +kom | come +noen | some +noe | some +ville | would +dere | you +som | who/which/that +deres | their/theirs +kun | only/just +ja | yes +etter | after +ned | down +skulle | should +denne | this +for | for/because +deg | you +si | hers/his +sine | hers/his +sitt | hers/his +mot | against +? | to +meget | much +hvorfor | why +dette | this +disse | these/those +uten | without +hvordan | how +ingen | none +din | your +ditt | your +blir | become +samme | same +hvilken | which +hvilke | which (plural) +s?nn | such a +inni | inside/within +mellom | between +v?r | our +hver | each +hvem | who +vors | us/ours +hvis | whose +b?de | both +bare | only/just +enn | than +fordi | as/because +f?r | before +mange | many +ogs? | also +slik | just +v?rt | been +v?re | to be +b?e | both * +begge | both +siden | since +dykk | your * +dykkar | yours * +dei | they * +deira | them * +deires | theirs * +deim | them * +di | your (fem.) * +d? | as/when * +eg | I * +ein | a/an * +eit | a/an * +eitt | a/an * +elles | or * +honom | he * +hj? | at * +ho | she * +hoe | she * +henne | her +hennar | her/hers +hennes | hers +hoss | how * +hossen | how * +ikkje | not * +ingi | noone * +inkje | noone * +korleis | how * +korso | how * +kva | what/which * +kvar | where * +kvarhelst | where * +kven | who/whom * +kvi | why * +kvifor | why * +me | we * +medan | while * +mi | my * +mine | my * +mykje | much * +no | now * +nokon | some (masc./neut.) * +noka | some (fem.) * +nokor | some * +noko | some * +nokre | some * +si | his/hers * +sia | since * +sidan | since * +so | so * +somt | some * +somme | some * +um | about* +upp | up * +vere | be * +vore | was * +verte | become * +vort | become * +varte | became * +vart | became * + diff -r f31fadf622f2 -r 301d48ef7398 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Tue Nov 17 11:43:58 2015 +0200 +++ b/src/lib-fts/test-fts-filter.c Tue Nov 17 11:44:19 2015 +0200 @@ -13,6 +13,7 @@ static const char *const stopword_settings[] = {"stopwords_dir", TEST_STOPWORDS_DIR, NULL}; static struct fts_language english_language = { .name = "en" }; static struct fts_language french_language = { .name = "fr" }; +static struct fts_language norwegian_language = { .name = "no" }; static struct fts_language swedish_language = { .name = "sv" }; static void test_fts_filter_find(void) @@ -282,6 +283,54 @@ test_end(); } +static void test_fts_filter_stopwords_no(void) +{ + struct fts_filter *filter; + const char *error; + int ret; + + const char *input[] = {"og", "d\xC3\xA5", "medlemsstatane", "har", + "bunde", "seg", "til", "\xC3\xA5", "fremje", + "allmenn", "v\xC3\xB8rdnad", "for", "pakta", + "og", "halde", "seg", "etter", "menneskerettane", + "og", "den", "grunnleggjande", "fridomen", "i", + "samarbeid", "med", "Dei", "Sameinte", + "Nasjonane", NULL}; + + const char *output[] = {NULL, NULL, "medlemsstatane", NULL, + "bunde", NULL, NULL, NULL, "fremje", + "allmenn", "v\xC3\xB8rdnad", NULL, "pakta", + NULL, "halde", NULL, NULL, "menneskerettane", + NULL, NULL, "grunnleggjande", "fridomen", NULL, + "samarbeid", NULL, "Dei", "Sameinte", + "Nasjonane"}; + const char **ip, **op; + const char *token; + + test_begin("fts filter stopwords, Norwegian"); + test_assert(fts_filter_create(fts_filter_stopwords, NULL, &norwegian_language, stopword_settings, &filter, &error) == 0); + + ip = input; + op = output; + while (*ip != NULL) { + token = *ip; + ret = fts_filter_filter(filter, &token, &error); + if (ret <= 0) { + test_assert(ret == 0); + test_assert(*op == NULL); + } else { + test_assert(*op != NULL); + test_assert(strcmp(*ip, token) == 0); + } + op++; + ip++; + } + + fts_filter_unref(&filter); + test_assert(filter == NULL); + test_end(); +} + static void test_fts_filter_stopwords_fail_lazy_init(void) { const struct fts_language unknown = { .name = "bebobidoop" }; @@ -655,6 +704,68 @@ test_end(); } +static void test_fts_filter_stopwords_normalizer_stemmer_no(void) From dovecot at dovecot.org Tue Nov 17 09:56:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 09:56:32 +0000 Subject: dovecot-2.2: lib-fts: Minor code cleanup - Rename some internal ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/af09ba0a1360 changeset: 19375:af09ba0a1360 user: Teemu Huovila date: Tue Nov 17 11:44:53 2015 +0200 description: lib-fts: Minor code cleanup - Rename some internal functions. Maybe the names are more logical this way. diffstat: src/lib-fts/fts-tokenizer-generic.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diffs (35 lines): diff -r 301d48ef7398 -r af09ba0a1360 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Tue Nov 17 11:44:19 2015 +0200 +++ b/src/lib-fts/fts-tokenizer-generic.c Tue Nov 17 11:44:53 2015 +0200 @@ -212,7 +212,7 @@ } static int -fts_tokenizer_generic_next_simple(struct fts_tokenizer *_tok, +fts_tokenizer_generic_simple_next(struct fts_tokenizer *_tok, const unsigned char *data, size_t size, size_t *skip_r, const char **token_r, const char **error_r ATTR_UNUSED) @@ -666,7 +666,7 @@ } static int -fts_tokenizer_generic_next_tr29(struct fts_tokenizer *_tok, +fts_tokenizer_generic_tr29_next(struct fts_tokenizer *_tok, const unsigned char *data, size_t size, size_t *skip_r, const char **token_r, const char **error_r ATTR_UNUSED) @@ -760,11 +760,11 @@ fts_tokenizer_generic_create, fts_tokenizer_generic_destroy, fts_tokenizer_generic_reset, - fts_tokenizer_generic_next_simple + fts_tokenizer_generic_simple_next }; const struct fts_tokenizer_vfuncs generic_tokenizer_vfuncs_tr29 = { fts_tokenizer_generic_create, fts_tokenizer_generic_destroy, fts_tokenizer_generic_reset, - fts_tokenizer_generic_next_tr29 + fts_tokenizer_generic_tr29_next }; From dovecot at dovecot.org Tue Nov 17 09:56:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 09:56:37 +0000 Subject: dovecot-2.2: lib-fts: Removed TODO comment. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f015c3dc7b4b changeset: 19376:f015c3dc7b4b user: Teemu Huovila date: Tue Nov 17 11:45:24 2015 +0200 description: lib-fts: Removed TODO comment. The call to uni_utf8_get_char_n() already does easy cases first. Adding more special code paths is not necessary. diffstat: src/lib-fts/fts-tokenizer-generic.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diffs (11 lines): diff -r af09ba0a1360 -r f015c3dc7b4b src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Tue Nov 17 11:44:53 2015 +0200 +++ b/src/lib-fts/fts-tokenizer-generic.c Tue Nov 17 11:45:24 2015 +0200 @@ -678,7 +678,6 @@ enum letter_type lt; int char_size; - /* TODO: Process 8bit chars separately, to speed things up. */ for (i = 0; i < size; ) { char_start_i = i; char_size = uni_utf8_get_char_n(data + i, size - i, &c); From dovecot at dovecot.org Tue Nov 17 09:56:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 09:56:37 +0000 Subject: dovecot-2.2: lib-fts: Explicitly state encoding used for stemming. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d2aded3f7065 changeset: 19377:d2aded3f7065 user: Teemu Huovila date: Tue Nov 17 11:45:44 2015 +0200 description: lib-fts: Explicitly state encoding used for stemming. diffstat: src/lib-fts/fts-filter-stemmer-snowball.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r f015c3dc7b4b -r d2aded3f7065 src/lib-fts/fts-filter-stemmer-snowball.c --- a/src/lib-fts/fts-filter-stemmer-snowball.c Tue Nov 17 11:45:24 2015 +0200 +++ b/src/lib-fts/fts-filter-stemmer-snowball.c Tue Nov 17 11:45:44 2015 +0200 @@ -55,7 +55,7 @@ fts_filter_stemmer_snowball_create_stemmer(struct fts_filter_stemmer_snowball *sp, const char **error_r) { - sp->stemmer = sb_stemmer_new(sp->lang->name, NULL); + sp->stemmer = sb_stemmer_new(sp->lang->name, "UTF_8"); if (sp->stemmer == NULL) { *error_r = t_strdup_printf( "Creating a Snowball stemmer for language '%s' failed.", From dovecot at dovecot.org Tue Nov 17 09:56:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 09:56:37 +0000 Subject: dovecot-2.2: dovecot-config: Added LIBFTS. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ea447bc283e1 changeset: 19378:ea447bc283e1 user: Teemu Huovila date: Tue Nov 17 11:46:49 2015 +0200 description: dovecot-config: Added LIBFTS. diffstat: configure.ac | 4 ++++ dovecot-config.in.in | 3 +++ dovecot.m4 | 8 ++++---- 3 files changed, 11 insertions(+), 4 deletions(-) diffs (80 lines): diff -r d2aded3f7065 -r ea447bc283e1 configure.ac --- a/configure.ac Tue Nov 17 11:45:44 2015 +0200 +++ b/configure.ac Tue Nov 17 11:46:49 2015 +0200 @@ -2523,6 +2523,7 @@ LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/libdovecot-login.la' LIBDOVECOT_COMPRESS='$(top_builddir)/src/lib-compression/libcompression.la' LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/libdovecot-lda.la' + LIBDOVECOT_LIBFTS='$(top_builddir)/src/lib-fts/libfts.la' else LIBDOVECOT_DEPS='$(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-stats/libstats.la $(top_builddir)/src/lib-http/libhttp.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-sasl/libsasl.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la' LIBDOVECOT="$LIBDOVECOT_DEPS \$(LIBICONV) \$(MODULE_LIBS)" @@ -2530,6 +2531,7 @@ LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/liblogin.la' LIBDOVECOT_COMPRESS='$(top_builddir)/src/lib-compression/libcompression.la' LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/liblda.la' + LIBDOVECOT_LIBFTS_DEPS='$(top_builddir)/src/lib-fts/libfts.la' fi LIBDOVECOT_STORAGE="$LIBDOVECOT_STORAGE_DEPS" LIBDOVECOT_DSYNC='$(top_builddir)/src/doveadm/dsync/libdovecot-dsync.la' @@ -2543,6 +2545,8 @@ AC_SUBST(LIBDOVECOT_COMPRESS) AC_SUBST(LIBDOVECOT_DSYNC) AC_SUBST(LIBDOVECOT_LDA) +AC_SUBST(LIBDOVECOT_LIBFTS) +AC_SUBST(LIBDOVECOT_LIBFTS_DEPS) dnl ** dnl ** SQL drivers diff -r d2aded3f7065 -r ea447bc283e1 dovecot-config.in.in --- a/dovecot-config.in.in Tue Nov 17 11:45:44 2015 +0200 +++ b/dovecot-config.in.in Tue Nov 17 11:46:49 2015 +0200 @@ -11,6 +11,7 @@ LIBDOVECOT_LDA="@LIBDOVECOT_LDA@" LIBDOVECOT_STORAGE="@LIBDOVECOT_STORAGE@" LIBDOVECOT_DSYNC="@LIBDOVECOT_DSYNC@" +LIBDOVECOT_LIBFTS="@LIBDOVECOT_LIBFTS@" LIBDOVECOT_DEPS="@LIBDOVECOT_DEPS@" LIBDOVECOT_LOGIN_DEPS="@LIBDOVECOT_LOGIN@" @@ -19,6 +20,7 @@ LIBDOVECOT_LDA_DEPS="@LIBDOVECOT_LDA@" LIBDOVECOT_STORAGE_DEPS="@LIBDOVECOT_STORAGE_DEPS@" LIBDOVECOT_DSYNC_DEPS="@LIBDOVECOT_DSYNC@" +LIBDOVECOT_LIBFTS_DEPS="@LIBDOVECOT_LIBFTS_DEPS@" LIBDOVECOT_INCLUDE="-I$(incdir) -I$(incdir)/src/lib -I$(incdir)/src/lib-dict -I$(incdir)/src/lib-dns -I$(incdir)/src/lib-http -I$(incdir)/src/lib-mail -I$(incdir)/src/lib-imap -I$(incdir)/src/lib-fs -I$(incdir)/src/lib-charset -I$(incdir)/src/lib-auth -I$(incdir)/src/lib-master -I$(incdir)/src/lib-ssl-iostream -I$(incdir)/src/lib-compression -I$(incdir)/src/lib-settings -I$(incdir)/src/lib-test -I$(incdir)/src/lib-sasl -I$(incdir)/src/lib-stats" LIBDOVECOT_LDA_INCLUDE="-I$(incdir)/src/lib-lda -I$(incdir)/src/lda" @@ -33,6 +35,7 @@ LIBDOVECOT_NOTIFY_INCLUDE="-I$(incdir)/src/plugins/notify" LIBDOVECOT_PUSH_NOTIFICATION_INCLUDE="-I$(incdir)/src/plugins/push-notification" LIBDOVECOT_ACL_INCLUDE="-I$(incdir)/src/plugins/acl" +LIBDOVECOT_LIBFTS_INCLUDE="-I$(incdir)/src/lib-fts" dovecot_pkgincludedir= dovecot_pkglibdir= diff -r d2aded3f7065 -r ea447bc283e1 dovecot.m4 --- a/dovecot.m4 Tue Nov 17 11:45:44 2015 +0200 +++ b/dovecot.m4 Tue Nov 17 11:46:49 2015 +0200 @@ -6,7 +6,7 @@ # unlimited permission to copy and/or distribute it, with or without # modifications, as long as this notice is preserved. -# serial 19 +# serial 20 AC_DEFUN([DC_DOVECOT_MODULEDIR],[ AC_ARG_WITH(moduledir, @@ -126,9 +126,9 @@ AX_SUBST_L([DISTCHECK_CONFIGURE_FLAGS], [dovecotdir], [dovecot_moduledir], [dovecot_installed_moduledir], [dovecot_pkgincludedir], [dovecot_pkglibexecdir], [dovecot_pkglibdir], [dovecot_docdir]) AX_SUBST_L([DOVECOT_INSTALLED], [DOVECOT_CFLAGS], [DOVECOT_LIBS], [DOVECOT_SSL_LIBS], [DOVECOT_SQL_LIBS], [DOVECOT_COMPRESS_LIBS]) - AX_SUBST_L([LIBDOVECOT], [LIBDOVECOT_LOGIN], [LIBDOVECOT_SQL], [LIBDOVECOT_SSL], [LIBDOVECOT_COMPRESS], [LIBDOVECOT_LDA], [LIBDOVECOT_STORAGE], [LIBDOVECOT_DSYNC]) - AX_SUBST_L([LIBDOVECOT_DEPS], [LIBDOVECOT_LOGIN_DEPS], [LIBDOVECOT_SQL_DEPS], [LIBDOVECOT_SSL_DEPS], [LIBDOVECOT_COMPRESS_DEPS], [LIBDOVECOT_LDA_DEPS], [LIBDOVECOT_STORAGE_DEPS], [LIBDOVECOT_DSYNC_DEPS]) - AX_SUBST_L([LIBDOVECOT_INCLUDE], [LIBDOVECOT_LDA_INCLUDE], [LIBDOVECOT_DOVEADM_INCLUDE], [LIBDOVECOT_SERVICE_INCLUDE], [LIBDOVECOT_STORAGE_INCLUDE], [LIBDOVECOT_LOGIN_INCLUDE], [LIBDOVECOT_CONFIG_INCLUDE], [LIBDOVECOT_IMAP_INCLUDE], [LIBDOVECOT_DSYNC_INCLUDE], [LIBDOVECOT_IMAPC_INCLUDE], [LIBDOVECOT_FTS_INCLUDE], [LIBDOVECOT_NOTIFY_INCLUDE], [LIBDOVECOT_ACL_INCLUDE]) + AX_SUBST_L([LIBDOVECOT], [LIBDOVECOT_LOGIN], [LIBDOVECOT_SQL], [LIBDOVECOT_SSL], [LIBDOVECOT_COMPRESS], [LIBDOVECOT_LDA], [LIBDOVECOT_STORAGE], [LIBDOVECOT_DSYNC], [LIBDOVECOT_LIBFTS]) + AX_SUBST_L([LIBDOVECOT_DEPS], [LIBDOVECOT_LOGIN_DEPS], [LIBDOVECOT_SQL_DEPS], [LIBDOVECOT_SSL_DEPS], [LIBDOVECOT_COMPRESS_DEPS], [LIBDOVECOT_LDA_DEPS], [LIBDOVECOT_STORAGE_DEPS], [LIBDOVECOT_DSYNC_DEPS], [LIBDOVECOT_LIBFTS_DEPS]) + AX_SUBST_L([LIBDOVECOT_INCLUDE], [LIBDOVECOT_LDA_INCLUDE], [LIBDOVECOT_DOVEADM_INCLUDE], [LIBDOVECOT_SERVICE_INCLUDE], [LIBDOVECOT_STORAGE_INCLUDE], [LIBDOVECOT_LOGIN_INCLUDE], [LIBDOVECOT_CONFIG_INCLUDE], [LIBDOVECOT_IMAP_INCLUDE], [LIBDOVECOT_DSYNC_INCLUDE], [LIBDOVECOT_IMAPC_INCLUDE], [LIBDOVECOT_FTS_INCLUDE], [LIBDOVECOT_NOTIFY_INCLUDE], [LIBDOVECOT_ACL_INCLUDE], [LIBDOVECOT_LIBFTS_INCLUDE]) AM_CONDITIONAL(DOVECOT_INSTALLED, test "$DOVECOT_INSTALLED" = "yes") From dovecot at dovecot.org Tue Nov 17 09:56:45 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 09:56:45 +0000 Subject: dovecot-2.2: lib-fts: Added fts_language_register() to register ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/92aa48461150 changeset: 19379:92aa48461150 user: Teemu Huovila date: Tue Nov 17 11:50:16 2015 +0200 description: lib-fts: Added fts_language_register() to register more languages in plugins. diffstat: src/lib-fts/fts-language.c | 50 +++++++++++++++++++++++++++++++++++----- src/lib-fts/fts-language.h | 10 ++++++++ src/lib-fts/test-fts-language.c | 27 +++++++++++++++++++++- 3 files changed, 79 insertions(+), 8 deletions(-) diffs (150 lines): diff -r ea447bc283e1 -r 92aa48461150 src/lib-fts/fts-language.c --- a/src/lib-fts/fts-language.c Tue Nov 17 11:46:49 2015 +0200 +++ b/src/lib-fts/fts-language.c Tue Nov 17 11:50:16 2015 +0200 @@ -2,9 +2,9 @@ #include "lib.h" #include "array.h" +#include "llist.h" #include "fts-language.h" -#include "strfuncs.h" -#include "llist.h" + #ifdef HAVE_LIBEXTTEXTCAT_TEXTCAT_H # include @@ -29,8 +29,11 @@ bool textcat_failed; }; +pool_t fts_languages_pool; +ARRAY_TYPE(fts_language) fts_languages; + /* ISO 639-1 alpha 2 codes for languages */ -const struct fts_language fts_languages[] = { +const struct fts_language fts_languages_builtin [] = { { "da" }, /* Danish */ { "de" }, /* German */ { "en" }, /* English */ @@ -50,13 +53,46 @@ "data" }; +void fts_languages_init(void) +{ + unsigned int i; + const struct fts_language *lp; + + fts_languages_pool = pool_alloconly_create("fts_language", + sizeof(fts_languages_builtin)); + p_array_init(&fts_languages, fts_languages_pool, + N_ELEMENTS(fts_languages_builtin)); + for (i = 0; i < N_ELEMENTS(fts_languages_builtin); i++){ + lp = &fts_languages_builtin[i]; + array_append(&fts_languages, &lp, 1); + } +} + +void fts_languages_deinit(void) +{ + if (fts_languages_pool != NULL) + pool_unref(&fts_languages_pool); +} + +void fts_language_register(const char *name) +{ + struct fts_language *lang; + + if (fts_language_find(name) != NULL) + return; + + lang = p_new(fts_languages_pool, struct fts_language, 1); + lang->name = p_strdup(fts_languages_pool, name); + array_append(&fts_languages, (const struct fts_language **)&lang, 1); +} + const struct fts_language *fts_language_find(const char *name) { - unsigned int i; + const struct fts_language *const *langp = NULL; - for (i = 0; i < N_ELEMENTS(fts_languages); i++) { - if (strcmp(fts_languages[i].name, name) == 0) - return &fts_languages[i]; + array_foreach(&fts_languages, langp) { + if (strcmp((*langp)->name, name) == 0) + return *langp; } return NULL; } diff -r ea447bc283e1 -r 92aa48461150 src/lib-fts/fts-language.h --- a/src/lib-fts/fts-language.h Tue Nov 17 11:46:49 2015 +0200 +++ b/src/lib-fts/fts-language.h Tue Nov 17 11:50:16 2015 +0200 @@ -24,10 +24,20 @@ language-specific filters. */ extern const struct fts_language fts_language_data; +/* + Language module API. +*/ +void fts_languages_init(void); +void fts_languages_deinit(void); +/* Add a language to the list of supported languages. */ +void fts_language_register(const char *name); /* Find a specified language by name. This finds from the internal list of supported languages. */ const struct fts_language *fts_language_find(const char *name); +/* + Language list API +*/ int fts_language_list_init(const char *const *settings, struct fts_language_list **list_r, const char **error_r); diff -r ea447bc283e1 -r 92aa48461150 src/lib-fts/test-fts-language.c --- a/src/lib-fts/test-fts-language.c Tue Nov 17 11:46:49 2015 +0200 +++ b/src/lib-fts/test-fts-language.c Tue Nov 17 11:50:16 2015 +0200 @@ -278,9 +278,29 @@ fts_language_list_deinit(&lp); test_end(); } +static void test_fts_language_find_builtin(void) +{ + const struct fts_language *lp; + test_begin("fts language find built-in"); + lp = fts_language_find("en"); + test_assert(lp != NULL); + test_assert(strcmp(lp->name, "en") == 0); + test_end(); +} +static void test_fts_language_register(void) +{ + const struct fts_language *lp; + test_begin("fts language register"); + fts_language_register("jp"); + lp = fts_language_find("jp"); + test_assert(lp != NULL); + test_assert(strcmp(lp->name, "jp") == 0); + test_end(); +} int main(void) { + int ret; static void (*test_functions[])(void) = { test_fts_language_detect_finnish, test_fts_language_detect_english, @@ -292,7 +312,12 @@ test_fts_language_detect_finnish_as_english, test_fts_language_detect_na, test_fts_language_detect_unknown, + test_fts_language_find_builtin, + test_fts_language_register, NULL }; - return test_run(test_functions); + fts_languages_init(); + ret = test_run(test_functions); + fts_languages_deinit(); + return ret; } From dovecot at dovecot.org Tue Nov 17 09:56:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 09:56:50 +0000 Subject: dovecot-2.2: fts: Added fts_library_init() and _deinit() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6bf404e276c6 changeset: 19380:6bf404e276c6 user: Teemu Huovila date: Tue Nov 17 11:51:45 2015 +0200 description: fts: Added fts_library_init() and _deinit() Replaces calling three different functions on init and deinit. diffstat: src/lib-fts/Makefile.am | 2 ++ src/lib-fts/fts-library.c | 19 +++++++++++++++++++ src/lib-fts/fts-library.h | 7 +++++++ src/plugins/fts/fts-plugin.c | 8 +++----- 4 files changed, 31 insertions(+), 5 deletions(-) diffs (83 lines): diff -r 92aa48461150 -r 6bf404e276c6 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Tue Nov 17 11:50:16 2015 +0200 +++ b/src/lib-fts/Makefile.am Tue Nov 17 11:51:45 2015 +0200 @@ -71,6 +71,7 @@ fts-filter-stopwords.c \ fts-filter-stemmer-snowball.c \ fts-language.c \ + fts-library.c \ fts-tokenizer.c \ fts-tokenizer-address.c \ fts-tokenizer-generic.c \ @@ -82,6 +83,7 @@ fts-filter-private.h \ fts-icu.h \ fts-language.h \ + fts-library.h \ fts-tokenizer.h \ fts-tokenizer-private.h \ fts-tokenizer-generic-private.h diff -r 92aa48461150 -r 6bf404e276c6 src/lib-fts/fts-library.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/fts-library.c Tue Nov 17 11:51:45 2015 +0200 @@ -0,0 +1,19 @@ +#include "lib.h" +#include "fts-language.h" +#include "fts-tokenizer.h" +#include "fts-filter.h" +#include "fts-library.h" + +void fts_library_init(void) +{ + fts_languages_init(); + fts_tokenizers_init(); + fts_filters_init(); +} + +void fts_library_deinit(void) +{ + fts_languages_deinit(); + fts_tokenizers_deinit(); + fts_filters_deinit(); +} diff -r 92aa48461150 -r 6bf404e276c6 src/lib-fts/fts-library.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/fts-library.h Tue Nov 17 11:51:45 2015 +0200 @@ -0,0 +1,7 @@ +#ifndef FTS_LIBRARY_H +#define FTS_LIBRARY_H + +void fts_library_init(void); +void fts_library_deinit(void); + +#endif diff -r 92aa48461150 -r 6bf404e276c6 src/plugins/fts/fts-plugin.c --- a/src/plugins/fts/fts-plugin.c Tue Nov 17 11:50:16 2015 +0200 +++ b/src/plugins/fts/fts-plugin.c Tue Nov 17 11:51:45 2015 +0200 @@ -8,7 +8,7 @@ #include "fts-storage.h" #include "fts-user.h" #include "fts-plugin.h" - +#include "fts-library.h" const char *fts_plugin_version = DOVECOT_ABI_VERSION; @@ -20,15 +20,13 @@ void fts_plugin_init(struct module *module) { - fts_filters_init(); - fts_tokenizers_init(); + fts_library_init(); mail_storage_hooks_add(module, &fts_mail_storage_hooks); } void fts_plugin_deinit(void) { - fts_filters_deinit(); - fts_tokenizers_deinit(); + fts_library_deinit(); fts_parsers_unload(); mail_storage_hooks_remove(&fts_mail_storage_hooks); } From dovecot at dovecot.org Tue Nov 17 14:30:42 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 14:30:42 +0000 Subject: dovecot-2.2: lib-fs: Added asserts to fs_set_metadata() to make ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d39daee9d72c changeset: 19381:d39daee9d72c user: Timo Sirainen date: Tue Nov 17 16:30:16 2015 +0200 description: lib-fs: Added asserts to fs_set_metadata() to make sure key and value are non-NULL Otherwise the bug would be noticed only much later when trying to dereference NULL pointer. diffstat: src/lib-fs/fs-api.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 6bf404e276c6 -r d39daee9d72c src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Tue Nov 17 11:51:45 2015 +0200 +++ b/src/lib-fs/fs-api.c Tue Nov 17 16:30:16 2015 +0200 @@ -302,6 +302,9 @@ void fs_set_metadata(struct fs_file *file, const char *key, const char *value) { + i_assert(key != NULL); + i_assert(value != NULL); + if (file->fs->v.set_metadata != NULL) T_BEGIN { file->fs->v.set_metadata(file, key, value); file->metadata_changed = TRUE; From dovecot at dovecot.org Tue Nov 17 15:27:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 15:27:24 +0000 Subject: dovecot-2.2: lib-master: Added extensible master_auth_request_fu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b4a406dce5b8 changeset: 19382:b4a406dce5b8 user: Timo Sirainen date: Tue Nov 17 17:11:05 2015 +0200 description: lib-master: Added extensible master_auth_request_full() call. Only the API was changed, no changes to functionality. diffstat: src/lib-master/master-auth.c | 44 ++++++++++++++++++++++++++++------------- src/lib-master/master-auth.h | 23 ++++++++++++++++++--- src/login-common/sasl-server.c | 10 +++++++- 3 files changed, 57 insertions(+), 20 deletions(-) diffs (158 lines): diff -r d39daee9d72c -r b4a406dce5b8 src/lib-master/master-auth.c --- a/src/lib-master/master-auth.c Tue Nov 17 16:30:16 2015 +0200 +++ b/src/lib-master/master-auth.c Tue Nov 17 17:11:05 2015 +0200 @@ -148,11 +148,10 @@ master_auth_connection_deinit(&conn); } -void master_auth_request(struct master_auth *auth, int fd, - const struct master_auth_request *request, - const unsigned char *data, - master_auth_callback_t *callback, - void *context, unsigned int *tag_r) +void master_auth_request_full(struct master_auth *auth, + const struct master_auth_request_params *params, + master_auth_callback_t *callback, void *context, + unsigned int *tag_r) { struct master_auth_connection *conn; struct master_auth_request req; @@ -160,27 +159,27 @@ struct stat st; ssize_t ret; - i_assert(request->client_pid != 0); - i_assert(request->auth_pid != 0); + i_assert(params->request.client_pid != 0); + i_assert(params->request.auth_pid != 0); conn = i_new(struct master_auth_connection, 1); conn->auth = auth; conn->callback = callback; conn->context = context; - req = *request; + req = params->request; req.tag = ++auth->tag_counter; if (req.tag == 0) req.tag = ++auth->tag_counter; - if (fstat(fd, &st) < 0) + if (fstat(params->client_fd, &st) < 0) i_fatal("fstat(auth dest fd) failed: %m"); req.ino = st.st_ino; buf = buffer_create_dynamic(pool_datastack_create(), sizeof(req) + req.data_size); buffer_append(buf, &req, sizeof(req)); - buffer_append(buf, data, req.data_size); + buffer_append(buf, params->data, req.data_size); conn->fd = net_connect_unix_with_retries(auth->path, SOCKET_CONNECT_RETRY_MSECS); @@ -192,10 +191,11 @@ return; } - ret = fd_send(conn->fd, fd, buf->data, buf->used); - if (ret < 0) - i_error("fd_send(%s, %d) failed: %m", auth->path, fd); - else if ((size_t)ret != buf->used) { + ret = fd_send(conn->fd, params->client_fd, buf->data, buf->used); + if (ret < 0) { + i_error("fd_send(%s, %d) failed: %m", auth->path, + params->client_fd); + } else if ((size_t)ret != buf->used) { i_error("fd_send(%s) sent only %d of %d bytes", auth->path, (int)ret, (int)buf->used); ret = -1; @@ -215,6 +215,22 @@ *tag_r = req.tag; } +void master_auth_request(struct master_auth *auth, int fd, + const struct master_auth_request *request, + const unsigned char *data, + master_auth_callback_t *callback, + void *context, unsigned int *tag_r) +{ + struct master_auth_request_params params; + + memset(¶ms, 0, sizeof(params)); + params.client_fd = fd; + params.request = *request; + params.data = data; + + master_auth_request_full(auth, ¶ms, callback, context, tag_r); +} + void master_auth_request_abort(struct master_auth *auth, unsigned int tag) { struct master_auth_connection *conn; diff -r d39daee9d72c -r b4a406dce5b8 src/lib-master/master-auth.h --- a/src/lib-master/master-auth.h Tue Nov 17 16:30:16 2015 +0200 +++ b/src/lib-master/master-auth.h Tue Nov 17 17:11:05 2015 +0200 @@ -66,6 +66,18 @@ pid_t mail_pid; }; +struct master_auth_request_params { + /* Client fd to transfer to post-login process or -1 if no fd is + wanted to be transferred. */ + int client_fd; + + /* Authentication request that is sent to post-login process. + tag is ignored. */ + struct master_auth_request request; + /* Client input of size request.data_size */ + const unsigned char *data; +}; + /* reply=NULL if the auth lookup was cancelled due to some error */ typedef void master_auth_callback_t(const struct master_auth_reply *reply, void *context); @@ -74,10 +86,13 @@ master_auth_init(struct master_service *service, const char *path); void master_auth_deinit(struct master_auth **auth); -/* Send an authentication request. The fd contains the file descriptor to - transfer, or -1 if no fd is wanted to be transferred. Returns tag which can - be used to abort the request (ie. ignore the reply from master). - request->tag is ignored. */ +/* Send an authentication request. Returns tag which can be used to abort the + request (ie. ignore the reply from master). */ +void master_auth_request_full(struct master_auth *auth, + const struct master_auth_request_params *params, + master_auth_callback_t *callback, void *context, + unsigned int *tag_r); +/* For backwards compatibility: */ void master_auth_request(struct master_auth *auth, int fd, const struct master_auth_request *request, const unsigned char *data, diff -r d39daee9d72c -r b4a406dce5b8 src/login-common/sasl-server.c --- a/src/login-common/sasl-server.c Tue Nov 17 16:30:16 2015 +0200 +++ b/src/login-common/sasl-server.c Tue Nov 17 17:11:05 2015 +0200 @@ -121,6 +121,7 @@ static void master_send_request(struct anvil_request *anvil_request) { struct client *client = anvil_request->client; + struct master_auth_request_params params; struct master_auth_request req; const unsigned char *data; size_t size; @@ -151,8 +152,13 @@ client->auth_finished = ioloop_time; client->master_auth_id = req.auth_id; - master_auth_request(master_auth, client->fd, &req, buf->data, - master_auth_callback, client, &client->master_tag); + + memset(¶ms, 0, sizeof(params)); + params.client_fd = client->fd; + params.request = req; + params.data = buf->data; + master_auth_request_full(master_auth, ¶ms, master_auth_callback, + client, &client->master_tag); } static void ATTR_NULL(1) From dovecot at dovecot.org Tue Nov 17 15:27:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 15:27:25 +0000 Subject: dovecot-2.2: lib-master: master_auth_request_full() now supports... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8fa5d235065b changeset: 19383:8fa5d235065b user: Timo Sirainen date: Tue Nov 17 17:23:31 2015 +0200 description: lib-master: master_auth_request_full() now supports per-request path. diffstat: src/lib-master/master-auth.c | 26 +++++++++++++++----------- src/lib-master/master-auth.h | 2 ++ 2 files changed, 17 insertions(+), 11 deletions(-) diffs (122 lines): diff -r b4a406dce5b8 -r 8fa5d235065b src/lib-master/master-auth.c --- a/src/lib-master/master-auth.c Tue Nov 17 17:11:05 2015 +0200 +++ b/src/lib-master/master-auth.c Tue Nov 17 17:23:31 2015 +0200 @@ -18,6 +18,7 @@ struct master_auth *auth; unsigned int tag; + char *path; int fd; struct io *io; struct timeout *to; @@ -33,7 +34,7 @@ struct master_service *service; pool_t pool; - const char *path; + const char *default_path; unsigned int tag_counter; HASH_TABLE(void *, struct master_auth_connection *) connections; @@ -49,7 +50,7 @@ auth = p_new(pool, struct master_auth, 1); auth->pool = pool; auth->service = service; - auth->path = p_strdup(pool, path); + auth->default_path = p_strdup(pool, path); hash_table_create_direct(&auth->connections, pool, 0); return auth; } @@ -74,9 +75,10 @@ io_remove(&conn->io); if (conn->fd != -1) { if (close(conn->fd) < 0) - i_fatal("close(%s) failed: %m", conn->auth->path); + i_fatal("close(%s) failed: %m", conn->path); conn->fd = -1; } + i_free(conn->path); i_free(conn); } @@ -110,11 +112,11 @@ if (ret == 0 || errno == ECONNRESET) { i_error("read(%s) failed: Remote closed connection " "(destination service { process_limit } reached?)", - conn->auth->path); + conn->path); } else { if (errno == EAGAIN) return; - i_error("read(%s) failed: %m", conn->auth->path); + i_error("read(%s) failed: %m", conn->path); } master_auth_connection_deinit(&conn); return; @@ -130,7 +132,7 @@ if (conn->tag != reply->tag) i_error("master(%s): Received reply with unknown tag %u", - conn->auth->path, reply->tag); + conn->path, reply->tag); else if (conn->callback == NULL) { /* request aborted */ } else { @@ -143,7 +145,7 @@ static void master_auth_connection_timeout(struct master_auth_connection *conn) { i_error("master(%s): Auth request timed out (received %u/%u bytes)", - conn->auth->path, conn->buf_pos, + conn->path, conn->buf_pos, (unsigned int)sizeof(conn->buf)); master_auth_connection_deinit(&conn); } @@ -166,6 +168,8 @@ conn->auth = auth; conn->callback = callback; conn->context = context; + conn->path = params->socket_path != NULL ? + i_strdup(params->socket_path) : i_strdup(auth->default_path); req = params->request; req.tag = ++auth->tag_counter; @@ -181,11 +185,11 @@ buffer_append(buf, &req, sizeof(req)); buffer_append(buf, params->data, req.data_size); - conn->fd = net_connect_unix_with_retries(auth->path, + conn->fd = net_connect_unix_with_retries(conn->path, SOCKET_CONNECT_RETRY_MSECS); if (conn->fd == -1) { i_error("net_connect_unix(%s) failed: %m%s", - auth->path, errno != EAGAIN ? "" : + conn->path, errno != EAGAIN ? "" : " - http://wiki2.dovecot.org/SocketUnavailable"); master_auth_connection_deinit(&conn); return; @@ -193,11 +197,11 @@ ret = fd_send(conn->fd, params->client_fd, buf->data, buf->used); if (ret < 0) { - i_error("fd_send(%s, %d) failed: %m", auth->path, + i_error("fd_send(%s, %d) failed: %m", conn->path, params->client_fd); } else if ((size_t)ret != buf->used) { i_error("fd_send(%s) sent only %d of %d bytes", - auth->path, (int)ret, (int)buf->used); + conn->path, (int)ret, (int)buf->used); ret = -1; } if (ret < 0) { diff -r b4a406dce5b8 -r 8fa5d235065b src/lib-master/master-auth.h --- a/src/lib-master/master-auth.h Tue Nov 17 17:11:05 2015 +0200 +++ b/src/lib-master/master-auth.h Tue Nov 17 17:23:31 2015 +0200 @@ -70,6 +70,8 @@ /* Client fd to transfer to post-login process or -1 if no fd is wanted to be transferred. */ int client_fd; + /* Override master_auth->default_path if non-NULL */ + const char *socket_path; /* Authentication request that is sent to post-login process. tag is ignored. */ From dovecot at dovecot.org Tue Nov 17 15:27:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 15:27:25 +0000 Subject: dovecot-2.2: *-login: Added postlogin_socket=path passdb extra f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/605dd1749578 changeset: 19384:605dd1749578 user: Timo Sirainen date: Tue Nov 17 17:26:56 2015 +0200 description: *-login: Added postlogin_socket=path passdb extra field. By default e.g. "imap" or "pop3" is the post-login socket, but this can override it. This could be used for example for per-user debugging (e.g. setting executable to be run via strace or valgrind). diffstat: src/login-common/client-common-auth.c | 5 +++-- src/login-common/client-common.h | 2 +- src/login-common/sasl-server.c | 5 +++++ 3 files changed, 9 insertions(+), 3 deletions(-) diffs (56 lines): diff -r 8fa5d235065b -r 605dd1749578 src/login-common/client-common-auth.c --- a/src/login-common/client-common-auth.c Tue Nov 17 17:23:31 2015 +0200 +++ b/src/login-common/client-common-auth.c Tue Nov 17 17:26:56 2015 +0200 @@ -133,8 +133,9 @@ PROXY_SSL_FLAG_STARTTLS; if (strcmp(value, "any-cert") == 0) reply_r->ssl_flags |= PROXY_SSL_FLAG_ANY_CERT; - } else if (strcmp(key, "user") == 0) { - /* already handled in login-common */ + } else if (strcmp(key, "user") == 0 || + strcmp(key, "postlogin_socket") == 0) { + /* already handled in sasl-server.c */ } else if (client->set->auth_debug) i_debug("Ignoring unknown passdb extra field: %s", key); } diff -r 8fa5d235065b -r 605dd1749578 src/login-common/client-common.h --- a/src/login-common/client-common.h Tue Nov 17 17:23:31 2015 +0200 +++ b/src/login-common/client-common.h Tue Nov 17 17:26:56 2015 +0200 @@ -113,7 +113,7 @@ struct ssl_proxy *ssl_proxy; const struct login_settings *set; const struct master_service_ssl_settings *ssl_set; - const char *session_id, *listener_name; + const char *session_id, *listener_name, *postlogin_socket_path; int fd; struct istream *input; diff -r 8fa5d235065b -r 605dd1749578 src/login-common/sasl-server.c --- a/src/login-common/sasl-server.c Tue Nov 17 17:23:31 2015 +0200 +++ b/src/login-common/sasl-server.c Tue Nov 17 17:26:56 2015 +0200 @@ -155,6 +155,7 @@ memset(¶ms, 0, sizeof(params)); params.client_fd = client->fd; + params.socket_path = client->postlogin_socket_path; params.request = req; params.data = buf->data; master_auth_request_full(master_auth, ¶ms, master_auth_callback, @@ -246,6 +247,7 @@ client->auth_request = NULL; client->auth_successes++; client->auth_passdb_args = p_strarray_dup(client->pool, args); + client->postlogin_socket_path = NULL; nologin = FALSE; for (i = 0; args[i] != NULL; i++) { @@ -261,6 +263,9 @@ i_free(client->virtual_auth_user); client->virtual_auth_user = i_strdup(args[i] + 10); + } else if (strncmp(args[i], "postlogin_socket=", 17) == 0) { + client->postlogin_socket_path = + p_strdup(client->pool, args[i] + 17); } else if (strcmp(args[i], "nologin") == 0 || strcmp(args[i], "proxy") == 0) { /* user can't login */ From dovecot at dovecot.org Tue Nov 17 17:02:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 17:02:40 +0000 Subject: dovecot-2.2: lib-fs: Fixed fs_stat() for fs-metawrap Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/392b4cd7a47a changeset: 19385:392b4cd7a47a user: Timo Sirainen date: Tue Nov 17 19:01:29 2015 +0200 description: lib-fs: Fixed fs_stat() for fs-metawrap diffstat: src/lib-fs/istream-metawrap.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 605dd1749578 -r 392b4cd7a47a src/lib-fs/istream-metawrap.c --- a/src/lib-fs/istream-metawrap.c Tue Nov 17 17:26:56 2015 +0200 +++ b/src/lib-fs/istream-metawrap.c Tue Nov 17 19:01:29 2015 +0200 @@ -100,7 +100,7 @@ if (mstream->in_metadata) { ret = i_stream_read(&stream->istream); - if (ret < 0) + if (ret < 0 && stream->istream.stream_errno != 0) return -1; if (ret == 0) { stream->statbuf.st_size = -1; From dovecot at dovecot.org Tue Nov 17 17:02:41 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Nov 2015 17:02:41 +0000 Subject: dovecot-2.2: lib-fs: Improved fs-metawrap's fs_stat() error mess... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f8dcf8b91282 changeset: 19386:f8dcf8b91282 user: Timo Sirainen date: Tue Nov 17 19:02:13 2015 +0200 description: lib-fs: Improved fs-metawrap's fs_stat() error message. diffstat: src/lib-fs/fs-metawrap.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 392b4cd7a47a -r f8dcf8b91282 src/lib-fs/fs-metawrap.c --- a/src/lib-fs/fs-metawrap.c Tue Nov 17 19:01:29 2015 +0200 +++ b/src/lib-fs/fs-metawrap.c Tue Nov 17 19:02:13 2015 +0200 @@ -471,8 +471,8 @@ input = fs_read_stream(_file, IO_BLOCK_SIZE); if ((ret = i_stream_get_size(input, TRUE, &input_size)) < 0) { - fs_set_error(_file->fs, "i_stream_get_size(%s) failed: %m", - fs_file_path(_file)); + fs_set_error(_file->fs, "i_stream_get_size(%s) failed: %s", + fs_file_path(_file), i_stream_get_error(input)); i_stream_unref(&input); return -1; } From dovecot at dovecot.org Wed Nov 18 10:41:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Nov 2015 10:41:09 +0000 Subject: dovecot-2.2: imap: When disconnected during FETCH, include the l... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b638e19d3bd4 changeset: 19387:b638e19d3bd4 user: Timo Sirainen date: Wed Nov 18 12:39:13 2015 +0200 description: imap: When disconnected during FETCH, include the last byte counts in disconnection log message. diffstat: src/imap/cmd-fetch.c | 10 ++++++++-- src/imap/imap-client.c | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diffs (32 lines): diff -r f8dcf8b91282 -r b638e19d3bd4 src/imap/cmd-fetch.c --- a/src/imap/cmd-fetch.c Tue Nov 17 19:02:13 2015 +0200 +++ b/src/imap/cmd-fetch.c Wed Nov 18 12:39:13 2015 +0200 @@ -192,8 +192,14 @@ const char *errstr; if (cmd->client->output->closed) { - client_disconnect(cmd->client, NULL); - return TRUE; + /* If we're canceling we need to finish this command + or we'll assert crash. But normally we want to + return FALSE so that the disconnect message logs + about this fetch command and that these latest + output bytes are included in it (which wouldn't + happen if we called client_disconnect() here + directly). */ + return cmd->cancel; } errstr = mailbox_get_last_error(cmd->client->mailbox, &error); diff -r f8dcf8b91282 -r b638e19d3bd4 src/imap/imap-client.c --- a/src/imap/imap-client.c Tue Nov 17 19:02:13 2015 +0200 +++ b/src/imap/imap-client.c Wed Nov 18 12:39:13 2015 +0200 @@ -1135,7 +1135,7 @@ o_stream_uncork(client->output); imap_refresh_proctitle(); - if (client->disconnected) + if (client->output->closed) client_destroy(client, NULL); else client_continue_pending_input(client); From dovecot at dovecot.org Wed Nov 18 10:41:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Nov 2015 10:41:10 +0000 Subject: dovecot-2.2: imap: Disconnection log message incorrectly logged ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3d209e0982bf changeset: 19388:3d209e0982bf user: Timo Sirainen date: Wed Nov 18 12:40:42 2015 +0200 description: imap: Disconnection log message incorrectly logged "input/output" even if only "input" was set. diffstat: src/imap/imap-client.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r b638e19d3bd4 -r 3d209e0982bf src/imap/imap-client.c --- a/src/imap/imap-client.c Wed Nov 18 12:39:13 2015 +0200 +++ b/src/imap/imap-client.c Wed Nov 18 12:40:42 2015 +0200 @@ -288,7 +288,7 @@ } cond = io_loop_find_fd_conditions(current_ioloop, client->fd_out); - if ((cond & (IO_READ | IO_WRITE)) != 0) + if ((cond & (IO_READ | IO_WRITE)) == (IO_READ | IO_WRITE)) cond_str = "input/output"; else if ((cond & IO_READ) != 0) cond_str = "input"; From dovecot at dovecot.org Wed Nov 18 11:12:49 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Nov 2015 11:12:49 +0000 Subject: dovecot-2.2: pop3: Send "OK Logged in" before reading mailbox. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5528fd720619 changeset: 19389:5528fd720619 user: Timo Sirainen date: Wed Nov 18 13:08:57 2015 +0200 description: pop3: Send "OK Logged in" before reading mailbox. This way if the reading takes a long time, the client still sees that the login itself was successful. This is especially useful to avoid unnecessary errors logged by proxies. diffstat: src/pop3/main.c | 13 +++++- src/pop3/pop3-client.c | 90 +++++++++++++++++++++++++------------------------ src/pop3/pop3-client.h | 1 + 3 files changed, 57 insertions(+), 47 deletions(-) diffs (172 lines): diff -r 3d209e0982bf -r 5528fd720619 src/pop3/main.c --- a/src/pop3/main.c Wed Nov 18 12:40:42 2015 +0200 +++ b/src/pop3/main.c Wed Nov 18 13:08:57 2015 +0200 @@ -85,8 +85,6 @@ output = client->output; o_stream_ref(output); o_stream_cork(output); - if (!IS_STANDALONE()) - client_send_line(client, "+OK Logged in."); (void)client_handle_input(client); o_stream_uncork(output); o_stream_unref(&output); @@ -103,6 +101,7 @@ struct mail_user *mail_user; struct client *client; const struct pop3_settings *set; + const char *error; if (mail_storage_service_lookup_next(storage_service, input, &user, &mail_user, error_r) <= 0) { @@ -118,8 +117,16 @@ verbose_proctitle = TRUE; if (client_create(fd_in, fd_out, input->session_id, - mail_user, user, set, &client) == 0) + mail_user, user, set, &client) < 0) + return 0; + if (!IS_STANDALONE()) + client_send_line(client, "+OK Logged in."); + if (client_init_mailbox(client, &error) == 0) client_add_input(client, input_buf); + else { + i_error("%s", error); + client_destroy(client, error); + } return 0; } diff -r 3d209e0982bf -r 5528fd720619 src/pop3/pop3-client.c --- a/src/pop3/pop3-client.c Wed Nov 18 12:40:42 2015 +0200 +++ b/src/pop3/pop3-client.c Wed Nov 18 13:08:57 2015 +0200 @@ -369,11 +369,7 @@ struct mail_storage_service_user *service_user, const struct pop3_settings *set, struct client **client_r) { - struct mail_storage *storage; - const char *ident; struct client *client; - enum mailbox_flags flags; - const char *errmsg; pool_t pool; int ret; @@ -405,41 +401,7 @@ client->user = user; - pop3_client_count++; - DLLIST_PREPEND(&pop3_clients, client); - - client->inbox_ns = mail_namespace_find_inbox(user->namespaces); - i_assert(client->inbox_ns != NULL); - - if (set->pop3_lock_session && (ret = pop3_lock_session(client)) <= 0) { - client_send_line(client, ret < 0 ? - "-ERR [SYS/TEMP] Failed to create POP3 session lock." : - "-ERR [IN-USE] Mailbox is locked by another POP3 session."); - client_destroy(client, "Couldn't lock POP3 session"); - return -1; - } - - flags = MAILBOX_FLAG_POP3_SESSION; - if (!set->pop3_no_flag_updates) - flags |= MAILBOX_FLAG_DROP_RECENT; - client->mailbox = mailbox_alloc(client->inbox_ns->list, "INBOX", flags); - storage = mailbox_get_storage(client->mailbox); - if (mailbox_open(client->mailbox) < 0) { - i_error("Couldn't open INBOX: %s", - mailbox_get_last_error(client->mailbox, NULL)); - client_send_storage_error(client); - client_destroy(client, "Couldn't open INBOX"); - return -1; - } - client->mail_set = mail_storage_get_settings(storage); - - if (init_pop3_deleted_flag(client, &errmsg) < 0 || - init_mailbox(client, &errmsg) < 0) { - i_error("Couldn't init INBOX: %s", errmsg); - client_destroy(client, "Mailbox init failed"); - return -1; - } - + client->mail_set = mail_user_set_get_storage_set(user); client->uidl_keymask = parse_uidl_keymask(client->mail_set->pop3_uidl_format); if (client->uidl_keymask == 0) @@ -454,7 +416,51 @@ client->message_uidls_save = TRUE; } - if (!set->pop3_no_flag_updates && client->messages_count > 0) + pop3_client_count++; + DLLIST_PREPEND(&pop3_clients, client); + + client->inbox_ns = mail_namespace_find_inbox(user->namespaces); + i_assert(client->inbox_ns != NULL); + + if (hook_client_created != NULL) + hook_client_created(&client); + + if (set->pop3_lock_session && (ret = pop3_lock_session(client)) <= 0) { + client_send_line(client, ret < 0 ? + "-ERR [SYS/TEMP] Failed to create POP3 session lock." : + "-ERR [IN-USE] Mailbox is locked by another POP3 session."); + client_destroy(client, "Couldn't lock POP3 session"); + return -1; + } + + *client_r = client; + return 0; + +} + +int client_init_mailbox(struct client *client, const char **error_r) +{ + enum mailbox_flags flags; + const char *ident, *errmsg; + + flags = MAILBOX_FLAG_POP3_SESSION; + if (!client->set->pop3_no_flag_updates) + flags |= MAILBOX_FLAG_DROP_RECENT; + client->mailbox = mailbox_alloc(client->inbox_ns->list, "INBOX", flags); + if (mailbox_open(client->mailbox) < 0) { + *error_r = t_strdup_printf("Couldn't open INBOX: %s", + mailbox_get_last_error(client->mailbox, NULL)); + client_send_storage_error(client); + return -1; + } + + if (init_pop3_deleted_flag(client, &errmsg) < 0 || + init_mailbox(client, &errmsg) < 0) { + *error_r = t_strdup_printf("Couldn't init INBOX: %s", errmsg); + return -1; + } + + if (!client->set->pop3_no_flag_updates && client->messages_count > 0) client->seen_bitmask = i_malloc(MSGS_BITMASK_SIZE(client)); ident = mail_user_get_anvil_userip_ident(client->user); @@ -464,11 +470,7 @@ client->anvil_sent = TRUE; } - if (hook_client_created != NULL) - hook_client_created(&client); - pop3_refresh_proctitle(); - *client_r = client; return 0; } diff -r 3d209e0982bf -r 5528fd720619 src/pop3/pop3-client.h --- a/src/pop3/pop3-client.h Wed Nov 18 12:40:42 2015 +0200 +++ b/src/pop3/pop3-client.h Wed Nov 18 13:08:57 2015 +0200 @@ -123,6 +123,7 @@ struct mail_user *user, struct mail_storage_service_user *service_user, const struct pop3_settings *set, struct client **client_r); +int client_init_mailbox(struct client *client, const char **error_r); void client_destroy(struct client *client, const char *reason) ATTR_NULL(2); /* Disconnect client connection */ From dovecot at dovecot.org Wed Nov 18 11:12:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Nov 2015 11:12:50 +0000 Subject: dovecot-2.2: pop3: When creating session lock file, prefer to wr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fa46e882e8bf changeset: 19390:fa46e882e8bf user: Timo Sirainen date: Wed Nov 18 13:12:23 2015 +0200 description: pop3: When creating session lock file, prefer to write it to the index root dir. This is especially important for mail storage backends that don't have a mail root directory. diffstat: src/pop3/pop3-client.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (15 lines): diff -r 5528fd720619 -r fa46e882e8bf src/pop3/pop3-client.c --- a/src/pop3/pop3-client.c Wed Nov 18 13:08:57 2015 +0200 +++ b/src/pop3/pop3-client.c Wed Nov 18 13:12:23 2015 +0200 @@ -339,9 +339,9 @@ int ret; if (!mailbox_list_get_root_path(client->inbox_ns->list, - MAILBOX_LIST_PATH_TYPE_DIR, &dir) && + MAILBOX_LIST_PATH_TYPE_INDEX, &dir) && !mailbox_list_get_root_path(client->inbox_ns->list, - MAILBOX_LIST_PATH_TYPE_INDEX, &dir)) { + MAILBOX_LIST_PATH_TYPE_DIR, &dir)) { i_error("pop3_lock_session: Storage has no root/index directory, " "can't create a POP3 session lock file"); return -1; From dovecot at dovecot.org Wed Nov 18 12:16:00 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Nov 2015 12:16:00 +0000 Subject: dovecot-2.2: lib-master: Fixed timeout leak at deinit when we co... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f78c312bc5a9 changeset: 19391:f78c312bc5a9 user: Timo Sirainen date: Wed Nov 18 14:15:32 2015 +0200 description: lib-master: Fixed timeout leak at deinit when we couldn't connect to ipc-server diffstat: src/lib-master/ipc-server.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r fa46e882e8bf -r f78c312bc5a9 src/lib-master/ipc-server.c --- a/src/lib-master/ipc-server.c Wed Nov 18 13:12:23 2015 +0200 +++ b/src/lib-master/ipc-server.c Wed Nov 18 14:15:32 2015 +0200 @@ -154,6 +154,8 @@ i_assert(server->ipc_cmd_refcount == 0); ipc_server_disconnect(server); + if (server->to != NULL) + timeout_remove(&server->to); i_free(server->name); i_free(server->path); i_free(server); From dovecot at dovecot.org Wed Nov 18 15:46:04 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Nov 2015 15:46:04 +0000 Subject: dovecot-2.2: lib-master: master_service_is_master_stopped() now ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/52cbfa793147 changeset: 19392:52cbfa793147 user: Timo Sirainen date: Wed Nov 18 17:45:35 2015 +0200 description: lib-master: master_service_is_master_stopped() now returns FALSE if running standalone. The only caller of this function was stats plugin to see if it should send stats updates. So this fixes dovecot-lda, doveadm and other standalone tools to send stats updates. diffstat: src/lib-master/master-service.c | 3 ++- src/lib-master/master-service.h | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diffs (26 lines): diff -r f78c312bc5a9 -r 52cbfa793147 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Wed Nov 18 14:15:32 2015 +0200 +++ b/src/lib-master/master-service.c Wed Nov 18 17:45:35 2015 +0200 @@ -682,7 +682,8 @@ bool master_service_is_master_stopped(struct master_service *service) { - return service->io_status_error == NULL; + return service->io_status_error == NULL && + (service->flags & MASTER_SERVICE_FLAG_STANDALONE) == 0; } void master_service_anvil_send(struct master_service *service, const char *cmd) diff -r f78c312bc5a9 -r 52cbfa793147 src/lib-master/master-service.h --- a/src/lib-master/master-service.h Wed Nov 18 14:15:32 2015 +0200 +++ b/src/lib-master/master-service.h Wed Nov 18 17:45:35 2015 +0200 @@ -155,7 +155,8 @@ /* Returns TRUE if we've received a SIGINT/SIGTERM and we've decided to stop. */ bool master_service_is_killed(struct master_service *service); /* Returns TRUE if our master process is already stopped. This process may or - may not be dying itself. */ + may not be dying itself. Returns FALSE always if the process was started + standalone. */ bool master_service_is_master_stopped(struct master_service *service); /* Send command to anvil process, if we have fd to it. */ From dovecot at dovecot.org Wed Nov 18 16:11:12 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Nov 2015 16:11:12 +0000 Subject: dovecot-2.2: lib-storage: Create ioloop context for users even i... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/910656f4b7a0 changeset: 19393:910656f4b7a0 user: Timo Sirainen date: Wed Nov 18 18:10:46 2015 +0200 description: lib-storage: Create ioloop context for users even if log prefix changing is disabled. This fixes stats plugin to work with lmtp, imap-urlauth and doveadm-server. diffstat: src/lib-storage/mail-storage-service.c | 17 +++++++++++------ 1 files changed, 11 insertions(+), 6 deletions(-) diffs (41 lines): diff -r 52cbfa793147 -r 910656f4b7a0 src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Wed Nov 18 17:45:35 2015 +0200 +++ b/src/lib-storage/mail-storage-service.c Wed Nov 18 18:10:46 2015 +0200 @@ -779,7 +779,6 @@ if (master_service_get_client_limit(master_service) == 1) i_set_failure_send_prefix(user->log_prefix); - user->ioloop_ctx = io_loop_context_new(current_ioloop); io_loop_context_add_callbacks(user->ioloop_ctx, mail_storage_service_io_activate_user, mail_storage_service_io_deactivate_user, @@ -1342,6 +1341,10 @@ set_keyval(ctx, user, "mail_home", priv.home); } + /* create ioloop context regardless of logging. it's also used by + stats plugin. */ + user->ioloop_ctx = io_loop_context_new(current_ioloop); + if ((user->flags & MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT) == 0) mail_storage_service_init_log(ctx, user, &priv); @@ -1430,11 +1433,13 @@ *_user = NULL; if (user->ioloop_ctx != NULL) { - io_loop_context_remove_callbacks(user->ioloop_ctx, - mail_storage_service_io_activate_user, - mail_storage_service_io_deactivate_user, user); - if (io_loop_get_current_context(current_ioloop) == user->ioloop_ctx) - mail_storage_service_io_deactivate_user(user); + if ((user->flags & MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT) == 0) { + io_loop_context_remove_callbacks(user->ioloop_ctx, + mail_storage_service_io_activate_user, + mail_storage_service_io_deactivate_user, user); + if (io_loop_get_current_context(current_ioloop) == user->ioloop_ctx) + mail_storage_service_io_deactivate_user(user); + } io_loop_context_unref(&user->ioloop_ctx); } settings_parser_deinit(&user->set_parser); From dovecot at dovecot.org Wed Nov 18 17:16:05 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Nov 2015 17:16:05 +0000 Subject: dovecot-2.2: stats: When logging UPDATE-SESSION stats, log also ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/109c23a7412a changeset: 19394:109c23a7412a user: Timo Sirainen date: Wed Nov 18 18:54:51 2015 +0200 description: stats: When logging UPDATE-SESSION stats, log also the session ID. diffstat: src/stats/mail-session.c | 13 +++++++------ 1 files changed, 7 insertions(+), 6 deletions(-) diffs (37 lines): diff -r 910656f4b7a0 -r 109c23a7412a src/stats/mail-session.c --- a/src/stats/mail-session.c Wed Nov 18 18:10:46 2015 +0200 +++ b/src/stats/mail-session.c Wed Nov 18 18:54:51 2015 +0200 @@ -267,9 +267,9 @@ buf = buffer_create_dynamic(pool_datastack_create(), 256); if (args[1] == NULL || base64_decode(args[1], strlen(args[1]), NULL, buf) < 0) { - *error_r = t_strdup_printf("UPDATE-SESSION %s %s: Invalid base64 input", + *error_r = t_strdup_printf("UPDATE-SESSION %s %s %s: Invalid base64 input", session->user->name, - session->service); + session->service, session->id); return -1; } @@ -277,16 +277,17 @@ diff_stats = stats_alloc(pool_datastack_create()); if (!stats_import(buf->data, buf->used, session->stats, new_stats, &error)) { - *error_r = t_strdup_printf("UPDATE-SESSION %s %s: %s", + *error_r = t_strdup_printf("UPDATE-SESSION %s %s %s: %s", session->user->name, - session->service, error); + session->service, session->id, error); return -1; } if (!stats_diff(session->stats, new_stats, diff_stats, &error)) { - *error_r = t_strdup_printf("UPDATE-SESSION %s %s: stats shrank: %s", + *error_r = t_strdup_printf("UPDATE-SESSION %s %s %s: stats shrank: %s", session->user->name, - session->service, error); + session->service, session->id, error); + i_panic("%s", *error_r); return -1; } mail_session_refresh(session, diff_stats); From dovecot at dovecot.org Wed Nov 18 17:16:06 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Nov 2015 17:16:06 +0000 Subject: dovecot-2.2: stats: Forced sending of UPDATE-SESSION every 5 min... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dc08ba342085 changeset: 19395:dc08ba342085 user: Timo Sirainen date: Wed Nov 18 19:15:36 2015 +0200 description: stats: Forced sending of UPDATE-SESSION every 5 minutes wasn't working. This caused stats process to forget about idling (imap) sessions if they didn't do anything for over 15 minutes. diffstat: src/plugins/stats/stats-plugin.c | 15 ++++++--------- 1 files changed, 6 insertions(+), 9 deletions(-) diffs (32 lines): diff -r 109c23a7412a -r dc08ba342085 src/plugins/stats/stats-plugin.c --- a/src/plugins/stats/stats-plugin.c Wed Nov 18 18:54:51 2015 +0200 +++ b/src/plugins/stats/stats-plugin.c Wed Nov 18 19:15:36 2015 +0200 @@ -103,22 +103,19 @@ } *changed_r = FALSE; + diff = now - suser->last_session_update; + if (diff >= SESSION_STATS_FORCE_REFRESH_SECS) + return TRUE; + *to_next_secs_r = SESSION_STATS_FORCE_REFRESH_SECS - diff; + if (!suser->session_sent_duplicate) { if (suser->last_session_update != now) { /* send one duplicate notification so stats reader knows that this session is idle now */ return TRUE; } - *to_next_secs_r = 1; - return FALSE; } - - diff = now - suser->last_session_update; - if (diff < SESSION_STATS_FORCE_REFRESH_SECS) { - *to_next_secs_r = SESSION_STATS_FORCE_REFRESH_SECS - diff; - return FALSE; - } - return TRUE; + return FALSE; } static void session_stats_refresh(struct mail_user *user) From dovecot at dovecot.org Wed Nov 18 18:51:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Nov 2015 18:51:22 +0000 Subject: dovecot-2.2: stats: Improved logging on invalid CONNECT input. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/626758356cce changeset: 19396:626758356cce user: Timo Sirainen date: Wed Nov 18 20:50:55 2015 +0200 description: stats: Improved logging on invalid CONNECT input. diffstat: src/stats/mail-session.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (22 lines): diff -r dc08ba342085 -r 626758356cce src/stats/mail-session.c --- a/src/stats/mail-session.c Wed Nov 18 19:15:36 2015 +0200 +++ b/src/stats/mail-session.c Wed Nov 18 20:50:55 2015 +0200 @@ -84,13 +84,16 @@ } session_id = args[0]; if (str_to_pid(args[3], &pid) < 0) { - *error_r = "CONNECT: Invalid pid"; + *error_r = t_strdup_printf("CONNECT: Invalid pid %s for session ID %s", + args[3], session_id); return -1; } session = hash_table_lookup(mail_sessions_hash, session_id); if (session != NULL) { - *error_r = "CONNECT: Duplicate session ID"; + *error_r = t_strdup_printf( + "CONNECT: Duplicate session ID %s for user %s service %s", + session_id, args[1], args[2]); return -1; } session = i_malloc(sizeof(struct mail_session) + stats_alloc_size()); From dovecot at dovecot.org Wed Nov 18 19:06:48 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Nov 2015 19:06:48 +0000 Subject: dovecot-2.2: lib-storage: Added mail_storage_service_input.sessi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e976d6f988ec changeset: 19397:e976d6f988ec user: Timo Sirainen date: Wed Nov 18 21:03:41 2015 +0200 description: lib-storage: Added mail_storage_service_input.session_id_prefix This should make it easier to pass through the original session_id to worker process sessions. Especially if these sessions are somewhat short-lived it can be useful to assign them a unique session ID while still being able to match it to the parent's session ID. diffstat: src/lib-storage/mail-storage-service.c | 19 ++++++++++++++----- src/lib-storage/mail-storage-service.h | 4 ++++ 2 files changed, 18 insertions(+), 5 deletions(-) diffs (62 lines): diff -r 626758356cce -r e976d6f988ec src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Wed Nov 18 20:50:55 2015 +0200 +++ b/src/lib-storage/mail-storage-service.c Wed Nov 18 21:03:41 2015 +0200 @@ -1071,10 +1071,15 @@ i_set_failure_prefix("%s", str_c(str)); } -static const char *mail_storage_service_generate_session_id(pool_t pool) +static const char * +mail_storage_service_generate_session_id(pool_t pool, const char *prefix) { guid_128_t guid; - string_t *str = str_new(pool, MAX_BASE64_ENCODED_SIZE(sizeof(guid))); + unsigned int prefix_len = prefix == NULL ? 0 : strlen(prefix); + string_t *str = str_new(pool, MAX_BASE64_ENCODED_SIZE(prefix_len + 1 + sizeof(guid))); + + if (prefix != NULL) + str_printfa(str, "%s-", prefix); guid_128_generate(guid); base64_encode(guid, sizeof(guid), str); @@ -1187,7 +1192,8 @@ user->input.session_id = p_strdup(user_pool, input->session_id); if (user->input.session_id == NULL) { user->input.session_id = - mail_storage_service_generate_session_id(user_pool); + mail_storage_service_generate_session_id(user_pool, + input->session_id_prefix); } user->user_info = user_info; user->flags = flags; @@ -1250,10 +1256,13 @@ /* no user yet. log prefix should be just "imap:" or something equally unhelpful. we don't know the proper log format yet, but initialize it to something better until we know it. */ + const char *session_id = + input->session_id != NULL ? input->session_id : + (input->session_id_prefix != NULL ? + input->session_id_prefix : NULL); i_set_failure_prefix("%s(%s%s,%s)", master_service_get_name(ctx->service), input->username, - input->session_id == NULL ? "" : - t_strdup_printf(",%s", input->session_id), + session_id == NULL ? "" : t_strdup_printf(",%s", session_id), input->remote_ip.family == 0 ? "" : t_strdup_printf(",%s", net_ip2addr(&input->remote_ip))); update_log_prefix = TRUE; diff -r 626758356cce -r e976d6f988ec src/lib-storage/mail-storage-service.h --- a/src/lib-storage/mail-storage-service.h Wed Nov 18 20:50:55 2015 +0200 +++ b/src/lib-storage/mail-storage-service.h Wed Nov 18 21:03:41 2015 +0200 @@ -40,7 +40,11 @@ const char *module; const char *service; const char *username; + /* If set, use this string as the session ID */ const char *session_id; + /* If set, use this string as the session ID prefix, but also append + a unique session ID suffix to it. */ + const char *session_id_prefix; struct ip_addr local_ip, remote_ip; in_port_t local_port, remote_port; From dovecot at dovecot.org Wed Nov 18 19:06:48 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Nov 2015 19:06:48 +0000 Subject: dovecot-2.2: indexer-worker: Use provided session-id only as a p... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cd347fd1791a changeset: 19398:cd347fd1791a user: Timo Sirainen date: Wed Nov 18 21:06:17 2015 +0200 description: indexer-worker: Use provided session-id only as a prefix for a unique session ID. Fixes stats process's "Duplicate session ID" errors when LMTP delivers to multiple recipients. diffstat: src/indexer/master-connection.c | 11 ++++++----- 1 files changed, 6 insertions(+), 5 deletions(-) diffs (21 lines): diff -r e976d6f988ec -r cd347fd1791a src/indexer/master-connection.c --- a/src/indexer/master-connection.c Wed Nov 18 21:03:41 2015 +0200 +++ b/src/indexer/master-connection.c Wed Nov 18 21:06:17 2015 +0200 @@ -212,11 +212,12 @@ input.module = "mail"; input.service = "indexer-worker"; input.username = args[0]; - /* if session-id is given, add -idx suffix to it so that stats process - doesn't complain about duplicates. also it's nicer to keep the stats - separate for the indexer and the caller. */ - input.session_id = args[2][0] == '\0' ? NULL : - t_strconcat(args[2], "-idx", NULL); + /* if session-id is given, use it as a prefix to a unique session ID. + we can't use the session-id directly or stats process will complain + about duplicates. (especially LMTP would use the same session-id for + multiple users' indexing at the same time.) */ + if (args[2][0] != '\0') + input.session_id_prefix = args[2]; if (mail_storage_service_lookup_next(conn->storage_service, &input, &service_user, &user, &error) <= 0) { From pigeonhole at rename-it.nl Wed Nov 18 21:11:39 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 18 Nov 2015 22:11:39 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Moved handling of implicit ke... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/d23225e408af changeset: 2133:d23225e408af user: Stephan Bosch date: Wed Nov 18 22:11:31 2015 +0100 description: lib-sieve: Moved handling of implicit keep during multiscript execution outside the script sequence itself. Before, implicit keep was executed as part of the final script execution. This caused the implicit keep to be executed as part of that script, including global context. This caused insignificant errors during delivery to be logged as errors in the administrator log when the last script is global. diffstat: src/lib-sieve/sieve-result.c | 8 ++++- src/lib-sieve/sieve.c | 46 ++++++++++++++++--------------- src/lib-sieve/sieve.h | 6 ++-- src/plugins/lda-sieve/lda-sieve-plugin.c | 7 ++-- src/sieve-tools/sieve-test.c | 4 +- src/testsuite/testsuite-script.c | 3 +- 6 files changed, 40 insertions(+), 34 deletions(-) diffs (228 lines): diff -r f2c5a9f92e0d -r d23225e408af src/lib-sieve/sieve-result.c --- a/src/lib-sieve/sieve-result.c Sun Nov 15 22:46:01 2015 +0100 +++ b/src/lib-sieve/sieve-result.c Wed Nov 18 22:11:31 2015 +0100 @@ -1033,9 +1033,15 @@ (struct sieve_result *result, struct sieve_error_handler *ehandler) { + int ret; + _sieve_result_prepare_execution(result, ehandler); - return _sieve_result_implicit_keep(result, TRUE); + ret = _sieve_result_implicit_keep(result, TRUE); + + result->action_env.ehandler = NULL; + + return ret; } void sieve_result_mark_executed(struct sieve_result *result) diff -r f2c5a9f92e0d -r d23225e408af src/lib-sieve/sieve.c --- a/src/lib-sieve/sieve.c Sun Nov 15 22:46:01 2015 +0100 +++ b/src/lib-sieve/sieve.c Wed Nov 18 22:11:31 2015 +0100 @@ -664,13 +664,10 @@ (struct sieve_multiscript *mscript, struct sieve_binary *sbin, struct sieve_error_handler *exec_ehandler, struct sieve_error_handler *action_ehandler, - enum sieve_runtime_flags flags, bool final) + enum sieve_runtime_flags flags) { if ( !mscript->active ) return FALSE; - if ( final ) - sieve_result_set_keep_action(mscript->result, NULL, &act_store); - /* Run the script */ mscript->status = sieve_run(sbin, &mscript->result, mscript->msgdata, mscript->scriptenv, exec_ehandler, flags); @@ -685,7 +682,7 @@ (mscript, action_ehandler, &mscript->keep); } mscript->active = - ( mscript->active && mscript->keep && !final && mscript->status > 0 ); + ( mscript->active && mscript->keep && mscript->status > 0 ); } if ( mscript->status <= 0 ) @@ -699,16 +696,20 @@ return mscript->status; } -int sieve_multiscript_tempfail(struct sieve_multiscript **mscript, +int sieve_multiscript_tempfail(struct sieve_multiscript **_mscript, struct sieve_error_handler *action_ehandler) { - struct sieve_result *result = (*mscript)->result; - int ret = (*mscript)->status; + struct sieve_multiscript *mscript = *_mscript; + struct sieve_result *result = mscript->result; + int ret = mscript->status; - if ( (*mscript)->active ) { + sieve_result_set_keep_action + (mscript->result, NULL, &act_store); + + if ( mscript->active ) { ret = SIEVE_EXEC_TEMP_FAILURE; - if ( !(*mscript)->teststream && sieve_result_executed(result) ) { + if ( !mscript->teststream && sieve_result_executed(result) ) { /* Part of the result is already executed, need to fall back to * to implicit keep (FIXME) */ @@ -725,27 +726,29 @@ /* Cleanup */ sieve_result_unref(&result); - *mscript = NULL; + *_mscript = NULL; return ret; } -int sieve_multiscript_finish(struct sieve_multiscript **mscript, +int sieve_multiscript_finish(struct sieve_multiscript **_mscript, struct sieve_error_handler *action_ehandler, bool *keep) { - struct sieve_result *result = (*mscript)->result; - int ret = (*mscript)->status; + struct sieve_multiscript *mscript = *_mscript; + struct sieve_result *result = mscript->result; + int ret = mscript->status; - if ( (*mscript)->active ) { - ret = SIEVE_EXEC_FAILURE; + sieve_result_set_keep_action + (mscript->result, NULL, &act_store); - if ( (*mscript)->teststream ) { - (*mscript)->keep = TRUE; + if ( mscript->active ) { + if ( mscript->teststream ) { + mscript->keep = TRUE; } else { switch ( sieve_result_implicit_keep (result, action_ehandler) ) { case SIEVE_EXEC_OK: - (*mscript)->keep = TRUE; + mscript->keep = TRUE; break; case SIEVE_EXEC_TEMP_FAILURE: if (!sieve_result_executed(result)) { @@ -759,12 +762,11 @@ } } - if ( keep != NULL ) *keep = (*mscript)->keep; + if ( keep != NULL ) *keep = mscript->keep; /* Cleanup */ sieve_result_unref(&result); - *mscript = NULL; - + *_mscript = NULL; return ret; } diff -r f2c5a9f92e0d -r d23225e408af src/lib-sieve/sieve.h --- a/src/lib-sieve/sieve.h Sun Nov 15 22:46:01 2015 +0100 +++ b/src/lib-sieve/sieve.h Wed Nov 18 22:11:31 2015 +0100 @@ -192,15 +192,15 @@ (struct sieve_multiscript *mscript, struct sieve_binary *sbin, struct sieve_error_handler *exec_ehandler, struct sieve_error_handler *action_ehandler, - enum sieve_runtime_flags flags, bool final); + enum sieve_runtime_flags flags); int sieve_multiscript_status(struct sieve_multiscript *mscript); int sieve_multiscript_tempfail - (struct sieve_multiscript **mscript, + (struct sieve_multiscript **_mscript, struct sieve_error_handler *action_ehandler); int sieve_multiscript_finish - (struct sieve_multiscript **mscript, + (struct sieve_multiscript **_mscript, struct sieve_error_handler *action_ehandler, bool *keep); /* diff -r f2c5a9f92e0d -r d23225e408af src/plugins/lda-sieve/lda-sieve-plugin.c --- a/src/plugins/lda-sieve/lda-sieve-plugin.c Sun Nov 15 22:46:01 2015 +0100 +++ b/src/plugins/lda-sieve/lda-sieve-plugin.c Wed Nov 18 22:11:31 2015 +0100 @@ -528,7 +528,6 @@ struct sieve_script *script = scripts[i]; enum sieve_compile_flags cpflags = 0; enum sieve_runtime_flags rtflags = 0; - bool final = ( i == count - 1 ); user_script = ( script == srctx->user_script ); last_script = script; @@ -565,7 +564,7 @@ action_ehandler = lda_sieve_log_ehandler_create (exec_ehandler, mdctx); more = sieve_multiscript_run(mscript, sbin, - exec_ehandler, action_ehandler, rtflags, final); + exec_ehandler, action_ehandler, rtflags); sieve_error_handler_unref(&action_ehandler); if ( !more ) { @@ -589,7 +588,7 @@ action_ehandler = lda_sieve_log_ehandler_create (exec_ehandler, mdctx); more = sieve_multiscript_run(mscript, sbin, - exec_ehandler, action_ehandler, rtflags, final); + exec_ehandler, action_ehandler, rtflags); sieve_error_handler_unref(&action_ehandler); /* Save new version */ @@ -606,7 +605,7 @@ /* Finish execution */ action_ehandler = lda_sieve_log_ehandler_create - (exec_ehandler, mdctx); + (srctx->user_ehandler, mdctx); if ( compile_error && error == SIEVE_ERROR_TEMP_FAILURE ) ret = sieve_multiscript_tempfail(&mscript, action_ehandler); else diff -r f2c5a9f92e0d -r d23225e408af src/sieve-tools/sieve-test.c --- a/src/sieve-tools/sieve-test.c Sun Nov 15 22:46:01 2015 +0100 +++ b/src/sieve-tools/sieve-test.c Wed Nov 18 22:11:31 2015 +0100 @@ -362,7 +362,7 @@ /* Execute/Test script */ more = sieve_multiscript_run(mscript, sbin, - ehandler, action_ehandler, 0, FALSE); + ehandler, action_ehandler, 0); } /* Execute/Test main script */ @@ -379,7 +379,7 @@ main_sbin = NULL; (void)sieve_multiscript_run(mscript, sbin, - ehandler, ehandler, 0, TRUE); + ehandler, ehandler, 0); } result = sieve_multiscript_finish(&mscript, ehandler, NULL); diff -r f2c5a9f92e0d -r d23225e408af src/testsuite/testsuite-script.c --- a/src/testsuite/testsuite-script.c Sun Nov 15 22:46:01 2015 +0100 +++ b/src/testsuite/testsuite-script.c Wed Nov 18 22:11:31 2015 +0100 @@ -203,7 +203,6 @@ for ( i = 0; i < count && more; i++ ) { struct sieve_binary *sbin = NULL; const char *script = scripts[i]; - bool final = ( i == count - 1 ); /* Open */ if ( (sbin=_testsuite_script_compile(renv, script)) == NULL ) { @@ -216,7 +215,7 @@ sieve_runtime_trace(renv, SIEVE_TRLVL_TESTS, "run script `%s'", script); more = sieve_multiscript_run(mscript, sbin, - testsuite_log_ehandler, testsuite_log_ehandler, 0, final); + testsuite_log_ehandler, testsuite_log_ehandler, 0); sieve_close(&sbin); } From dovecot at dovecot.org Thu Nov 19 12:47:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 19 Nov 2015 12:47:56 +0000 Subject: dovecot-2.2: lib-fs: Added fs_stats_get_read/write_usecs() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fac18fa49f2d changeset: 19399:fac18fa49f2d user: Timo Sirainen date: Thu Nov 19 14:46:54 2015 +0200 description: lib-fs: Added fs_stats_get_read/write_usecs() These can be easily used to sum up all the timings for read and write categories. diffstat: src/lib-fs/fs-api.c | 18 ++++++++++++++++++ src/lib-fs/fs-api.h | 4 ++++ 2 files changed, 22 insertions(+), 0 deletions(-) diffs (37 lines): diff -r cd347fd1791a -r fac18fa49f2d src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Wed Nov 18 21:06:17 2015 +0200 +++ b/src/lib-fs/fs-api.c Thu Nov 19 14:46:54 2015 +0200 @@ -987,3 +987,21 @@ fs_set_error(fs, "Asynchronous operation in progress"); errno = EAGAIN; } + +uint64_t fs_stats_get_read_usecs(const struct fs_stats *stats) +{ + return timing_get_sum(stats->timings[FS_OP_METADATA]) + + timing_get_sum(stats->timings[FS_OP_PREFETCH]) + + timing_get_sum(stats->timings[FS_OP_READ]) + + timing_get_sum(stats->timings[FS_OP_EXISTS]) + + timing_get_sum(stats->timings[FS_OP_STAT]) + + timing_get_sum(stats->timings[FS_OP_ITER]); +} + +uint64_t fs_stats_get_write_usecs(const struct fs_stats *stats) +{ + return timing_get_sum(stats->timings[FS_OP_WRITE]) + + timing_get_sum(stats->timings[FS_OP_COPY]) + + timing_get_sum(stats->timings[FS_OP_DELETE]); +} + diff -r cd347fd1791a -r fac18fa49f2d src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Wed Nov 18 21:06:17 2015 +0200 +++ b/src/lib-fs/fs-api.h Thu Nov 19 14:46:54 2015 +0200 @@ -333,4 +333,8 @@ filesystem whose stats you want to see. */ const struct fs_stats *fs_get_stats(struct fs *fs); +/* Helper functions to count number of usecs for read/write operations. */ +uint64_t fs_stats_get_read_usecs(const struct fs_stats *stats); +uint64_t fs_stats_get_write_usecs(const struct fs_stats *stats); + #endif From dovecot at dovecot.org Thu Nov 19 12:47:57 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 19 Nov 2015 12:47:57 +0000 Subject: dovecot-2.2: fts plugin: Install fts-storage.h Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1f9f28ce605c changeset: 19400:1f9f28ce605c user: Timo Sirainen date: Thu Nov 19 14:47:27 2015 +0200 description: fts plugin: Install fts-storage.h diffstat: src/plugins/fts/Makefile.am | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r fac18fa49f2d -r 1f9f28ce605c src/plugins/fts/Makefile.am --- a/src/plugins/fts/Makefile.am Thu Nov 19 14:46:54 2015 +0200 +++ b/src/plugins/fts/Makefile.am Thu Nov 19 14:47:27 2015 +0200 @@ -45,6 +45,7 @@ fts-expunge-log.h \ fts-indexer.h \ fts-parser.h \ + fts-storage.h \ fts-user.h noinst_HEADERS = \ @@ -52,8 +53,7 @@ fts-build-mail.h \ fts-plugin.h \ fts-search-args.h \ - fts-search-serialize.h \ - fts-storage.h + fts-search-serialize.h pkglibexec_PROGRAMS = xml2text From dovecot at dovecot.org Thu Nov 19 15:40:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 19 Nov 2015 15:40:24 +0000 Subject: dovecot-2.2: lib-fts: Silence clang warnings. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/db85dbd62706 changeset: 19401:db85dbd62706 user: Teemu Huovila date: Thu Nov 19 17:39:46 2015 +0200 description: lib-fts: Silence clang warnings. diffstat: src/lib-fts/test-fts-filter.c | 4 ++-- src/lib-fts/test-fts-language.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diffs (42 lines): diff -r 1f9f28ce605c -r db85dbd62706 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Thu Nov 19 14:47:27 2015 +0200 +++ b/src/lib-fts/test-fts-filter.c Thu Nov 19 17:39:46 2015 +0200 @@ -753,7 +753,7 @@ test_assert(*bpp == NULL); } else { test_assert(*bpp != NULL); - test_assert(strcmp(*bpp, token) == 0); + test_assert(null_strcmp(*bpp, token) == 0); } bpp++; } @@ -804,7 +804,7 @@ test_assert(*bpp == NULL); } else { test_assert(*bpp != NULL); - test_assert(strcmp(*bpp, token) == 0); + test_assert(null_strcmp(*bpp, token) == 0); } bpp++; } diff -r 1f9f28ce605c -r db85dbd62706 src/lib-fts/test-fts-language.c --- a/src/lib-fts/test-fts-language.c Thu Nov 19 14:47:27 2015 +0200 +++ b/src/lib-fts/test-fts-language.c Thu Nov 19 17:39:46 2015 +0200 @@ -283,7 +283,7 @@ const struct fts_language *lp; test_begin("fts language find built-in"); lp = fts_language_find("en"); - test_assert(lp != NULL); + i_assert(lp != NULL); test_assert(strcmp(lp->name, "en") == 0); test_end(); } @@ -293,7 +293,7 @@ test_begin("fts language register"); fts_language_register("jp"); lp = fts_language_find("jp"); - test_assert(lp != NULL); + i_assert(lp != NULL); test_assert(strcmp(lp->name, "jp") == 0); test_end(); } From dovecot at dovecot.org Thu Nov 19 15:44:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 19 Nov 2015 15:44:15 +0000 Subject: dovecot-2.2: fs-posix: Code cleanup - don't store dir_mode perma... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4564f4300c91 changeset: 19402:4564f4300c91 user: Timo Sirainen date: Thu Nov 19 17:19:19 2015 +0200 description: fs-posix: Code cleanup - don't store dir_mode permanently. Cleanup in preparation for mode=auto. diffstat: src/lib-fs/fs-posix.c | 15 +++++++++------ 1 files changed, 9 insertions(+), 6 deletions(-) diffs (44 lines): diff -r db85dbd62706 -r 4564f4300c91 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Thu Nov 19 17:39:46 2015 +0200 +++ b/src/lib-fs/fs-posix.c Thu Nov 19 17:19:19 2015 +0200 @@ -33,7 +33,7 @@ char *temp_file_prefix, *root_path, *path_prefix; unsigned int temp_file_prefix_len; enum fs_posix_lock_method lock_method; - mode_t mode, dir_mode; + mode_t mode; }; struct posix_fs_file { @@ -118,10 +118,6 @@ return -1; } } - fs->dir_mode = fs->mode; - if ((fs->dir_mode & 0600) != 0) fs->dir_mode |= 0100; - if ((fs->dir_mode & 0060) != 0) fs->dir_mode |= 0010; - if ((fs->dir_mode & 0006) != 0) fs->dir_mode |= 0001; return 0; } @@ -147,12 +143,19 @@ static int fs_posix_mkdir_parents(struct posix_fs *fs, const char *path) { const char *dir, *fname; + mode_t dir_mode; fname = strrchr(path, '/'); if (fname == NULL) return 1; + + dir_mode = fs->mode; + if ((dir_mode & 0600) != 0) dir_mode |= 0100; + if ((dir_mode & 0060) != 0) dir_mode |= 0010; + if ((dir_mode & 0006) != 0) dir_mode |= 0001; + dir = t_strdup_until(path, fname); - if (mkdir_parents(dir, fs->dir_mode) == 0) + if (mkdir_parents(dir, dir_mode) == 0) return 0; else if (errno == EEXIST) return 1; From dovecot at dovecot.org Thu Nov 19 15:44:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 19 Nov 2015 15:44:15 +0000 Subject: dovecot-2.2: fs-posix: Added mode=auto parameter to copy mode fr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/52e0fb4f2693 changeset: 19403:52e0fb4f2693 user: Timo Sirainen date: Thu Nov 19 17:43:47 2015 +0200 description: fs-posix: Added mode=auto parameter to copy mode from parent directory if setgid-bit is set diffstat: src/lib-fs/fs-posix.c | 55 ++++++++++++++++++++++++++++++++++++++++++++------ 1 files changed, 48 insertions(+), 7 deletions(-) diffs (108 lines): diff -r 4564f4300c91 -r 52e0fb4f2693 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Thu Nov 19 17:19:19 2015 +0200 +++ b/src/lib-fs/fs-posix.c Thu Nov 19 17:43:47 2015 +0200 @@ -34,6 +34,7 @@ unsigned int temp_file_prefix_len; enum fs_posix_lock_method lock_method; mode_t mode; + bool mode_auto; }; struct posix_fs_file { @@ -102,6 +103,8 @@ fs->path_prefix = i_strconcat(arg + 7, "/", NULL); else fs->path_prefix = i_strdup(arg + 7); + } else if (strcmp(arg, "mode=auto") == 0) { + fs->mode_auto = TRUE; } else if (strncmp(arg, "mode=", 5) == 0) { unsigned int mode; if (str_to_uint_oct(arg+5, &mode) < 0) { @@ -140,21 +143,51 @@ FS_PROPERTY_STAT | FS_PROPERTY_ITER | FS_PROPERTY_RELIABLEITER; } +static int +fs_posix_get_mode(struct posix_fs *fs, const char *path, mode_t *mode_r) +{ + struct stat st; + const char *p; + + *mode_r = fs->mode; + + while (stat(path, &st) < 0) { + if (errno != ENOENT) { + fs_set_error(&fs->fs, "stat(%s) failed: %m", path); + return -1; + } + p = strrchr(path, '/'); + if (p != NULL) + path = t_strdup_until(path, p); + else if (strcmp(path, ".") != 0) + path = "."; + else + return 0; + } + if ((st.st_mode & S_ISGID) != 0) { + /* setgid set - copy mode from parent */ + *mode_r = st.st_mode & 0666; + } + return 0; +} + static int fs_posix_mkdir_parents(struct posix_fs *fs, const char *path) { const char *dir, *fname; - mode_t dir_mode; + mode_t mode, dir_mode; fname = strrchr(path, '/'); if (fname == NULL) return 1; + dir = t_strdup_until(path, fname); - dir_mode = fs->mode; + if (fs_posix_get_mode(fs, dir, &mode) < 0) + return -1; + dir_mode = mode; if ((dir_mode & 0600) != 0) dir_mode |= 0100; if ((dir_mode & 0060) != 0) dir_mode |= 0010; if ((dir_mode & 0006) != 0) dir_mode |= 0001; - dir = t_strdup_until(path, fname); if (mkdir_parents(dir, dir_mode) == 0) return 0; else if (errno == EEXIST) @@ -198,20 +231,28 @@ string_t *str = t_str_new(256); const char *slash; unsigned int try_count = 0; + mode_t mode; int fd; i_assert(file->temp_path == NULL); - if ((slash = strrchr(file->full_path, '/')) != NULL) - str_append_n(str, file->full_path, slash - file->full_path + 1); + if ((slash = strrchr(file->full_path, '/')) != NULL) { + str_append_n(str, file->full_path, slash - file->full_path); + if (fs_posix_get_mode(fs, str_c(str), &mode) < 0) + return -1; + str_append_c(str, '/'); + } else { + if (fs_posix_get_mode(fs, ".", &mode) < 0) + return -1; + } str_append(str, fs->temp_file_prefix); - fd = safe_mkstemp_hostpid(str, fs->mode, (uid_t)-1, (gid_t)-1); + fd = safe_mkstemp_hostpid(str, mode, (uid_t)-1, (gid_t)-1); while (fd == -1 && errno == ENOENT && try_count <= MAX_MKDIR_RETRY_COUNT) { if (fs_posix_mkdir_parents(fs, str_c(str)) < 0) return -1; - fd = safe_mkstemp_hostpid(str, fs->mode, (uid_t)-1, (gid_t)-1); + fd = safe_mkstemp_hostpid(str, mode, (uid_t)-1, (gid_t)-1); try_count++; } if (fd == -1) { From dovecot at dovecot.org Fri Nov 20 10:51:53 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Nov 2015 10:51:53 +0000 Subject: dovecot-2.2: fts: Remove default filters and tokenizers. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7b93d5d71f2e changeset: 19404:7b93d5d71f2e user: Teemu Huovila date: Wed Nov 18 16:33:26 2015 +0200 description: fts: Remove default filters and tokenizers. There aren't any specially good defaults that work for all the languages, so it's better to just enforce explicit settings. diffstat: src/plugins/fts/fts-user.c | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) diffs (38 lines): diff -r 52e0fb4f2693 -r 7b93d5d71f2e src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Thu Nov 19 17:43:47 2015 +0200 +++ b/src/plugins/fts/fts-user.c Wed Nov 18 16:33:26 2015 +0200 @@ -11,9 +11,6 @@ #define FTS_USER_CONTEXT(obj) \ MODULE_CONTEXT(obj, fts_user_module) -#define FTS_DEFAULT_TOKENIZERS "generic email-address" -#define FTS_DEFAULT_FILTERS "normalizer-icu snowball" - struct fts_user { union mail_user_module_context module_ctx; int refcount; @@ -103,8 +100,9 @@ filters_key = "fts_filters"; str = mail_user_plugin_getenv(user, filters_key); if (str == NULL) { - str = FTS_DEFAULT_FILTERS; - filters_key = "fts_filters(built-in default)"; + /* No filters */ + *filter_r = NULL; + return 0; } } @@ -162,8 +160,10 @@ tokenizers_key = "fts_tokenizers"; str = mail_user_plugin_getenv(user, tokenizers_key); - if (str == NULL) - str = FTS_DEFAULT_TOKENIZERS; + if (str == NULL) { + *error_r = "fts_tokenizers setting is missing"; + return -1; + } tokenizers = t_strsplit_spaces(str, " "); From dovecot at dovecot.org Fri Nov 20 10:51:53 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Nov 2015 10:51:53 +0000 Subject: dovecot-2.2: lib-fts: Fix compilation for systems without libste... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/194e3622d5e6 changeset: 19405:194e3622d5e6 user: Teemu Huovila date: Wed Nov 18 16:33:26 2015 +0200 description: lib-fts: Fix compilation for systems without libstemmer. diffstat: src/lib-fts/test-fts-filter.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 7b93d5d71f2e -r 194e3622d5e6 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Wed Nov 18 16:33:26 2015 +0200 +++ b/src/lib-fts/test-fts-filter.c Wed Nov 18 16:33:26 2015 +0200 @@ -14,7 +14,9 @@ static struct fts_language english_language = { .name = "en" }; static struct fts_language french_language = { .name = "fr" }; static struct fts_language norwegian_language = { .name = "no" }; +#ifdef HAVE_FTS_STEMMER static struct fts_language swedish_language = { .name = "sv" }; +#endif static void test_fts_filter_find(void) { From dovecot at dovecot.org Tue Nov 24 09:15:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 24 Nov 2015 09:15:56 +0000 Subject: dovecot-2.2: director: With director_consistent_hashing=yes host... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/63516762a64c changeset: 19406:63516762a64c user: Timo Sirainen date: Mon Nov 23 19:35:03 2015 +0200 description: director: With director_consistent_hashing=yes hosts_hash wasn't always calculated right. If different servers had added hosts in different order, the hosts_hash would have become different, which caused errors and resyncs. diffstat: src/director/mail-host.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 194e3622d5e6 -r 63516762a64c src/director/mail-host.c --- a/src/director/mail-host.c Wed Nov 18 16:33:26 2015 +0200 +++ b/src/director/mail-host.c Mon Nov 23 19:35:03 2015 +0200 @@ -99,8 +99,6 @@ struct mail_host *const *hostp; unsigned int i; - array_sort(&list->hosts, mail_host_cmp); - /* rebuild vhosts */ array_clear(&list->vhosts); array_foreach(&list->hosts, hostp) { @@ -119,6 +117,8 @@ struct mail_host *const *hostp; uint32_t num; + array_sort(&list->hosts, mail_host_cmp); + if (list->consistent_hashing) mail_hosts_sort_ring(list); else From dovecot at dovecot.org Tue Nov 24 09:15:57 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 24 Nov 2015 09:15:57 +0000 Subject: dovecot-2.2: director: Code cleanup - make most mail_host_*() li... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/39cfca637d60 changeset: 19407:39cfca637d60 user: Timo Sirainen date: Mon Nov 23 19:38:31 2015 +0200 description: director: Code cleanup - make most mail_host_*() list parameters unnecessary. diffstat: src/director/director-connection.c | 6 ++---- src/director/director.c | 2 +- src/director/doveadm-connection.c | 5 ++--- src/director/mail-host.c | 14 +++++++------- src/director/mail-host.h | 10 +++++----- 5 files changed, 17 insertions(+), 20 deletions(-) diffs (126 lines): diff -r 63516762a64c -r 39cfca637d60 src/director/director-connection.c --- a/src/director/director-connection.c Mon Nov 23 19:35:03 2015 +0200 +++ b/src/director/director-connection.c Mon Nov 23 19:38:31 2015 +0200 @@ -931,10 +931,8 @@ } if (update) { - mail_host_set_down(conn->dir->mail_hosts, host, - down, last_updown_change); - mail_host_set_vhost_count(conn->dir->mail_hosts, - host, vhost_count); + mail_host_set_down(host, down, last_updown_change); + mail_host_set_vhost_count(host, vhost_count); director_update_host(conn->dir, src_host, dir_host, host); } else { dir_debug("Ignoring host %s update vhost_count=%u " diff -r 63516762a64c -r 39cfca637d60 src/director/director.c --- a/src/director/director.c Mon Nov 23 19:35:03 2015 +0200 +++ b/src/director/director.c Mon Nov 23 19:38:31 2015 +0200 @@ -601,7 +601,7 @@ } user_directory_remove_host(dir->users, host); - mail_host_remove(dir->mail_hosts, host); + mail_host_remove(host); director_sync(dir); } diff -r 63516762a64c -r 39cfca637d60 src/director/doveadm-connection.c --- a/src/director/doveadm-connection.c Mon Nov 23 19:35:03 2015 +0200 +++ b/src/director/doveadm-connection.c Mon Nov 23 19:38:31 2015 +0200 @@ -288,7 +288,7 @@ return TRUE; } if (vhost_count != UINT_MAX) - mail_host_set_vhost_count(dir->mail_hosts, host, vhost_count); + mail_host_set_vhost_count(host, vhost_count); /* NOTE: we don't support changing a tag for an existing host. it needs to be removed first. otherwise it would be a bit ugly to handle. */ @@ -334,8 +334,7 @@ "host is already being updated - try again later\n"); return TRUE; } else { - mail_host_set_down(conn->dir->mail_hosts, host, - down, ioloop_time); + mail_host_set_down(host, down, ioloop_time); director_update_host(conn->dir, conn->dir->self_host, NULL, host); } diff -r 63516762a64c -r 39cfca637d60 src/director/mail-host.c --- a/src/director/mail-host.c Mon Nov 23 19:35:03 2015 +0200 +++ b/src/director/mail-host.c Mon Nov 23 19:38:31 2015 +0200 @@ -144,6 +144,7 @@ i_assert(tag != NULL); host = i_new(struct mail_host, 1); + host->list = list; host->vhost_count = VHOST_MULTIPLIER; host->ip = *ip; host->tag = i_strdup(tag); @@ -305,21 +306,19 @@ host->tag = i_strdup(tag); } -void mail_host_set_down(struct mail_host_list *list, - struct mail_host *host, bool down, time_t timestamp) +void mail_host_set_down(struct mail_host *host, bool down, time_t timestamp) { if (host->down != down) { host->down = down; host->last_updown_change = timestamp; - list->hosts_unsorted = TRUE; + host->list->hosts_unsorted = TRUE; } } -void mail_host_set_vhost_count(struct mail_host_list *list, - struct mail_host *host, unsigned int vhost_count) +void mail_host_set_vhost_count(struct mail_host *host, unsigned int vhost_count) { host->vhost_count = vhost_count; - list->hosts_unsorted = TRUE; + host->list->hosts_unsorted = TRUE; } static void mail_host_free(struct mail_host *host) @@ -329,8 +328,9 @@ i_free(host); } -void mail_host_remove(struct mail_host_list *list, struct mail_host *host) +void mail_host_remove(struct mail_host *host) { + struct mail_host_list *list = host->list; struct mail_host *const *hosts; unsigned int i, count; diff -r 63516762a64c -r 39cfca637d60 src/director/mail-host.h --- a/src/director/mail-host.h Mon Nov 23 19:35:03 2015 +0200 +++ b/src/director/mail-host.h Mon Nov 23 19:38:31 2015 +0200 @@ -6,6 +6,8 @@ struct mail_host_list; struct mail_host { + struct mail_host_list *list; + unsigned int user_count; unsigned int vhost_count; /* server up/down. down=TRUE has effectively the same result as if @@ -37,12 +39,10 @@ int mail_hosts_parse_and_add(struct mail_host_list *list, const char *hosts_string); void mail_host_set_tag(struct mail_host *host, const char *tag); -void mail_host_set_down(struct mail_host_list *list, - struct mail_host *host, bool down, time_t timestamp); -void mail_host_set_vhost_count(struct mail_host_list *list, - struct mail_host *host, +void mail_host_set_down(struct mail_host *host, bool down, time_t timestamp); +void mail_host_set_vhost_count(struct mail_host *host, unsigned int vhost_count); -void mail_host_remove(struct mail_host_list *list, struct mail_host *host); +void mail_host_remove(struct mail_host *host); void mail_hosts_set_synced(struct mail_host_list *list); unsigned int mail_hosts_hash(struct mail_host_list *list); From dovecot at dovecot.org Tue Nov 24 09:16:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 24 Nov 2015 09:16:07 +0000 Subject: dovecot-2.2: director: Code cleanup - access host->tag via mail_... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7e47f561ad49 changeset: 19408:7e47f561ad49 user: Timo Sirainen date: Mon Nov 23 19:41:38 2015 +0200 description: director: Code cleanup - access host->tag via mail_host_get_tag() In preparation for the following changes. diffstat: src/director/director-connection.c | 12 +++++++----- src/director/director.c | 7 ++++--- src/director/doveadm-connection.c | 5 +++-- src/director/mail-host.c | 5 +++++ src/director/mail-host.h | 1 + 5 files changed, 20 insertions(+), 10 deletions(-) diffs (116 lines): diff -r 39cfca637d60 -r 7e47f561ad49 src/director/director-connection.c --- a/src/director/director-connection.c Mon Nov 23 19:38:31 2015 +0200 +++ b/src/director/director-connection.c Mon Nov 23 19:41:38 2015 +0200 @@ -863,7 +863,7 @@ struct director_host *src_host = conn->host; struct mail_host *host; struct ip_addr ip; - const char *tag = "", *hostname = NULL; + const char *tag = "", *host_tag, *hostname = NULL; unsigned int arg_count, vhost_count; bool update, down = FALSE; time_t last_updown_change = 0; @@ -904,10 +904,11 @@ host->down != down || host->last_updown_change != last_updown_change; - if (strcmp(tag, host->tag) != 0) { + host_tag = mail_host_get_tag(host); + if (strcmp(tag, host_tag) != 0) { i_error("director(%s): Host %s changed tag from '%s' to '%s'", conn->name, net_ip2addr(&host->ip), - host->tag, tag); + host_tag, tag); mail_host_set_tag(host, tag); update = TRUE; } @@ -1694,12 +1695,13 @@ str_printfa(str, "HOST-HAND-START\t%u\n", conn->dir->ring_handshaked); array_foreach(mail_hosts_get(conn->dir->mail_hosts), hostp) { struct mail_host *host = *hostp; + const char *host_tag = mail_host_get_tag(host); str_printfa(str, "HOST\t%s\t%u", net_ip2addr(&host->ip), host->vhost_count); - if (host->tag[0] != '\0' || send_updowns) { + if (host_tag[0] != '\0' || send_updowns) { str_append_c(str, '\t'); - str_append_tabescaped(str, host->tag); + str_append_tabescaped(str, host_tag); } if (send_updowns) { str_printfa(str, "\t%c%ld\t", host->down ? 'D' : 'U', diff -r 39cfca637d60 -r 7e47f561ad49 src/director/director.c --- a/src/director/director.c Mon Nov 23 19:38:31 2015 +0200 +++ b/src/director/director.c Mon Nov 23 19:41:38 2015 +0200 @@ -523,6 +523,7 @@ struct director_host *orig_src, struct mail_host *host) { + const char *host_tag = mail_host_get_tag(host); string_t *str; if (orig_src == NULL) { @@ -537,11 +538,11 @@ net_ip2addr(&host->ip), host->vhost_count); if (dir->ring_min_version >= DIRECTOR_VERSION_TAGS) { str_append_c(str, '\t'); - str_append_tabescaped(str, host->tag); - } else if (host->tag[0] != '\0' && + str_append_tabescaped(str, host_tag); + } else if (host_tag[0] != '\0' && dir->ring_min_version < DIRECTOR_VERSION_TAGS) { i_error("Ring has directors that don't support tags - removing host %s with tag '%s'", - net_ip2addr(&host->ip), host->tag); + net_ip2addr(&host->ip), host_tag); director_remove_host(dir, NULL, NULL, host); return; } diff -r 39cfca637d60 -r 7e47f561ad49 src/director/doveadm-connection.c --- a/src/director/doveadm-connection.c Mon Nov 23 19:38:31 2015 +0200 +++ b/src/director/doveadm-connection.c Mon Nov 23 19:41:38 2015 +0200 @@ -50,7 +50,7 @@ str_printfa(str, "%s\t%u\t%u\t", net_ip2addr(&(*hostp)->ip), (*hostp)->vhost_count, (*hostp)->user_count); - str_append_tabescaped(str, (*hostp)->tag); + str_append_tabescaped(str, mail_host_get_tag(*hostp)); str_printfa(str, "\t%c\t%ld", (*hostp)->down ? 'D' : 'U', (long)(*hostp)->last_updown_change); str_append_c(str, '\n'); @@ -422,7 +422,8 @@ if (user->host != host) continue; new_host = mail_host_get_by_hash(dir->mail_hosts, - user->username_hash, host->tag); + user->username_hash, + mail_host_get_tag(host)); if (new_host != host) T_BEGIN { director_move_user(dir, src, NULL, user->username_hash, new_host); diff -r 39cfca637d60 -r 7e47f561ad49 src/director/mail-host.c --- a/src/director/mail-host.c Mon Nov 23 19:38:31 2015 +0200 +++ b/src/director/mail-host.c Mon Nov 23 19:41:38 2015 +0200 @@ -298,6 +298,11 @@ return ret; } +const char *mail_host_get_tag(struct mail_host *host) +{ + return host->tag; +} + void mail_host_set_tag(struct mail_host *host, const char *tag) { i_assert(tag != NULL); diff -r 39cfca637d60 -r 7e47f561ad49 src/director/mail-host.h --- a/src/director/mail-host.h Mon Nov 23 19:38:31 2015 +0200 +++ b/src/director/mail-host.h Mon Nov 23 19:41:38 2015 +0200 @@ -38,6 +38,7 @@ int mail_hosts_parse_and_add(struct mail_host_list *list, const char *hosts_string); +const char *mail_host_get_tag(struct mail_host *host); void mail_host_set_tag(struct mail_host *host, const char *tag); void mail_host_set_down(struct mail_host *host, bool down, time_t timestamp); void mail_host_set_vhost_count(struct mail_host *host, From dovecot at dovecot.org Tue Nov 24 09:16:12 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 24 Nov 2015 09:16:12 +0000 Subject: dovecot-2.2: director: Code cleanup - rename tag to tag_name in ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/11a913488054 changeset: 19409:11a913488054 user: Timo Sirainen date: Mon Nov 23 19:44:50 2015 +0200 description: director: Code cleanup - rename tag to tag_name in mail_host_*() parameters. In preparation for the following changes. diffstat: src/director/mail-host.c | 42 ++++++++++++++++++++++-------------------- src/director/mail-host.h | 8 ++++---- 2 files changed, 26 insertions(+), 24 deletions(-) diffs (178 lines): diff -r 7e47f561ad49 -r 11a913488054 src/director/mail-host.c --- a/src/director/mail-host.c Mon Nov 23 19:41:38 2015 +0200 +++ b/src/director/mail-host.c Mon Nov 23 19:44:50 2015 +0200 @@ -137,17 +137,17 @@ struct mail_host * mail_host_add_ip(struct mail_host_list *list, const struct ip_addr *ip, - const char *tag) + const char *tag_name) { struct mail_host *host; - i_assert(tag != NULL); + i_assert(tag_name != NULL); host = i_new(struct mail_host, 1); host->list = list; host->vhost_count = VHOST_MULTIPLIER; host->ip = *ip; - host->tag = i_strdup(tag); + host->tag = i_strdup(tag_name); array_append(&list->hosts, &host, 1); list->hosts_unsorted = TRUE; @@ -156,23 +156,24 @@ struct mail_host * mail_host_add_hostname(struct mail_host_list *list, const char *hostname, - const struct ip_addr *ip, const char *tag) + const struct ip_addr *ip, const char *tag_name) { struct mail_host *host; - host = mail_host_add_ip(list, ip, tag); + host = mail_host_add_ip(list, ip, tag_name); host->hostname = i_strdup(hostname); return host; } static int -mail_host_add(struct mail_host_list *list, const char *hostname, const char *tag) +mail_host_add(struct mail_host_list *list, const char *hostname, + const char *tag_name) { struct ip_addr *ips, ip; unsigned int i, ips_count; if (net_addr2ip(hostname, &ip) == 0) { - (void)mail_host_add_ip(list, &ip, tag); + (void)mail_host_add_ip(list, &ip, tag_name); return 0; } @@ -182,13 +183,14 @@ } for (i = 0; i < ips_count; i++) - (void)mail_host_add_hostname(list, hostname, &ips[i], tag); + (void)mail_host_add_hostname(list, hostname, &ips[i], tag_name); return 0; } static int mail_hosts_add_range(struct mail_host_list *list, - struct ip_addr ip1, struct ip_addr ip2, const char *tag) + struct ip_addr ip1, struct ip_addr ip2, + const char *tag_name) { uint32_t *ip1_arr, *ip2_arr; uint32_t i1, i2; @@ -243,7 +245,7 @@ /* create hosts from the final bits */ do { ip1_arr[i] = ntohl(i1); - (void)mail_host_add_ip(list, &ip1, tag); + (void)mail_host_add_ip(list, &ip1, tag_name); i1++; } while (ip1_arr[i] != ip2_arr[i]); return 0; @@ -303,12 +305,12 @@ return host->tag; } -void mail_host_set_tag(struct mail_host *host, const char *tag) +void mail_host_set_tag(struct mail_host *host, const char *tag_name) { - i_assert(tag != NULL); + i_assert(tag_name != NULL); i_free(host->tag); - host->tag = i_strdup(tag); + host->tag = i_strdup(tag_name); } void mail_host_set_down(struct mail_host *host, bool down, time_t timestamp) @@ -368,7 +370,7 @@ static struct mail_host * mail_host_get_by_hash_ring(struct mail_host_list *list, unsigned int hash, - const char *tag) + const char *tag_name) { struct mail_host *host; const struct mail_vhost *vhosts; @@ -386,7 +388,7 @@ for (i = 0; i < count; i++) { host = vhosts[(idx + i) % count].host; - if (strcmp(host->tag, tag) == 0) + if (strcmp(host->tag, tag_name) == 0) return host; } return NULL; @@ -394,7 +396,7 @@ static struct mail_host * mail_host_get_by_hash_direct(struct mail_host_list *list, unsigned int hash, - const char *tag) + const char *tag_name) { struct mail_host *host; const struct mail_vhost *vhosts; @@ -406,7 +408,7 @@ for (i = 0; i < count; i++) { host = vhosts[(hash + i) % count].host; - if (strcmp(host->tag, tag) == 0) + if (strcmp(host->tag, tag_name) == 0) return host; } return NULL; @@ -414,15 +416,15 @@ struct mail_host * mail_host_get_by_hash(struct mail_host_list *list, unsigned int hash, - const char *tag) + const char *tag_name) { if (list->hosts_unsorted) mail_hosts_sort(list); if (list->consistent_hashing) - return mail_host_get_by_hash_ring(list, hash, tag); + return mail_host_get_by_hash_ring(list, hash, tag_name); else - return mail_host_get_by_hash_direct(list, hash, tag); + return mail_host_get_by_hash_direct(list, hash, tag_name); } void mail_hosts_set_synced(struct mail_host_list *list) diff -r 7e47f561ad49 -r 11a913488054 src/director/mail-host.h --- a/src/director/mail-host.h Mon Nov 23 19:41:38 2015 +0200 +++ b/src/director/mail-host.h Mon Nov 23 19:44:50 2015 +0200 @@ -26,20 +26,20 @@ struct mail_host * mail_host_add_ip(struct mail_host_list *list, const struct ip_addr *ip, - const char *tag); + const char *tag_name); struct mail_host * mail_host_add_hostname(struct mail_host_list *list, const char *hostname, - const struct ip_addr *ip, const char *tag); + const struct ip_addr *ip, const char *tag_name); struct mail_host * mail_host_lookup(struct mail_host_list *list, const struct ip_addr *ip); struct mail_host * mail_host_get_by_hash(struct mail_host_list *list, unsigned int hash, - const char *tag); + const char *tag_name); int mail_hosts_parse_and_add(struct mail_host_list *list, const char *hosts_string); const char *mail_host_get_tag(struct mail_host *host); -void mail_host_set_tag(struct mail_host *host, const char *tag); +void mail_host_set_tag(struct mail_host *host, const char *tag_name); void mail_host_set_down(struct mail_host *host, bool down, time_t timestamp); void mail_host_set_vhost_count(struct mail_host *host, unsigned int vhost_count); From dovecot at dovecot.org Tue Nov 24 09:16:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 24 Nov 2015 09:16:13 +0000 Subject: dovecot-2.2: director: Code cleanup - renamed hosts_unsorted to ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5168fdd5127e changeset: 19410:5168fdd5127e user: Timo Sirainen date: Mon Nov 23 19:47:08 2015 +0200 description: director: Code cleanup - renamed hosts_unsorted to vhosts_unsorted diffstat: src/director/mail-host.c | 24 ++++++++++++------------ 1 files changed, 12 insertions(+), 12 deletions(-) diffs (109 lines): diff -r 11a913488054 -r 5168fdd5127e src/director/mail-host.c --- a/src/director/mail-host.c Mon Nov 23 19:44:50 2015 +0200 +++ b/src/director/mail-host.c Mon Nov 23 19:47:08 2015 +0200 @@ -18,7 +18,7 @@ ARRAY_TYPE(mail_host) hosts; ARRAY(struct mail_vhost) vhosts; unsigned int hosts_hash; - bool hosts_unsorted; + bool vhosts_unsorted; bool consistent_hashing; }; @@ -90,7 +90,7 @@ array_foreach(&list->hosts, hostp) mail_vhost_add(list, *hostp); array_sort(&list->vhosts, mail_vhost_cmp); - list->hosts_unsorted = FALSE; + list->vhosts_unsorted = FALSE; } static void mail_hosts_sort_direct(struct mail_host_list *list) @@ -109,7 +109,7 @@ vhost->host = *hostp; } } - list->hosts_unsorted = FALSE; + list->vhosts_unsorted = FALSE; } static void mail_hosts_sort(struct mail_host_list *list) @@ -150,7 +150,7 @@ host->tag = i_strdup(tag_name); array_append(&list->hosts, &host, 1); - list->hosts_unsorted = TRUE; + list->vhosts_unsorted = TRUE; return host; } @@ -318,14 +318,14 @@ if (host->down != down) { host->down = down; host->last_updown_change = timestamp; - host->list->hosts_unsorted = TRUE; + host->list->vhosts_unsorted = TRUE; } } void mail_host_set_vhost_count(struct mail_host *host, unsigned int vhost_count) { host->vhost_count = vhost_count; - host->list->hosts_unsorted = TRUE; + host->list->vhosts_unsorted = TRUE; } static void mail_host_free(struct mail_host *host) @@ -350,7 +350,7 @@ } mail_host_free(host); - list->hosts_unsorted = TRUE; + list->vhosts_unsorted = TRUE; } struct mail_host * @@ -358,7 +358,7 @@ { struct mail_host *const *hostp; - if (list->hosts_unsorted) + if (list->vhosts_unsorted) mail_hosts_sort(list); array_foreach(&list->hosts, hostp) { @@ -418,7 +418,7 @@ mail_host_get_by_hash(struct mail_host_list *list, unsigned int hash, const char *tag_name) { - if (list->hosts_unsorted) + if (list->vhosts_unsorted) mail_hosts_sort(list); if (list->consistent_hashing) @@ -437,7 +437,7 @@ unsigned int mail_hosts_hash(struct mail_host_list *list) { - if (list->hosts_unsorted) + if (list->vhosts_unsorted) mail_hosts_sort(list); /* don't retun 0 as hash, since we're using it as "doesn't exist" in some places. */ @@ -446,14 +446,14 @@ bool mail_hosts_have_usable(struct mail_host_list *list) { - if (list->hosts_unsorted) + if (list->vhosts_unsorted) mail_hosts_sort(list); return array_count(&list->vhosts) > 0; } const ARRAY_TYPE(mail_host) *mail_hosts_get(struct mail_host_list *list) { - if (list->hosts_unsorted) + if (list->vhosts_unsorted) mail_hosts_sort(list); return &list->hosts; } From dovecot at dovecot.org Tue Nov 24 09:16:18 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 24 Nov 2015 09:16:18 +0000 Subject: dovecot-2.2: director: Fixed backend selection when multiple tag... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0e05efd14b39 changeset: 19411:0e05efd14b39 user: Timo Sirainen date: Tue Nov 24 11:15:47 2015 +0200 description: director: Fixed backend selection when multiple tags were used. The previous algorithm was causing an uneven load for backends. This change breaks compatibility with older director servers that were using tags because of the different selection algorithm. The new director code refuses to run within a cluster with old directors if tags are used. diffstat: src/director/director-connection.c | 9 + src/director/director.c | 13 +- src/director/director.h | 4 +- src/director/mail-host.c | 171 ++++++++++++++++++++++++------------ src/director/mail-host.h | 3 +- 5 files changed, 136 insertions(+), 64 deletions(-) diffs (truncated from 446 to 300 lines): diff -r 5168fdd5127e -r 0e05efd14b39 src/director/director-connection.c --- a/src/director/director-connection.c Mon Nov 23 19:47:08 2015 +0200 +++ b/src/director/director-connection.c Tue Nov 24 11:15:47 2015 +0200 @@ -893,6 +893,10 @@ i_assert(conn->handshake_sending_hosts); return TRUE; } + if (tag[0] != '\0' && conn->minor_version < DIRECTOR_VERSION_TAGS_V2) { + director_cmd_error(conn, "Received a host tag from older director version with incompatible tagging support"); + return FALSE; + } host = mail_host_lookup(conn->dir->mail_hosts, &ip); if (host == NULL) { @@ -1208,6 +1212,11 @@ DIRECTOR_VERSION_MAJOR); return -1; } + if (conn->minor_version < DIRECTOR_VERSION_TAGS_V2 && + mail_hosts_have_tags(conn->dir->mail_hosts)) { + i_error("director(%s): Director version supports incompatible tags", conn->name); + return FALSE; + } conn->version_received = TRUE; if (conn->done_pending) { if (director_connection_send_done(conn) < 0) diff -r 5168fdd5127e -r 0e05efd14b39 src/director/director.c --- a/src/director/director.c Mon Nov 23 19:47:08 2015 +0200 +++ b/src/director/director.c Tue Nov 24 11:15:47 2015 +0200 @@ -536,13 +536,18 @@ net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq, net_ip2addr(&host->ip), host->vhost_count); - if (dir->ring_min_version >= DIRECTOR_VERSION_TAGS) { + if (dir->ring_min_version >= DIRECTOR_VERSION_TAGS_V2) { str_append_c(str, '\t'); str_append_tabescaped(str, host_tag); } else if (host_tag[0] != '\0' && - dir->ring_min_version < DIRECTOR_VERSION_TAGS) { - i_error("Ring has directors that don't support tags - removing host %s with tag '%s'", - net_ip2addr(&host->ip), host_tag); + dir->ring_min_version < DIRECTOR_VERSION_TAGS_V2) { + if (dir->ring_min_version < DIRECTOR_VERSION_TAGS) { + i_error("Ring has directors that don't support tags - removing host %s with tag '%s'", + net_ip2addr(&host->ip), host_tag); + } else { + i_error("Ring has directors that support mixed versions of tags - removing host %s with tag '%s'", + net_ip2addr(&host->ip), host_tag); + } director_remove_host(dir, NULL, NULL, host); return; } diff -r 5168fdd5127e -r 0e05efd14b39 src/director/director.h --- a/src/director/director.h Mon Nov 23 19:47:08 2015 +0200 +++ b/src/director/director.h Tue Nov 24 11:15:47 2015 +0200 @@ -6,7 +6,7 @@ #define DIRECTOR_VERSION_NAME "director" #define DIRECTOR_VERSION_MAJOR 1 -#define DIRECTOR_VERSION_MINOR 6 +#define DIRECTOR_VERSION_MINOR 7 /* weak users supported in protocol */ #define DIRECTOR_VERSION_WEAK_USERS 1 @@ -22,6 +22,8 @@ #define DIRECTOR_VERSION_TAGS 5 /* up/down state is tracked */ #define DIRECTOR_VERSION_UPDOWN 6 +/* user tag version 2 supported */ +#define DIRECTOR_VERSION_TAGS_V2 7 /* Minimum time between even attempting to communicate with a director that failed due to a protocol error. */ diff -r 5168fdd5127e -r 0e05efd14b39 src/director/mail-host.c --- a/src/director/mail-host.c Mon Nov 23 19:47:08 2015 +0200 +++ b/src/director/mail-host.c Tue Nov 24 11:15:47 2015 +0200 @@ -14,12 +14,19 @@ struct mail_host *host; }; +struct mail_tag { + /* "" = no tag */ + char *name; + ARRAY(struct mail_vhost) vhosts; +}; + struct mail_host_list { + ARRAY(struct mail_tag *) tags; ARRAY_TYPE(mail_host) hosts; - ARRAY(struct mail_vhost) vhosts; unsigned int hosts_hash; + bool consistent_hashing; bool vhosts_unsorted; - bool consistent_hashing; + bool have_vhosts; }; static int @@ -51,7 +58,7 @@ return 0; } -static void mail_vhost_add(struct mail_host_list *list, struct mail_host *host) +static void mail_vhost_add(struct mail_tag *tag, struct mail_host *host) { struct mail_vhost *vhost; struct md5_context md5_ctx, md5_ctx2; @@ -74,56 +81,65 @@ md5_update(&md5_ctx2, num_str, strlen(num_str)); md5_final(&md5_ctx2, md5); - vhost = array_append_space(&list->vhosts); + vhost = array_append_space(&tag->vhosts); vhost->host = host; for (j = 0; j < sizeof(vhost->hash); j++) vhost->hash = (vhost->hash << CHAR_BIT) | md5[j]; } } -static void mail_hosts_sort_ring(struct mail_host_list *list) +static void +mail_tag_vhosts_sort_ring(struct mail_host_list *list, struct mail_tag *tag) { struct mail_host *const *hostp; /* rebuild vhosts */ - array_clear(&list->vhosts); + array_clear(&tag->vhosts); array_foreach(&list->hosts, hostp) - mail_vhost_add(list, *hostp); - array_sort(&list->vhosts, mail_vhost_cmp); - list->vhosts_unsorted = FALSE; + mail_vhost_add(tag, *hostp); + array_sort(&tag->vhosts, mail_vhost_cmp); } -static void mail_hosts_sort_direct(struct mail_host_list *list) +static void +mail_tag_vhosts_sort_direct(struct mail_host_list *list, struct mail_tag *tag) { struct mail_vhost *vhost; struct mail_host *const *hostp; unsigned int i; /* rebuild vhosts */ - array_clear(&list->vhosts); + array_clear(&tag->vhosts); array_foreach(&list->hosts, hostp) { if ((*hostp)->down) continue; for (i = 0; i < (*hostp)->vhost_count; i++) { - vhost = array_append_space(&list->vhosts); + vhost = array_append_space(&tag->vhosts); vhost->host = *hostp; } } - list->vhosts_unsorted = FALSE; } -static void mail_hosts_sort(struct mail_host_list *list) +static void +mail_hosts_sort(struct mail_host_list *list) { struct mail_host *const *hostp; + struct mail_tag *const *tagp; uint32_t num; array_sort(&list->hosts, mail_host_cmp); - if (list->consistent_hashing) - mail_hosts_sort_ring(list); - else - mail_hosts_sort_direct(list); + list->have_vhosts = FALSE; + array_foreach(&list->tags, tagp) { + if (list->consistent_hashing) + mail_tag_vhosts_sort_ring(list, *tagp); + else + mail_tag_vhosts_sort_direct(list, *tagp); + if (array_count(&(*tagp)->vhosts) > 0) + list->have_vhosts = TRUE; + } + list->vhosts_unsorted = FALSE; + /* recalculate the hosts_hash */ list->hosts_hash = 0; array_foreach(&list->hosts, hostp) { num = ((*hostp)->down ? 1 : 0) ^ (*hostp)->vhost_count; @@ -135,6 +151,40 @@ } } +static struct mail_tag * +mail_tag_find(struct mail_host_list *list, const char *tag_name) +{ + struct mail_tag *const *tagp; + + array_foreach(&list->tags, tagp) { + if (strcmp((*tagp)->name, tag_name) == 0) + return *tagp; + } + return NULL; +} + +static struct mail_tag * +mail_tag_get(struct mail_host_list *list, const char *tag_name) +{ + struct mail_tag *tag; + + tag = mail_tag_find(list, tag_name); + if (tag == NULL) { + tag = i_new(struct mail_tag, 1); + tag->name = i_strdup(tag_name); + i_array_init(&tag->vhosts, 16*VHOST_MULTIPLIER); + array_append(&list->tags, &tag, 1); + } + return tag; +} + +static void mail_tag_free(struct mail_tag *tag) +{ + array_free(&tag->vhosts); + i_free(tag->name); + i_free(tag); +} + struct mail_host * mail_host_add_ip(struct mail_host_list *list, const struct ip_addr *ip, const char *tag_name) @@ -147,7 +197,7 @@ host->list = list; host->vhost_count = VHOST_MULTIPLIER; host->ip = *ip; - host->tag = i_strdup(tag_name); + host->tag = mail_tag_get(list, tag_name); array_append(&list->hosts, &host, 1); list->vhosts_unsorted = TRUE; @@ -302,15 +352,15 @@ const char *mail_host_get_tag(struct mail_host *host) { - return host->tag; + return host->tag->name; } void mail_host_set_tag(struct mail_host *host, const char *tag_name) { i_assert(tag_name != NULL); - i_free(host->tag); - host->tag = i_strdup(tag_name); + host->tag = mail_tag_get(host->list, tag_name); + host->list->vhosts_unsorted = TRUE; } void mail_host_set_down(struct mail_host *host, bool down, time_t timestamp) @@ -330,7 +380,6 @@ static void mail_host_free(struct mail_host *host) { - i_free(host->tag); i_free(host->hostname); i_free(host); } @@ -344,11 +393,10 @@ hosts = array_get(&list->hosts, &count); for (i = 0; i < count; i++) { if (hosts[i] == host) { - array_delete(&list->hosts, i, 1); + array_delete(&host->list->hosts, i, 1); break; } } - mail_host_free(host); list->vhosts_unsorted = TRUE; } @@ -369,15 +417,13 @@ } static struct mail_host * -mail_host_get_by_hash_ring(struct mail_host_list *list, unsigned int hash, - const char *tag_name) +mail_host_get_by_hash_ring(struct mail_tag *tag, unsigned int hash) { - struct mail_host *host; const struct mail_vhost *vhosts; - unsigned int i, count, idx; + unsigned int count, idx; - vhosts = array_get(&list->vhosts, &count); - array_bsearch_insert_pos(&list->vhosts, &hash, + vhosts = array_get(&tag->vhosts, &count); + array_bsearch_insert_pos(&tag->vhosts, &hash, mail_vhost_hash_cmp, &idx); i_assert(idx <= count); if (idx == count) { @@ -385,46 +431,38 @@ return NULL; From dovecot at dovecot.org Tue Nov 24 09:50:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 24 Nov 2015 09:50:23 +0000 Subject: dovecot-2.2: imap: Fixed hanging if a pipelined IMAP command was... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/26c565042d9d changeset: 19412:26c565042d9d user: Timo Sirainen date: Tue Nov 24 11:50:15 2015 +0200 description: imap: Fixed hanging if a pipelined IMAP command was waiting for previous command to sync. ..And the previous command was waiting for the next command to finish before it would start syncing. For example FETCH+LOGOUT pipelined. diffstat: src/imap/imap-client.c | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diffs (19 lines): diff -r 0e05efd14b39 -r 26c565042d9d src/imap/imap-client.c --- a/src/imap/imap-client.c Tue Nov 24 11:15:47 2015 +0200 +++ b/src/imap/imap-client.c Tue Nov 24 11:50:15 2015 +0200 @@ -1016,8 +1016,14 @@ if (!handled_commands) return FALSE; - if (client->input_lock == NULL) + if (client->input_lock == NULL) { + /* finished handling all commands. sync them all at once now. */ cmd_sync_delayed(client); + } else if (client->input_lock->state == CLIENT_COMMAND_STATE_WAIT_UNAMBIGUITY) { + /* the command may be waiting for previous command to sync. */ + if (cmd_sync_delayed(client)) + client_continue_pending_input(client); + } return TRUE; } From dovecot at dovecot.org Tue Nov 24 12:47:48 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 24 Nov 2015 12:47:48 +0000 Subject: dovecot-2.2: imap: Fixed crash at FETCH deinit caused by b638e19... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8d9b48c59490 changeset: 19413:8d9b48c59490 user: Timo Sirainen date: Tue Nov 24 12:56:35 2015 +0200 description: imap: Fixed crash at FETCH deinit caused by b638e19d3bd4 imap_fetch_free() would have been called twice, which caused problems. diffstat: src/imap/cmd-fetch.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (23 lines): diff -r 26c565042d9d -r 8d9b48c59490 src/imap/cmd-fetch.c --- a/src/imap/cmd-fetch.c Tue Nov 24 11:50:15 2015 +0200 +++ b/src/imap/cmd-fetch.c Tue Nov 24 12:56:35 2015 +0200 @@ -172,6 +172,11 @@ return TRUE; } +static bool cmd_fetch_finished(struct client_command_context *cmd ATTR_UNUSED) +{ + return TRUE; +} + static bool cmd_fetch_finish(struct imap_fetch_context *ctx, struct client_command_context *cmd) { @@ -199,6 +204,7 @@ output bytes are included in it (which wouldn't happen if we called client_disconnect() here directly). */ + cmd->func = cmd_fetch_finished; return cmd->cancel; } From dovecot at dovecot.org Tue Nov 24 12:47:54 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 24 Nov 2015 12:47:54 +0000 Subject: dovecot-2.2: imap: IDLE may have called client_continue_pending_... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/20e51832875e changeset: 19414:20e51832875e user: Timo Sirainen date: Tue Nov 24 13:40:12 2015 +0200 description: imap: IDLE may have called client_continue_pending_input() unnecessarily. diffstat: src/imap/cmd-idle.c | 25 ++++++++++++------------- 1 files changed, 12 insertions(+), 13 deletions(-) diffs (66 lines): diff -r 8d9b48c59490 -r 20e51832875e src/imap/cmd-idle.c --- a/src/imap/cmd-idle.c Tue Nov 24 12:56:35 2015 +0200 +++ b/src/imap/cmd-idle.c Tue Nov 24 13:40:12 2015 +0200 @@ -72,7 +72,7 @@ return FALSE; } -static void idle_client_input_more(struct cmd_idle_context *ctx) +static bool idle_client_input_more(struct cmd_idle_context *ctx) { struct client *client = ctx->client; @@ -83,34 +83,33 @@ case -1: /* disconnected */ client_disconnect(client, NULL); - return; + return TRUE; case -2: client->input_skip_line = TRUE; idle_finish(ctx, FALSE, TRUE); - client_continue_pending_input(client); - return; + return TRUE; } if (ctx->sync_ctx != NULL) { /* we're still sending output to client. wait until it's all sent so we don't lose any changes. */ io_remove(&client->io); - return; + return FALSE; } - if (idle_client_handle_input(ctx, TRUE)) { - if (!client->disconnected) - client_continue_pending_input(client); - } + return idle_client_handle_input(ctx, TRUE); } static void idle_client_input(struct cmd_idle_context *ctx) { struct client *client = ctx->client; - idle_client_input_more(ctx); - if (client->disconnected) - client_destroy(client, NULL); + if (idle_client_input_more(ctx)) { + if (client->disconnected) + client_destroy(client, NULL); + else + client_continue_pending_input(client); + } } static void keepalive_timeout(struct cmd_idle_context *ctx) @@ -271,7 +270,7 @@ /* input is pending */ client->io = io_add_istream(client->input, idle_client_input, ctx); - idle_client_input_more(ctx); + (void)idle_client_input_more(ctx); } return FALSE; } From dovecot at dovecot.org Tue Nov 24 12:47:54 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 24 Nov 2015 12:47:54 +0000 Subject: dovecot-2.2: imap: Added extra assert checks to make sure comman... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ecfd706b0e21 changeset: 19415:ecfd706b0e21 user: Timo Sirainen date: Tue Nov 24 13:41:58 2015 +0200 description: imap: Added extra assert checks to make sure command states are consistent. diffstat: src/imap/imap-client.c | 72 ++++++++++++++++++++++++++++++++++++++++--------- src/imap/imap-client.h | 2 + 2 files changed, 61 insertions(+), 13 deletions(-) diffs (121 lines): diff -r 20e51832875e -r ecfd706b0e21 src/imap/imap-client.c --- a/src/imap/imap-client.c Tue Nov 24 13:40:12 2015 +0200 +++ b/src/imap/imap-client.c Tue Nov 24 13:41:58 2015 +0200 @@ -799,37 +799,84 @@ } } -void client_continue_pending_input(struct client *client) +static void client_check_command_hangs(struct client *client) { - i_assert(!client->handling_input); + struct client_command_context *cmd; + unsigned int unfinished_count = 0; + bool have_wait_unfinished = FALSE; + for (cmd = client->command_queue; cmd != NULL; cmd = cmd->next) { + switch (cmd->state) { + case CLIENT_COMMAND_STATE_WAIT_INPUT: + i_assert(client->io != NULL); + unfinished_count++; + break; + case CLIENT_COMMAND_STATE_WAIT_OUTPUT: + i_assert((io_loop_find_fd_conditions(current_ioloop, client->fd_out) & IO_WRITE) != 0); + unfinished_count++; + break; + case CLIENT_COMMAND_STATE_WAIT_EXTERNAL: + unfinished_count++; + break; + case CLIENT_COMMAND_STATE_WAIT_UNAMBIGUITY: + have_wait_unfinished = TRUE; + break; + case CLIENT_COMMAND_STATE_WAIT_SYNC: + if ((io_loop_find_fd_conditions(current_ioloop, client->fd_out) & IO_WRITE) == 0) + have_wait_unfinished = TRUE; + else { + /* we have an output callback, which will be + called soon and it'll run cmd_sync_delayed(). + FIXME: is this actually wanted? */ + } + break; + case CLIENT_COMMAND_STATE_DONE: + i_unreached(); + } + } + i_assert(!have_wait_unfinished || unfinished_count > 0); +} + +static bool client_remove_pending_unambiguity(struct client *client) +{ if (client->input_lock != NULL) { /* there's a command that has locked the input */ struct client_command_context *cmd = client->input_lock; if (cmd->state != CLIENT_COMMAND_STATE_WAIT_UNAMBIGUITY) - return; + return FALSE; /* the command is waiting for existing ambiguity causing commands to finish. */ if (client_command_is_ambiguous(cmd)) { /* we could be waiting for existing sync to finish */ if (!cmd_sync_delayed(client)) - return; + return FALSE; if (client_command_is_ambiguous(cmd)) - return; + return FALSE; } cmd->state = CLIENT_COMMAND_STATE_WAIT_INPUT; } + return TRUE; +} - client_add_missing_io(client); +void client_continue_pending_input(struct client *client) +{ + i_assert(!client->handling_input); - /* if there's unread data in buffer, handle it. */ - if (i_stream_get_data_size(client->input) > 0 && - !client->disconnected) { - if (client_handle_input(client)) - client_continue_pending_input(client); + /* this function is called at the end of I/O callbacks (and only there). + fix up the command states and verify that they're correct. */ + while (client_remove_pending_unambiguity(client)) { + client_add_missing_io(client); + + /* if there's unread data in buffer, handle it. */ + if (i_stream_get_data_size(client->input) == 0 || + client->disconnected) + break; + if (!client_handle_input(client)) + break; } + client_check_command_hangs(client); } /* Skip incoming data until newline is found, @@ -1021,8 +1068,7 @@ cmd_sync_delayed(client); } else if (client->input_lock->state == CLIENT_COMMAND_STATE_WAIT_UNAMBIGUITY) { /* the command may be waiting for previous command to sync. */ - if (cmd_sync_delayed(client)) - client_continue_pending_input(client); + cmd_sync_delayed(client); } return TRUE; } diff -r 20e51832875e -r ecfd706b0e21 src/imap/imap-client.h --- a/src/imap/imap-client.h Tue Nov 24 13:40:12 2015 +0200 +++ b/src/imap/imap-client.h Tue Nov 24 13:41:58 2015 +0200 @@ -270,6 +270,8 @@ void client_command_free(struct client_command_context **cmd); bool client_handle_unfinished_cmd(struct client_command_context *cmd); +/* Handle any pending command input. This must be run at the end of all + I/O callbacks after they've (potentially) finished some commands. */ void client_continue_pending_input(struct client *client); void client_add_missing_io(struct client *client); const char *client_stats(struct client *client); From dovecot at dovecot.org Tue Nov 24 12:48:05 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 24 Nov 2015 12:48:05 +0000 Subject: dovecot-2.2: imap: When logging command disconnection info, log ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d204b943dd21 changeset: 19416:d204b943dd21 user: Timo Sirainen date: Tue Nov 24 13:42:58 2015 +0200 description: imap: When logging command disconnection info, log the oldest command's info (not newest) diffstat: src/imap/imap-client.c | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diffs (38 lines): diff -r ecfd706b0e21 -r d204b943dd21 src/imap/imap-client.c --- a/src/imap/imap-client.c Tue Nov 24 13:41:58 2015 +0200 +++ b/src/imap/imap-client.c Tue Nov 24 13:42:58 2015 +0200 @@ -265,7 +265,7 @@ static const char *client_get_commands_status(struct client *client) { - struct client_command_context *cmd; + struct client_command_context *cmd, *last_cmd = NULL; unsigned int msecs_in_ioloop; uint64_t running_usecs = 0, ioloop_wait_usecs; unsigned long long bytes_in = 0, bytes_out = 0; @@ -285,6 +285,7 @@ running_usecs += cmd->running_usecs; bytes_in += cmd->bytes_in; bytes_out += cmd->bytes_out; + last_cmd = cmd; } cond = io_loop_find_fd_conditions(current_ioloop, client->fd_out); @@ -299,7 +300,7 @@ ioloop_wait_usecs = io_loop_get_wait_usecs(current_ioloop); msecs_in_ioloop = (ioloop_wait_usecs - - client->command_queue->start_ioloop_wait_usecs + 999) / 1000; + last_cmd->start_ioloop_wait_usecs + 999) / 1000; str_printfa(str, " running for %d.%03d + waiting %s for %d.%03d secs", (int)((running_usecs+999)/1000 / 1000), (int)((running_usecs+999)/1000 % 1000), cond_str, @@ -307,7 +308,7 @@ str_printfa(str, ", %llu B in + %llu+%"PRIuSIZE_T" B out, state=%s)", bytes_in, bytes_out, o_stream_get_buffer_used_size(client->output), - client_command_state_names[client->command_queue->state]); + client_command_state_names[last_cmd->state]); return str_c(str); } From dovecot at dovecot.org Wed Nov 25 13:10:12 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 25 Nov 2015 13:10:12 +0000 Subject: dovecot-2.2: lib-fts: Fix compilation for systems without libicu. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6dafc8b24681 changeset: 19417:6dafc8b24681 user: Teemu Huovila date: Wed Nov 25 15:09:52 2015 +0200 description: lib-fts: Fix compilation for systems without libicu. The earlier patch, 194e3622d5e6, did not consider both library dependencies. diffstat: src/lib-fts/test-fts-filter.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d204b943dd21 -r 6dafc8b24681 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Tue Nov 24 13:42:58 2015 +0200 +++ b/src/lib-fts/test-fts-filter.c Wed Nov 25 15:09:52 2015 +0200 @@ -14,7 +14,7 @@ static struct fts_language english_language = { .name = "en" }; static struct fts_language french_language = { .name = "fr" }; static struct fts_language norwegian_language = { .name = "no" }; -#ifdef HAVE_FTS_STEMMER +#if defined(HAVE_LIBICU) && defined(HAVE_FTS_STEMMER) static struct fts_language swedish_language = { .name = "sv" }; #endif From dovecot at dovecot.org Wed Nov 25 13:10:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 25 Nov 2015 13:10:13 +0000 Subject: dovecot-2.2: lib-fts: Move ICU transliterator creation to fts-icu.h Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/129e61ef9de4 changeset: 19418:129e61ef9de4 user: Teemu Huovila date: Wed Nov 25 15:09:52 2015 +0200 description: lib-fts: Move ICU transliterator creation to fts-icu.h This allows others to use transliterators without so much code duplication. The transliterator still has to be destroyed by the user, with utrans_close(). diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 41 +++----------------------------- src/lib-fts/fts-icu.c | 33 ++++++++++++++++++++++++++ src/lib-fts/fts-icu.h | 3 ++ 3 files changed, 40 insertions(+), 37 deletions(-) diffs (129 lines): diff -r 6dafc8b24681 -r 129e61ef9de4 src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Wed Nov 25 15:09:52 2015 +0200 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Wed Nov 25 15:09:52 2015 +0200 @@ -14,8 +14,6 @@ struct fts_filter filter; pool_t pool; const char *transliterator_id; - const UChar *transliterator_id_utf16; - unsigned int transliterator_id_utf16_len; UTransliterator *transliterator; buffer_t *utf16_token, *trans_token; @@ -63,53 +61,22 @@ np->utf16_token = buffer_create_dynamic(pp, 128); np->trans_token = buffer_create_dynamic(pp, 128); np->utf8_token = buffer_create_dynamic(pp, 128); - fts_icu_utf8_to_utf16(np->utf16_token, id); - np->transliterator_id_utf16 = - p_memdup(pp, np->utf16_token->data, np->utf16_token->used); - np->transliterator_id_utf16_len = np->utf16_token->used / sizeof(UChar); *filter_r = &np->filter; return 0; } static int -fts_filter_normalizer_icu_create_trans(struct fts_filter_normalizer_icu *np, - const char **error_r) -{ - UErrorCode err = U_ZERO_ERROR; - UParseError perr; - - memset(&perr, 0, sizeof(perr)); - - np->transliterator = utrans_openU(np->transliterator_id_utf16, - np->transliterator_id_utf16_len, - UTRANS_FORWARD, NULL, 0, &perr, &err); - if (U_FAILURE(err)) { - string_t *str = t_str_new(128); - - str_printfa(str, "Failed to open transliterator for id '%s': %s", - np->transliterator_id, u_errorName(err)); - if (perr.line >= 1) { - /* we have only one line in our ID */ - str_printfa(str, " (parse error on offset %u)", - perr.offset); - } - *error_r = str_c(str); - return -1; - } - return 0; -} - -static int fts_filter_normalizer_icu_filter(struct fts_filter *filter, const char **token, const char **error_r) { struct fts_filter_normalizer_icu *np = (struct fts_filter_normalizer_icu *)filter; - if (np->transliterator == NULL) { - if (fts_filter_normalizer_icu_create_trans(np, error_r) < 0) + if (np->transliterator == NULL) + if (fts_icu_transliterator_create(np->transliterator_id, + &np->transliterator, + error_r) < 0) return -1; - } fts_icu_utf8_to_utf16(np->utf16_token, *token); buffer_append_zero(np->utf16_token, 2); diff -r 6dafc8b24681 -r 129e61ef9de4 src/lib-fts/fts-icu.c --- a/src/lib-fts/fts-icu.c Wed Nov 25 15:09:52 2015 +0200 +++ b/src/lib-fts/fts-icu.c Wed Nov 25 15:09:52 2015 +0200 @@ -1,7 +1,9 @@ /* Copyright (c) 2014-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "mempool.h" #include "buffer.h" +#include "str.h" #include "unichar.h" #include "fts-icu.h" @@ -164,3 +166,34 @@ } u_cleanup(); } + +int fts_icu_transliterator_create(const char *id, + UTransliterator **transliterator_r, + const char **error_r) +{ + UErrorCode err = U_ZERO_ERROR; + UParseError perr; + buffer_t *id_utf16_buf = buffer_create_dynamic(pool_datastack_create(), 2 * strlen(id)); + UChar *id_utf16; + memset(&perr, 0, sizeof(perr)); + + fts_icu_utf8_to_utf16(id_utf16_buf, id); + id_utf16 = (UChar *)str_c(id_utf16_buf); + *transliterator_r = utrans_openU(id_utf16, + id_utf16_buf->used / sizeof(UChar), + UTRANS_FORWARD, NULL, 0, &perr, &err); + if (U_FAILURE(err)) { + string_t *str = t_str_new(128); + + str_printfa(str, "Failed to open transliterator for id '%s': %s", + id, u_errorName(err)); + if (perr.line >= 1) { + /* we have only one line in our ID */ + str_printfa(str, " (parse error on offset %u)", + perr.offset); + } + *error_r = str_c(str); + return -1; + } + return 0; +} diff -r 6dafc8b24681 -r 129e61ef9de4 src/lib-fts/fts-icu.h --- a/src/lib-fts/fts-icu.h Wed Nov 25 15:09:52 2015 +0200 +++ b/src/lib-fts/fts-icu.h Wed Nov 25 15:09:52 2015 +0200 @@ -19,4 +19,7 @@ /* Free all the memory used by ICU functions. */ void fts_icu_deinit(void); +int fts_icu_transliterator_create(const char *id, + UTransliterator **transliterator_r, + const char **error_r) ; #endif From dovecot at dovecot.org Thu Nov 26 13:42:44 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 26 Nov 2015 13:42:44 +0000 Subject: dovecot-2.2: quota-clone: Avoid assert-crash when quota recalcul... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/66a04329b5e9 changeset: 19419:66a04329b5e9 user: Timo Sirainen date: Thu Nov 26 15:42:35 2015 +0200 description: quota-clone: Avoid assert-crash when quota recalculation is triggered. Quota recalculation opened all mailboxes and got us back to quota_clone_flush(), which caused another dict transaction to be opened, which caused a crash with some dict backends. diffstat: src/plugins/quota-clone/quota-clone-plugin.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diffs (27 lines): diff -r 129e61ef9de4 -r 66a04329b5e9 src/plugins/quota-clone/quota-clone-plugin.c --- a/src/plugins/quota-clone/quota-clone-plugin.c Wed Nov 25 15:09:52 2015 +0200 +++ b/src/plugins/quota-clone/quota-clone-plugin.c Thu Nov 26 15:42:35 2015 +0200 @@ -29,6 +29,7 @@ struct quota_clone_mailbox { union mailbox_module_context module_ctx; bool quota_changed; + bool quota_flushing; }; static void quota_clone_flush(struct mailbox *box) @@ -114,8 +115,14 @@ struct quota_clone_mailbox *qbox = QUOTA_CLONE_CONTEXT(box); qbox->module_ctx.super.close(box); - if (qbox->quota_changed) + + if (qbox->quota_flushing) { + /* recursing back from quota recalculation */ + } else if (qbox->quota_changed) { + qbox->quota_flushing = TRUE; quota_clone_flush(box); + qbox->quota_flushing = FALSE; + } } static void quota_clone_mailbox_allocated(struct mailbox *box) From dovecot at dovecot.org Fri Nov 27 11:06:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 27 Nov 2015 11:06:10 +0000 Subject: dovecot-2.2: imap: SETMETADATA didn't set ostream output handler... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/db90e76f44dc changeset: 19420:db90e76f44dc user: Timo Sirainen date: Fri Nov 27 13:06:01 2015 +0200 description: imap: SETMETADATA didn't set ostream output handler back This caused hangs afterwards when receiving long replies (e.g. FETCH). diffstat: src/imap/cmd-setmetadata.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 66a04329b5e9 -r db90e76f44dc src/imap/cmd-setmetadata.c --- a/src/imap/cmd-setmetadata.c Thu Nov 26 15:42:35 2015 +0200 +++ b/src/imap/cmd-setmetadata.c Fri Nov 27 13:06:01 2015 +0200 @@ -27,6 +27,9 @@ static void cmd_setmetadata_deinit(struct imap_setmetadata_context *ctx) { + o_stream_set_flush_callback(ctx->cmd->client->output, + client_output, ctx->cmd->client); + ctx->cmd->client->input_lock = NULL; imap_parser_unref(&ctx->parser); if (ctx->trans != NULL) From dovecot at dovecot.org Fri Nov 27 11:59:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 27 Nov 2015 11:59:31 +0000 Subject: dovecot-2.2: Added mailbox { autoexpunge } setting. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/82e6a3baa001 changeset: 19421:82e6a3baa001 user: Timo Sirainen date: Fri Nov 27 13:59:22 2015 +0200 description: Added mailbox { autoexpunge } setting. This can be used to automatically expunge mails from specified mailboxes after they're old enough. The expunges are done when the user is being deinitialized. mailbox_list_index=yes should be enabled to have the best performance with this setting. Example: namespace inbox { mailbox Spam { auto = create special_use = \Junk autoexpunge = 30d } } diffstat: src/lib-storage/Makefile.am | 1 + src/lib-storage/list/mailbox-list-index-status.c | 117 ++++++++++++++++++++++- src/lib-storage/list/mailbox-list-index.h | 2 +- src/lib-storage/mail-autoexpunge.c | 93 ++++++++++++++++++ src/lib-storage/mail-autoexpunge.h | 6 + src/lib-storage/mail-storage-private.h | 5 + src/lib-storage/mail-storage-settings.c | 4 +- src/lib-storage/mail-storage-settings.h | 1 + src/lib-storage/mail-storage.h | 11 +- src/lib-storage/mail-user.c | 3 + 10 files changed, 237 insertions(+), 6 deletions(-) diffs (truncated from 415 to 300 lines): diff -r db90e76f44dc -r 82e6a3baa001 src/lib-storage/Makefile.am --- a/src/lib-storage/Makefile.am Fri Nov 27 13:06:01 2015 +0200 +++ b/src/lib-storage/Makefile.am Fri Nov 27 13:59:22 2015 +0200 @@ -24,6 +24,7 @@ fail-mailbox.c \ fail-mail.c \ mail.c \ + mail-autoexpunge.c \ mail-copy.c \ mail-error.c \ mail-namespace.c \ diff -r db90e76f44dc -r 82e6a3baa001 src/lib-storage/list/mailbox-list-index-status.c --- a/src/lib-storage/list/mailbox-list-index-status.c Fri Nov 27 13:06:01 2015 +0200 +++ b/src/lib-storage/list/mailbox-list-index-status.c Fri Nov 27 13:59:22 2015 +0200 @@ -15,11 +15,13 @@ guid_128_t guid; uint32_t seq; struct mailbox_index_vsize vsize; + uint32_t first_uid; bool rec_changed; bool msgs_changed; bool hmodseq_changed; bool vsize_changed; + bool first_saved_changed; }; struct index_list_storage_module index_list_storage_module = @@ -268,6 +270,30 @@ } static int +index_list_get_cached_first_saved(struct mailbox *box, + struct mailbox_index_first_saved *first_saved_r) +{ + struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(box->list); + struct mail_index_view *view; + const void *data; + bool expunged; + uint32_t seq; + int ret; + + memset(first_saved_r, 0, sizeof(*first_saved_r)); + + if ((ret = index_list_open_view(box, &view, &seq)) <= 0) + return ret; + + mail_index_lookup_ext(view, seq, ilist->first_saved_ext_id, + &data, &expunged); + if (data != NULL) + memcpy(first_saved_r, data, sizeof(*first_saved_r)); + mail_index_view_close(&view); + return first_saved_r->timestamp != 0 ? 1 : 0; +} + +static int index_list_try_get_metadata(struct mailbox *box, enum mailbox_metadata_items items, struct mailbox_metadata *metadata_r) @@ -286,7 +312,8 @@ /* see if we have a chance of fulfilling this without opening the mailbox. */ noncached_items = items & ~(MAILBOX_METADATA_GUID | - MAILBOX_METADATA_VIRTUAL_SIZE); + MAILBOX_METADATA_VIRTUAL_SIZE | + MAILBOX_METADATA_FIRST_SAVE_DATE); if ((noncached_items & MAILBOX_METADATA_PHYSICAL_SIZE) != 0 && box->mail_vfuncs->get_physical_size == box->mail_vfuncs->get_virtual_size) @@ -306,6 +333,15 @@ if ((items & MAILBOX_METADATA_PHYSICAL_SIZE) != 0) metadata_r->physical_size = metadata_r->virtual_size; } + if ((items & MAILBOX_METADATA_FIRST_SAVE_DATE) != 0) { + struct mailbox_index_first_saved first_saved; + + if ((ret = index_list_get_cached_first_saved(box, &first_saved)) <= 0) + return ret; + metadata_r->first_save_date = + first_saved.timestamp == (uint32_t)-1 ? (time_t)-1 : + first_saved.timestamp; + } return 1; } @@ -386,6 +422,33 @@ return TRUE; } +static void +index_list_first_saved_update_changes(struct mailbox *box, + struct mail_index_view *list_view, + struct index_list_changes *changes) +{ + struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(box->list); + struct mailbox_index_first_saved first_saved; + const void *data; + bool expunged; + + mail_index_lookup_ext(list_view, changes->seq, + ilist->first_saved_ext_id, &data, &expunged); + if (data == NULL) + memset(&first_saved, 0, sizeof(first_saved)); + else + memcpy(&first_saved, data, sizeof(first_saved)); + if (mail_index_view_get_messages_count(box->view) > 0) + mail_index_lookup_uid(box->view, 1, &changes->first_uid); + if (first_saved.uid == 0 && first_saved.timestamp == 0) { + /* first time setting this */ + changes->first_saved_changed = TRUE; + } else { + changes->first_saved_changed = + changes->first_uid != first_saved.uid; + } +} + static bool index_list_has_changed(struct mailbox *box, struct mail_index_view *list_view, struct index_list_changes *changes) @@ -428,12 +491,57 @@ } if (memcmp(&old_vsize, &changes->vsize, sizeof(old_vsize)) != 0) changes->vsize_changed = TRUE; + index_list_first_saved_update_changes(box, list_view, changes); return changes->rec_changed || changes->msgs_changed || - changes->hmodseq_changed || changes->vsize_changed; + changes->hmodseq_changed || changes->vsize_changed || + changes->first_saved_changed; } static void +index_list_update_first_saved(struct mailbox *box, + struct mail_index_transaction *list_trans, + const struct index_list_changes *changes) +{ + struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(box->list); + struct mailbox_transaction_context *t; + struct mail *mail; + struct mailbox_index_first_saved first_saved; + uint32_t seq, messages_count; + time_t save_date; + int ret = 0; + + memset(&first_saved, 0, sizeof(first_saved)); + first_saved.uid = changes->first_uid; + first_saved.timestamp = (uint32_t)-1; + + if (changes->first_uid != 0) { + t = mailbox_transaction_begin(box, 0); + mail = mail_alloc(t, MAIL_FETCH_SAVE_DATE, NULL); + messages_count = mail_index_view_get_messages_count(box->view); + for (seq = 1; seq <= messages_count; seq++) { + mail_set_seq(mail, seq); + if (mail_get_save_date(mail, &save_date) == 0) { + first_saved.timestamp = save_date; + break; + } + if (mailbox_get_last_mail_error(box) != MAIL_ERROR_EXPUNGED) { + ret = -1; + break; + } + } + mail_free(&mail); + (void)mailbox_transaction_commit(&t); + } + if (ret == 0) { + mail_index_update_ext(list_trans, changes->seq, + ilist->first_saved_ext_id, + &first_saved, NULL); + } +} + + +static void index_list_update(struct mailbox *box, struct mail_index_view *list_view, struct mail_index_transaction *list_trans, const struct index_list_changes *changes) @@ -480,6 +588,8 @@ ilist->vsize_ext_id, &changes->vsize, NULL); } + if (changes->first_saved_changed) + index_list_update_first_saved(box, list_trans, changes); } static int index_list_update_mailbox(struct mailbox *box) @@ -705,4 +815,7 @@ ilist->vsize_ext_id = mail_index_ext_register(ilist->index, "vsize", 0, sizeof(struct mailbox_index_vsize), sizeof(uint64_t)); + ilist->first_saved_ext_id = + mail_index_ext_register(ilist->index, "1saved", 0, + sizeof(struct mailbox_index_first_saved), sizeof(uint32_t)); } diff -r db90e76f44dc -r 82e6a3baa001 src/lib-storage/list/mailbox-list-index.h --- a/src/lib-storage/list/mailbox-list-index.h Fri Nov 27 13:06:01 2015 +0200 +++ b/src/lib-storage/list/mailbox-list-index.h Fri Nov 27 13:59:22 2015 +0200 @@ -89,7 +89,7 @@ const char *path; struct mail_index *index; uint32_t ext_id, msgs_ext_id, hmodseq_ext_id, subs_hdr_ext_id; - uint32_t vsize_ext_id; + uint32_t vsize_ext_id, first_saved_ext_id; struct timeval last_refresh_timeval; pool_t mailbox_pool; diff -r db90e76f44dc -r 82e6a3baa001 src/lib-storage/mail-autoexpunge.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/mail-autoexpunge.c Fri Nov 27 13:59:22 2015 +0200 @@ -0,0 +1,93 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "mail-storage-private.h" +#include "mail-namespace.h" +#include "mail-user.h" +#include "mail-autoexpunge.h" + +static int mailbox_autoexpunge(struct mailbox *box, time_t expire_time) +{ + struct mailbox_transaction_context *t; + struct mail *mail; + struct mailbox_metadata metadata; + const struct mail_index_header *hdr; + uint32_t seq; + time_t timestamp; + int ret = 0; + + /* first try to check quickly from mailbox list index if we should + bother opening this mailbox. */ + if (mailbox_get_metadata(box, MAILBOX_METADATA_FIRST_SAVE_DATE, + &metadata) == 0) { + if (metadata.first_save_date == (time_t)-1 || + metadata.first_save_date > expire_time) + return 0; + } + + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) < 0) { + if (mailbox_get_last_mail_error(box) == MAIL_ERROR_NOTFOUND) { + /* autocreated mailbox doesn't exist yet */ + return 0; + } + return -1; + } + + t = mailbox_transaction_begin(box, 0); + mail = mail_alloc(t, 0, NULL); + + hdr = mail_index_get_header(box->view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_set_seq(mail, seq); + if (mail_get_save_date(mail, ×tamp) == 0) { + if (timestamp > expire_time) + break; + mail_expunge(mail); + } else if (mailbox_get_last_mail_error(box) == MAIL_ERROR_EXPUNGED) { + /* already expunged */ + } else { + /* failed */ + ret = -1; + break; + } + } + mail_free(&mail); + if (mailbox_transaction_commit(&t) < 0) + ret = -1; + return ret; +} + +static void mail_namespace_autoexpunge(struct mail_namespace *ns) +{ + struct mailbox_settings *const *box_set; + struct mailbox *box; + time_t expire_time; + + if (!array_is_created(&ns->set->mailboxes)) + return; + + array_foreach(&ns->set->mailboxes, box_set) { + if ((*box_set)->autoexpunge == 0 || + ioloop_time < (*box_set)->autoexpunge) + continue; + expire_time = ioloop_time - (*box_set)->autoexpunge; + box = mailbox_alloc(ns->list, (*box_set)->name, 0); + if (mailbox_autoexpunge(box, expire_time) < 0) { + i_error("Failed to autoexpunge mailbox '%s': %s", + mailbox_get_vname(box), + mailbox_get_last_error(box, NULL)); + } + mailbox_free(&box); + } +} + +void mail_user_autoexpunge(struct mail_user *user) +{ + struct mail_namespace *ns; + + for (ns = user->namespaces; ns != NULL; ns = ns->next) { From dovecot at dovecot.org Fri Nov 27 12:12:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 27 Nov 2015 12:12:09 +0000 Subject: dovecot-2.2: imap: Free mail_user only after client is disconnec... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1217e5610865 changeset: 19422:1217e5610865 user: Timo Sirainen date: Fri Nov 27 14:01:14 2015 +0200 description: imap: Free mail_user only after client is disconnected. diffstat: src/imap/imap-client.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (22 lines): diff -r 82e6a3baa001 -r 1217e5610865 src/imap/imap-client.c --- a/src/imap/imap-client.c Fri Nov 27 13:59:22 2015 +0200 +++ b/src/imap/imap-client.c Fri Nov 27 14:01:14 2015 +0200 @@ -367,7 +367,6 @@ mail_user_get_anvil_userip_ident(client->user), "\n", NULL)); } - mail_user_unref(&client->user); if (client->free_parser != NULL) imap_parser_unref(&client->free_parser); @@ -386,6 +385,10 @@ if (client->fd_in != client->fd_out) net_disconnect(client->fd_out); + /* Free the user after client is already disconnected. It may start + some background work like autoexpunging. */ + mail_user_unref(&client->user); + if (array_is_created(&client->search_saved_uidset)) array_free(&client->search_saved_uidset); if (array_is_created(&client->search_updates)) From dovecot at dovecot.org Fri Nov 27 12:12:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 27 Nov 2015 12:12:09 +0000 Subject: dovecot-2.2: lib-storage: Don't add first_saved to mailbox list ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d4e5b4ab01dd changeset: 19423:d4e5b4ab01dd user: Timo Sirainen date: Fri Nov 27 14:11:57 2015 +0200 description: lib-storage: Don't add first_saved to mailbox list index unless autoexpunge is set. diffstat: src/lib-storage/list/mailbox-list-index-status.c | 9 +++++++-- src/lib-storage/mail-storage-private.h | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diffs (38 lines): diff -r 1217e5610865 -r d4e5b4ab01dd src/lib-storage/list/mailbox-list-index-status.c --- a/src/lib-storage/list/mailbox-list-index-status.c Fri Nov 27 14:01:14 2015 +0200 +++ b/src/lib-storage/list/mailbox-list-index-status.c Fri Nov 27 14:11:57 2015 +0200 @@ -336,6 +336,10 @@ if ((items & MAILBOX_METADATA_FIRST_SAVE_DATE) != 0) { struct mailbox_index_first_saved first_saved; + /* start writing first_saved to mailbox list index if it wasn't + there already. */ + box->update_first_saved = TRUE; + if ((ret = index_list_get_cached_first_saved(box, &first_saved)) <= 0) return ret; metadata_r->first_save_date = @@ -441,8 +445,9 @@ if (mail_index_view_get_messages_count(box->view) > 0) mail_index_lookup_uid(box->view, 1, &changes->first_uid); if (first_saved.uid == 0 && first_saved.timestamp == 0) { - /* first time setting this */ - changes->first_saved_changed = TRUE; + /* it's not in the index yet. we'll set it only if we've + just called MAILBOX_METADATA_FIRST_SAVE_DATE. */ + changes->first_saved_changed = box->update_first_saved; } else { changes->first_saved_changed = changes->first_uid != first_saved.uid; diff -r 1217e5610865 -r d4e5b4ab01dd src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Fri Nov 27 14:01:14 2015 +0200 +++ b/src/lib-storage/mail-storage-private.h Fri Nov 27 14:11:57 2015 +0200 @@ -378,6 +378,8 @@ unsigned int synced:1; /* Updating cache file is disabled */ unsigned int mail_cache_disabled:1; + /* Update first_saved field to mailbox list index. */ + unsigned int update_first_saved:1; }; struct mail_vfuncs { From dovecot at dovecot.org Fri Nov 27 12:24:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 27 Nov 2015 12:24:37 +0000 Subject: dovecot-2.2: Compiler warning fix for 32bit systems Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9bebd787450f changeset: 19424:9bebd787450f user: Timo Sirainen date: Fri Nov 27 14:24:23 2015 +0200 description: Compiler warning fix for 32bit systems diffstat: src/lib-storage/list/mailbox-list-index-status.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d4e5b4ab01dd -r 9bebd787450f src/lib-storage/list/mailbox-list-index-status.c --- a/src/lib-storage/list/mailbox-list-index-status.c Fri Nov 27 14:11:57 2015 +0200 +++ b/src/lib-storage/list/mailbox-list-index-status.c Fri Nov 27 14:24:23 2015 +0200 @@ -344,7 +344,7 @@ return ret; metadata_r->first_save_date = first_saved.timestamp == (uint32_t)-1 ? (time_t)-1 : - first_saved.timestamp; + (time_t)first_saved.timestamp; } return 1; } From dovecot at dovecot.org Fri Nov 27 12:42:51 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 27 Nov 2015 12:42:51 +0000 Subject: dovecot-2.2: auth: Fixed test-auth-request-var-expand unit test ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cf956e34dc11 changeset: 19425:cf956e34dc11 user: Timo Sirainen date: Fri Nov 27 14:39:32 2015 +0200 description: auth: Fixed test-auth-request-var-expand unit test on big endian CPUs diffstat: src/auth/test-auth-request-var-expand.c | 20 +++++++++++++------- 1 files changed, 13 insertions(+), 7 deletions(-) diffs (61 lines): diff -r 9bebd787450f -r cf956e34dc11 src/auth/test-auth-request-var-expand.c --- a/src/auth/test-auth-request-var-expand.c Fri Nov 27 14:24:23 2015 +0200 +++ b/src/auth/test-auth-request-var-expand.c Fri Nov 27 14:39:32 2015 +0200 @@ -22,12 +22,12 @@ .userdb = &test_userdb }; -static const struct auth_request default_test_request = { +static struct auth_request default_test_request = { .user = "-user at +domain1@+domain2", .service = "-service", - .local_ip = { .family = AF_INET, .u.ip4.s_addr = 123456789 }, - .remote_ip = { .family = AF_INET, .u.ip4.s_addr = 1234567890 }, + .local_ip = { .family = AF_INET }, + .remote_ip = { .family = AF_INET }, .client_pid = 54321, .mech_password = "-password", .mech_name = "-mech", @@ -38,8 +38,8 @@ .requested_login_user = "-loginuser at +logindomain1@+logindomain2", .session_id = "-session", - .real_local_ip = { .family = AF_INET, .u.ip4.s_addr = 223456788 }, - .real_remote_ip = { .family = AF_INET, .u.ip4.s_addr = 223456789 }, + .real_local_ip = { .family = AF_INET }, + .real_remote_ip = { .family = AF_INET }, .real_local_port = 200, .real_remote_port = 201, .master_user = "-masteruser at -masterdomain1@-masterdomain2", @@ -88,7 +88,7 @@ static const char *test_output = /* %{home} is intentionally always expanding to empty */ "+user at +domain1@+domain2\n+user\n+domain1 at +domain2\n+service\n\n" - "21.205.91.7\n210.2.150.73\n54321\n+password\n+mech\nsecured\n" + "7.91.205.21\n73.150.2.210\n54321\n+password\n+mech\nsecured\n" "21\n210\nvalid\n"; string_t *str = t_str_new(256); @@ -146,7 +146,7 @@ "%{orig_user}\n%{orig_username}\n%{orig_domain}\n"; static const char *test_output = "+loginuser at +logindomain1@+logindomain2\n+loginuser\n+logindomain1 at +logindomain2\n+session\n" - "20.174.81.13\n21.174.81.13\n200\n201\n" + "13.81.174.20\n13.81.174.21\n200\n201\n" "+masteruser at +masterdomain1@+masterdomain2\n5000\n" "+origuser at +origdomain1@+origdomain2\n+origuser\n+origdomain1 at +origdomain2\n"; string_t *str = t_str_new(256); @@ -234,6 +234,12 @@ test_auth_request_var_expand_funcs, NULL }; + + default_test_request.local_ip.u.ip4.s_addr = htonl(123456789); + default_test_request.remote_ip.u.ip4.s_addr = htonl(1234567890); + default_test_request.real_local_ip.u.ip4.s_addr = htonl(223456788); + default_test_request.real_remote_ip.u.ip4.s_addr = htonl(223456789); + test_request = default_test_request; return test_run(test_functions); } From dovecot at dovecot.org Fri Nov 27 12:42:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 27 Nov 2015 12:42:52 +0000 Subject: dovecot-2.2: lib: Fixed compiling unit test in systems where NUL... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2ddbdf05b19b changeset: 19426:2ddbdf05b19b user: Timo Sirainen date: Fri Nov 27 14:42:43 2015 +0200 description: lib: Fixed compiling unit test in systems where NULL isn't of type void* For example Solaris. diffstat: src/lib/test-ioloop.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (15 lines): diff -r cf956e34dc11 -r 2ddbdf05b19b src/lib/test-ioloop.c --- a/src/lib/test-ioloop.c Fri Nov 27 14:39:32 2015 +0200 +++ b/src/lib/test-ioloop.c Fri Nov 27 14:42:43 2015 +0200 @@ -79,9 +79,9 @@ for (i = 0; i < N_ELEMENTS(tests); i++) { if (socketpair(AF_UNIX, SOCK_STREAM, 0, tests[i].fd) < 0) i_fatal("socketpair() failed: %m"); - tests[i].io = io_add(tests[i].fd[0], tests[i].condition, io_callback, NULL); + tests[i].io = io_add(tests[i].fd[0], tests[i].condition, io_callback, (void *)NULL); } - io = io_add(tests[i-1].fd[0], IO_WRITE, io_callback, NULL); + io = io_add(tests[i-1].fd[0], IO_WRITE, io_callback, (void *)NULL); tests[i-1].condition |= IO_WRITE; for (i = 0; i < N_ELEMENTS(tests); i++) From dovecot at dovecot.org Fri Nov 27 12:57:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 27 Nov 2015 12:57:30 +0000 Subject: dovecot-2.2: zlib: Fixed copying causing cache corruption when z... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/574c1e5b3d90 changeset: 19427:574c1e5b3d90 user: Timo Sirainen date: Fri Nov 27 14:57:03 2015 +0200 description: zlib: Fixed copying causing cache corruption when zlib_save wasn't set. dovecot.index.cache entries were broken/missing if: - The zlib plugin is enabled; - The zlib_save/zlib_save_level options are NOT enabled; - The source message being copied is compressed; - The mail_log plugin is logging "copy" events; - The mail_log_fields setting includes at least one message header; - The destination mailbox folder has an index file that is recording the logged headers; - The source mailbox folder does NOT have an index file recording the logged headers. Found by Robert L Mathews. diffstat: src/plugins/zlib/zlib-plugin.c | 40 +++++++++++++++++++++++++--------------- 1 files changed, 25 insertions(+), 15 deletions(-) diffs (99 lines): diff -r 2ddbdf05b19b -r 574c1e5b3d90 src/plugins/zlib/zlib-plugin.c --- a/src/plugins/zlib/zlib-plugin.c Fri Nov 27 14:42:43 2015 +0200 +++ b/src/plugins/zlib/zlib-plugin.c Fri Nov 27 14:57:03 2015 +0200 @@ -26,6 +26,11 @@ #define MAX_INBUF_SIZE (1024*1024) #define ZLIB_MAIL_CACHE_EXPIRE_MSECS (60*1000) +struct zlib_mail { + union mail_module_context module_ctx; + bool verifying_save; +}; + struct zlib_transaction_context { union mailbox_transaction_module_context module_ctx; @@ -112,16 +117,15 @@ struct zlib_user *zuser = ZLIB_USER_CONTEXT(_mail->box->storage->user); struct zlib_mail_cache *cache = &zuser->cache; struct mail_private *mail = (struct mail_private *)_mail; - union mail_module_context *zmail = ZLIB_MAIL_CONTEXT(mail); + struct zlib_mail *zmail = ZLIB_MAIL_CONTEXT(mail); struct istream *input; const struct compression_handler *handler; - /* don't uncompress input when we are reading a mail that we're just - in the middle of saving, and we didn't do the compression ourself. - in such situation we're probably checking if the user-given input - looks compressed */ - if (_mail->saving && zuser->save_handler == NULL) - return zmail->super.istream_opened(_mail, stream); + if (zmail->verifying_save) { + /* zlib_mail_save_finish() is verifying that the user-given + input doesn't look compressed. */ + return zmail->module_ctx.super.istream_opened(_mail, stream); + } if (cache->uid == _mail->uid && cache->box == _mail->box) { /* use the cached stream. when doing partial reads it should @@ -129,7 +133,7 @@ i_stream_unref(stream); i_stream_seek(cache->input, 0); *stream = i_stream_create_limit(cache->input, (uoff_t)-1); - return zmail->super.istream_opened(_mail, stream); + return zmail->module_ctx.super.istream_opened(_mail, stream); } handler = compression_detect_handler(*stream); @@ -147,7 +151,7 @@ *stream = zlib_mail_cache_open(zuser, _mail, *stream); } - return zmail->super.istream_opened(_mail, stream); + return zmail->module_ctx.super.istream_opened(_mail, stream); } static void zlib_mail_allocated(struct mail *_mail) @@ -155,17 +159,17 @@ struct zlib_transaction_context *zt = ZLIB_CONTEXT(_mail->transaction); struct mail_private *mail = (struct mail_private *)_mail; struct mail_vfuncs *v = mail->vlast; - union mail_module_context *zmail; + struct zlib_mail *zmail; if (zt == NULL) return; - zmail = p_new(mail->pool, union mail_module_context, 1); - zmail->super = *v; - mail->vlast = &zmail->super; + zmail = p_new(mail->pool, struct zlib_mail, 1); + zmail->module_ctx.super = *v; + mail->vlast = &zmail->module_ctx.super; v->istream_opened = zlib_istream_opened; - MODULE_CONTEXT_SET_SELF(mail, zlib_mail_module, zmail); + MODULE_CONTEXT_SET(mail, zlib_mail_module, zmail); } static struct mailbox_transaction_context * @@ -235,12 +239,18 @@ { struct mailbox *box = ctx->transaction->box; union mailbox_module_context *zbox = ZLIB_CONTEXT(box); + struct mail_private *mail = (struct mail_private *)ctx->dest_mail; + struct zlib_mail *zmail = ZLIB_MAIL_CONTEXT(mail); struct istream *input; + int ret; if (zbox->super.save_finish(ctx) < 0) return -1; - if (mail_get_stream(ctx->dest_mail, NULL, NULL, &input) < 0) + zmail->verifying_save = TRUE; + ret = mail_get_stream(ctx->dest_mail, NULL, NULL, &input); + zmail->verifying_save = FALSE; + if (ret < 0) return -1; if (compression_detect_handler(input) != NULL) { From dovecot at dovecot.org Fri Nov 27 13:24:54 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 27 Nov 2015 13:24:54 +0000 Subject: dovecot-2.2: fts-solr: Fixed escaping query parameters. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9d5c59d98857 changeset: 19428:9d5c59d98857 user: Timo Sirainen date: Fri Nov 27 15:24:41 2015 +0200 description: fts-solr: Fixed escaping query parameters. Solr documentation says that "quoted string" would already work without escaping, but that doesn't seem to be true (we were also missing the \" escaping there). So we'll now escape all the special characters without quotes around it, which seems to work. Also added '/' to list of special characters, which is used by Solr 4.0. diffstat: src/plugins/fts-solr/fts-backend-solr-old.c | 24 ++++++++++++++++++------ src/plugins/fts-solr/fts-backend-solr.c | 24 ++++++++++++++++++------ 2 files changed, 36 insertions(+), 12 deletions(-) diffs (104 lines): diff -r 574c1e5b3d90 -r 9d5c59d98857 src/plugins/fts-solr/fts-backend-solr-old.c --- a/src/plugins/fts-solr/fts-backend-solr-old.c Fri Nov 27 14:57:03 2015 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr-old.c Fri Nov 27 15:24:41 2015 +0200 @@ -42,6 +42,8 @@ bool documents_added; }; +static const char *solr_escape_chars = "+-&|!(){}[]^\"~*?:\\/ "; + static bool is_valid_xml_char(unichar_t chr) { /* Valid characters in XML: @@ -139,18 +141,28 @@ return str_c(tmp); } +static const char *solr_escape(const char *str) +{ + string_t *ret; + unsigned int i; + + ret = t_str_new(strlen(str) + 16); + for (i = 0; str[i] != '\0'; i++) { + if (strchr(solr_escape_chars, str[i]) != NULL) + str_append_c(ret, '\\'); + str_append_c(ret, str[i]); + } + return str_c(ret); +} + static void solr_quote(string_t *dest, const char *str) { - str_append_c(dest, '"'); - str_append(dest, str_escape(str)); - str_append_c(dest, '"'); + str_append(dest, solr_escape(str)); } static void solr_quote_http(string_t *dest, const char *str) { - str_append(dest, "%22"); - http_url_escape_param(dest, str); - str_append(dest, "%22"); + http_url_escape_param(dest, solr_escape(str)); } static void fts_solr_set_default_ns(struct solr_fts_backend *backend) diff -r 574c1e5b3d90 -r 9d5c59d98857 src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Fri Nov 27 14:57:03 2015 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr.c Fri Nov 27 15:24:41 2015 +0200 @@ -63,6 +63,8 @@ unsigned int truncate_header:1; }; +static const char *solr_escape_chars = "+-&|!(){}[]^\"~*?:\\/ "; + static bool is_valid_xml_char(unichar_t chr) { /* Valid characters in XML: @@ -143,11 +145,23 @@ xml_encode_data(dest, (const unsigned char *)str, strlen(str)); } +static const char *solr_escape(const char *str) +{ + string_t *ret; + unsigned int i; + + ret = t_str_new(strlen(str) + 16); + for (i = 0; str[i] != '\0'; i++) { + if (strchr(solr_escape_chars, str[i]) != NULL) + str_append_c(ret, '\\'); + str_append_c(ret, str[i]); + } + return str_c(ret); +} + static void solr_quote_http(string_t *dest, const char *str) { - str_append(dest, "%22"); - http_url_escape_param(dest, str); - str_append(dest, "%22"); + http_url_escape_param(dest, solr_escape(str)); } static struct fts_backend *fts_backend_solr_alloc(void) @@ -623,8 +637,6 @@ static bool solr_need_escaping(const char *str) { - const char *solr_escape_chars = "+-&|!(){}[]^\"~*?:\\ "; - for (; *str != '\0'; str++) { if (strchr(solr_escape_chars, *str) != NULL) return TRUE; @@ -640,7 +652,7 @@ if (!arg->fuzzy || solr_need_escaping(arg->value.str)) solr_quote_http(str, arg->value.str); else { - str_append(str, arg->value.str); + http_url_escape_param(str, arg->value.str); str_append_c(str, '~'); } } From dovecot at dovecot.org Fri Nov 27 13:26:51 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 27 Nov 2015 13:26:51 +0000 Subject: dovecot-2.2: Compiler warning fix for 32bit systems Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/695391a09045 changeset: 19429:695391a09045 user: Timo Sirainen date: Fri Nov 27 15:26:31 2015 +0200 description: Compiler warning fix for 32bit systems diffstat: src/lib-storage/mail-autoexpunge.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 9d5c59d98857 -r 695391a09045 src/lib-storage/mail-autoexpunge.c --- a/src/lib-storage/mail-autoexpunge.c Fri Nov 27 15:24:41 2015 +0200 +++ b/src/lib-storage/mail-autoexpunge.c Fri Nov 27 15:26:31 2015 +0200 @@ -69,7 +69,7 @@ array_foreach(&ns->set->mailboxes, box_set) { if ((*box_set)->autoexpunge == 0 || - ioloop_time < (*box_set)->autoexpunge) + (unsigned int)ioloop_time < (*box_set)->autoexpunge) continue; expire_time = ioloop_time - (*box_set)->autoexpunge; box = mailbox_alloc(ns->list, (*box_set)->name, 0); From dovecot at dovecot.org Fri Nov 27 13:50:29 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 27 Nov 2015 13:50:29 +0000 Subject: dovecot-2.2: fts-solr: Fixed sending empty parameters. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/feba082aede2 changeset: 19430:feba082aede2 user: Timo Sirainen date: Fri Nov 27 15:49:58 2015 +0200 description: fts-solr: Fixed sending empty parameters. Solr probably doesn't do anything useful with them, but we shouldn't get 400 Bad Request errors. diffstat: src/plugins/fts-solr/fts-backend-solr-old.c | 3 +++ src/plugins/fts-solr/fts-backend-solr.c | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diffs (38 lines): diff -r 695391a09045 -r feba082aede2 src/plugins/fts-solr/fts-backend-solr-old.c --- a/src/plugins/fts-solr/fts-backend-solr-old.c Fri Nov 27 15:26:31 2015 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr-old.c Fri Nov 27 15:49:58 2015 +0200 @@ -146,6 +146,9 @@ string_t *ret; unsigned int i; + if (str[0] == '\0') + return "\"\""; + ret = t_str_new(strlen(str) + 16); for (i = 0; str[i] != '\0'; i++) { if (strchr(solr_escape_chars, str[i]) != NULL) diff -r 695391a09045 -r feba082aede2 src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Fri Nov 27 15:26:31 2015 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr.c Fri Nov 27 15:49:58 2015 +0200 @@ -161,7 +161,10 @@ static void solr_quote_http(string_t *dest, const char *str) { - http_url_escape_param(dest, solr_escape(str)); + if (str[0] != '\0') + http_url_escape_param(dest, solr_escape(str)); + else + str_append(dest, "\"\""); } static struct fts_backend *fts_backend_solr_alloc(void) @@ -649,7 +652,8 @@ /* currently we'll just disable fuzzy searching if there are any parameters that need escaping. solr doesn't seem to give good fuzzy results even if we did escape them.. */ - if (!arg->fuzzy || solr_need_escaping(arg->value.str)) + if (!arg->fuzzy || arg->value.str[0] == '\0' || + solr_need_escaping(arg->value.str)) solr_quote_http(str, arg->value.str); else { http_url_escape_param(str, arg->value.str); From dovecot at dovecot.org Fri Nov 27 13:50:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 27 Nov 2015 13:50:56 +0000 Subject: dovecot-2.2: lib: Added extra assert to i_stream_read() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/69cb603dec84 changeset: 19431:69cb603dec84 user: Timo Sirainen date: Fri Nov 27 15:50:44 2015 +0200 description: lib: Added extra assert to i_stream_read() diffstat: src/lib/istream.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (22 lines): diff -r feba082aede2 -r 69cb603dec84 src/lib/istream.c --- a/src/lib/istream.c Fri Nov 27 15:49:58 2015 +0200 +++ b/src/lib/istream.c Fri Nov 27 15:50:44 2015 +0200 @@ -6,6 +6,8 @@ #include "str.h" #include "istream-private.h" +static bool i_stream_is_buffer_invalid(const struct istream_private *stream); + void i_stream_set_name(struct istream *stream, const char *name) { i_free(stream->real_stream->iostream.name); @@ -184,6 +186,9 @@ } i_stream_update(_stream); + /* verify that parents' access_counters are valid. the parent's + i_stream_read() should guarantee this. */ + i_assert(!i_stream_is_buffer_invalid(_stream)); return ret; } From dovecot at dovecot.org Sat Nov 28 11:36:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 28 Nov 2015 11:36:24 +0000 Subject: dovecot-2.2: Makefile: Added missing mail-autoexpunge.h Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a9c75e48cf8c changeset: 19432:a9c75e48cf8c user: Timo Sirainen date: Sat Nov 28 13:36:14 2015 +0200 description: Makefile: Added missing mail-autoexpunge.h diffstat: src/lib-storage/Makefile.am | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 69cb603dec84 -r a9c75e48cf8c src/lib-storage/Makefile.am --- a/src/lib-storage/Makefile.am Fri Nov 27 15:50:44 2015 +0200 +++ b/src/lib-storage/Makefile.am Sat Nov 28 13:36:14 2015 +0200 @@ -61,6 +61,7 @@ headers = \ fail-mail-storage.h \ + mail-autoexpunge.h \ mail-copy.h \ mail-error.h \ mail-namespace.h \ From dovecot at dovecot.org Sat Nov 28 21:50:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 28 Nov 2015 21:50:30 +0000 Subject: dovecot-2.2: lib: Created t_str_trim() functions to trim charact... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/922de4bc4777 changeset: 19433:922de4bc4777 user: Stephan Bosch date: Sat Nov 28 23:50:14 2015 +0200 description: lib: Created t_str_trim() functions to trim characters from beginning and end of string. diffstat: src/lib/strfuncs.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++ src/lib/strfuncs.h | 6 +++++ src/lib/test-strfuncs.c | 51 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 0 deletions(-) diffs (148 lines): diff -r a9c75e48cf8c -r 922de4bc4777 src/lib/strfuncs.c --- a/src/lib/strfuncs.c Sat Nov 28 13:36:14 2015 +0200 +++ b/src/lib/strfuncs.c Sat Nov 28 23:50:14 2015 +0200 @@ -364,6 +364,63 @@ return str_ucase(t_strdup_noconst(str)); } +const char *t_str_trim(const char *str, const char *chars) +{ + const char *p, *pend, *begin; + + pend = str + strlen(str); + if (pend == str) + return ""; + + p = str; + while (p < pend && strchr(chars, *p) != NULL) + p++; + begin = p; + + p = pend - 1; + while (p > begin && strchr(chars, *p) != NULL) + p--; + + if (p <= begin) + return ""; + return t_strdup_until(begin, p+1); +} + +const char *str_ltrim(const char *str, const char *chars) +{ + const char *p; + + if (*str == '\0') + return ""; + + p = str; + while (*p != '\0' && strchr(chars, *p) != NULL) + p++; + + return p; +} + +const char *t_str_ltrim(const char *str, const char *chars) +{ + return t_strdup(str_ltrim(str, chars)); +} + +const char *t_str_rtrim(const char *str, const char *chars) +{ + const char *p, *pend; + + pend = str + strlen(str); + if (pend == str) + return ""; + + p = pend - 1; + while (p > str && strchr(chars, *p) != NULL) + p--; + if (p <= str) + return ""; + return t_strdup_until(str, p+1); +} + int null_strcmp(const char *s1, const char *s2) { if (s1 == NULL) diff -r a9c75e48cf8c -r 922de4bc4777 src/lib/strfuncs.h --- a/src/lib/strfuncs.h Sat Nov 28 13:36:14 2015 +0200 +++ b/src/lib/strfuncs.h Sat Nov 28 23:50:14 2015 +0200 @@ -53,6 +53,12 @@ const char *t_str_lcase(const char *str); const char *t_str_ucase(const char *str); +/* Trim matching chars from either side of the string */ +const char *str_ltrim(const char *str, const char *chars); +const char *t_str_ltrim(const char *str, const char *chars); +const char *t_str_rtrim(const char *str, const char *chars); +const char *t_str_trim(const char *str, const char *chars); + int null_strcmp(const char *s1, const char *s2) ATTR_PURE; int bsearch_strcmp(const char *key, const char *const *member) ATTR_PURE; int bsearch_strcasecmp(const char *key, const char *const *member) ATTR_PURE; diff -r a9c75e48cf8c -r 922de4bc4777 src/lib/test-strfuncs.c --- a/src/lib/test-strfuncs.c Sat Nov 28 13:36:14 2015 +0200 +++ b/src/lib/test-strfuncs.c Sat Nov 28 23:50:14 2015 +0200 @@ -109,10 +109,61 @@ test_end(); } +static void test_t_str_trim(void) +{ + test_begin("t_str_trim"); + test_assert(strcmp(t_str_trim("foo", ""), "foo") == 0); + test_assert(strcmp(t_str_trim("foo", " "), "foo") == 0); + test_assert(strcmp(t_str_trim("foo ", " "), "foo") == 0); + test_assert(strcmp(t_str_trim(" foo", " "), "foo") == 0); + test_assert(strcmp(t_str_trim(" foo ", " "), "foo") == 0); + test_assert(strcmp(t_str_trim("\tfoo ", "\t "), "foo") == 0); + test_assert(strcmp(t_str_trim(" \tfoo\t ", "\t "), "foo") == 0); + test_assert(strcmp(t_str_trim("\r \tfoo\t \r", "\t \r"), "foo") == 0); + test_assert(strcmp(t_str_trim("\r \tfoo foo\t \r", "\t \r"), "foo foo") == 0); + test_assert(strcmp(t_str_trim("\tfoo\tfoo\t", "\t \r"), "foo\tfoo") == 0); + test_end(); +} + +static void test_t_str_ltrim(void) +{ + test_begin("t_str_ltrim"); + test_assert(strcmp(t_str_ltrim("foo", ""), "foo") == 0); + test_assert(strcmp(t_str_ltrim("foo", " "), "foo") == 0); + test_assert(strcmp(t_str_ltrim("foo ", " "), "foo ") == 0); + test_assert(strcmp(t_str_ltrim(" foo", " "), "foo") == 0); + test_assert(strcmp(t_str_ltrim(" foo ", " "), "foo ") == 0); + test_assert(strcmp(t_str_ltrim("\tfoo ", "\t "), "foo ") == 0); + test_assert(strcmp(t_str_ltrim(" \tfoo\t ", "\t "), "foo\t ") == 0); + test_assert(strcmp(t_str_ltrim("\r \tfoo\t \r", "\t \r"), "foo\t \r") == 0); + test_assert(strcmp(t_str_ltrim("\r \tfoo foo\t \r", "\t \r"), "foo foo\t \r") == 0); + test_assert(strcmp(t_str_ltrim("\tfoo\tfoo\t", "\t \r"), "foo\tfoo\t") == 0); + test_end(); +} + +static void test_t_str_rtrim(void) +{ + test_begin("t_str_rtrim"); + test_assert(strcmp(t_str_rtrim("foo", ""), "foo") == 0); + test_assert(strcmp(t_str_rtrim("foo", " "), "foo") == 0); + test_assert(strcmp(t_str_rtrim("foo ", " "), "foo") == 0); + test_assert(strcmp(t_str_rtrim(" foo", " "), " foo") == 0); + test_assert(strcmp(t_str_rtrim(" foo ", " "), " foo") == 0); + test_assert(strcmp(t_str_rtrim("\tfoo ", "\t "), "\tfoo") == 0); + test_assert(strcmp(t_str_rtrim(" \tfoo\t ", "\t "), " \tfoo") == 0); + test_assert(strcmp(t_str_rtrim("\r \tfoo\t \r", "\t \r"), "\r \tfoo") == 0); + test_assert(strcmp(t_str_rtrim("\r \tfoo foo\t \r", "\t \r"), "\r \tfoo foo") == 0); + test_assert(strcmp(t_str_rtrim("\tfoo\tfoo\t", "\t \r"), "\tfoo\tfoo") == 0); + test_end(); +} + void test_strfuncs(void) { test_p_strarray_dup(); test_t_strsplit(); test_t_strsplit_tab(); test_t_str_replace(); + test_t_str_trim(); + test_t_str_ltrim(); + test_t_str_rtrim(); } From pigeonhole at rename-it.nl Sat Nov 28 22:53:32 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 28 Nov 2015 23:53:32 +0100 Subject: dovecot-2.2-pigeonhole: Added new TODO item. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/923e78f7e23a changeset: 2134:923e78f7e23a user: Stephan Bosch date: Sat Nov 28 23:53:27 2015 +0100 description: Added new TODO item. diffstat: TODO | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r d23225e408af -r 923e78f7e23a TODO --- a/TODO Wed Nov 18 22:11:31 2015 +0100 +++ b/TODO Sat Nov 28 23:53:27 2015 +0100 @@ -1,6 +1,7 @@ Current activities: * Rework string matching: + - Give Sieve its own runtime string type, rather than (ab)using string_t. - Add support for stream matching for handling large values, e.g. from the body extension. - Improve efficiency of :matches and :contains match types. From pigeonhole at rename-it.nl Sun Nov 29 10:53:29 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 11:53:29 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Implemented the foreverypart ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/feca1b976393 changeset: 2140:feca1b976393 user: Stephan Bosch date: Sun Nov 29 11:50:59 2015 +0100 description: lib-sieve: Implemented the foreverypart and mime extensions (RFC 5703). diffstat: Makefile.am | 2 + README | 4 +- TODO | 2 +- configure.ac | 1 + src/lib-sieve/Makefile.am | 1 + src/lib-sieve/plugins/Makefile.am | 1 + src/lib-sieve/plugins/mime/Makefile.am | 24 + src/lib-sieve/plugins/mime/cmd-break.c | 273 +++++++ src/lib-sieve/plugins/mime/cmd-foreverypart.c | 339 +++++++++ src/lib-sieve/plugins/mime/ext-foreverypart.c | 62 + src/lib-sieve/plugins/mime/ext-mime-common.c | 27 + src/lib-sieve/plugins/mime/ext-mime-common.h | 80 ++ src/lib-sieve/plugins/mime/ext-mime.c | 77 ++ src/lib-sieve/plugins/mime/tag-mime.c | 704 ++++++++++++++++++++ src/lib-sieve/sieve-extensions.c | 4 +- tests/extensions/mime/content-header.svtest | 161 ++++ tests/extensions/mime/errors.svtest | 54 + tests/extensions/mime/errors/address-mime-tag.sieve | 38 + tests/extensions/mime/errors/break.sieve | 157 ++++ tests/extensions/mime/errors/exists-mime-tag.sieve | 43 + tests/extensions/mime/errors/foreverypart.sieve | 45 + tests/extensions/mime/errors/header-mime-tag.sieve | 100 ++ tests/extensions/mime/execute.svtest | 23 + tests/extensions/mime/execute/foreverypart.sieve | 7 + 24 files changed, 2226 insertions(+), 3 deletions(-) diffs (truncated from 2381 to 300 lines): diff -r 7c516d25ad50 -r feca1b976393 Makefile.am --- a/Makefile.am Sun Nov 29 11:50:44 2015 +0100 +++ b/Makefile.am Sun Nov 29 11:50:59 2015 +0100 @@ -166,6 +166,8 @@ tests/extensions/duplicate/execute-vnd.svtest \ tests/extensions/metadata/execute.svtest \ tests/extensions/metadata/errors.svtest \ + tests/extensions/mime/errors.svtest \ + tests/extensions/mime/content-header.svtest \ tests/extensions/vnd.dovecot/debug/execute.svtest \ tests/extensions/vnd.dovecot/environment/basic.svtest \ tests/extensions/vnd.dovecot/environment/variables.svtest \ diff -r 7c516d25ad50 -r feca1b976393 README --- a/README Sun Nov 29 11:50:44 2015 +0100 +++ b/README Sun Nov 29 11:50:59 2015 +0100 @@ -120,6 +120,8 @@ mailbox (RFC 5490; Section 3): fully supported (v0.1.10+), but ACL permissions are not verified for mailboxexists. mboxmetadata and servermetadata (RFC 5490): fully supported (v0.4.7+) + foreverypart (RFC 5703; Section 3): fully supported (v0.4.10+). + mime (RFC 5703; Section 4): fully supported (v0.4.10+). include (RFC 6609): fully supported (v0.4.0+) duplicate (RFC 7352): fully supported (v0.4.3+). regex (draft v08; not latest version): almost fully supported, but @@ -154,7 +156,7 @@ useful for Dovecot in particular, but many of them are. Currently, the author has taken notice of the following extensions: - foreverypart, mime, replace, enclose, and extracttext (RFC 5703): planned. + replace, enclose, and extracttext (RFC 5703): planned. imapsieve (RFC 6785): planned. envelope-dsn, envelope-deliverby, redirect-dsn and redirect-deliverby (RFC 6009): planned; depends on lib-smtp changes in diff -r 7c516d25ad50 -r feca1b976393 TODO --- a/TODO Sun Nov 29 11:50:44 2015 +0100 +++ b/TODO Sun Nov 29 11:50:59 2015 +0100 @@ -33,7 +33,7 @@ - Adjust Sieve script API to support asynchronous script retrieval to retrieve scripts in parallel when possible. * Implement message modification and extraction API in order to: - - Implement replace, enclose, foreverypart, mime and extracttext extensions + - Implement replace, enclose, and extracttext extensions * Improve error handling. - Implement dropping errors in the user's mailbox as a mail message. * Finish body extension: diff -r 7c516d25ad50 -r feca1b976393 configure.ac --- a/configure.ac Sun Nov 29 11:50:44 2015 +0100 +++ b/configure.ac Sun Nov 29 11:50:59 2015 +0100 @@ -209,6 +209,7 @@ src/lib-sieve/plugins/metadata/Makefile src/lib-sieve/plugins/duplicate/Makefile src/lib-sieve/plugins/index/Makefile +src/lib-sieve/plugins/mime/Makefile src/lib-sieve/plugins/vnd.dovecot/Makefile src/lib-sieve/plugins/vnd.dovecot/debug/Makefile src/lib-sieve/plugins/vnd.dovecot/environment/Makefile diff -r 7c516d25ad50 -r feca1b976393 src/lib-sieve/Makefile.am --- a/src/lib-sieve/Makefile.am Sun Nov 29 11:50:44 2015 +0100 +++ b/src/lib-sieve/Makefile.am Sun Nov 29 11:50:59 2015 +0100 @@ -77,6 +77,7 @@ $(extdir)/duplicate/libsieve_ext_duplicate.la \ $(extdir)/index/libsieve_ext_index.la \ $(extdir)/metadata/libsieve_ext_metadata.la \ + $(extdir)/mime/libsieve_ext_mime.la \ $(extdir)/vnd.dovecot/debug/libsieve_ext_debug.la \ $(extdir)/vnd.dovecot/environment/libsieve_ext_vnd_environment.la \ $(unfinished_plugins) diff -r 7c516d25ad50 -r feca1b976393 src/lib-sieve/plugins/Makefile.am --- a/src/lib-sieve/plugins/Makefile.am Sun Nov 29 11:50:44 2015 +0100 +++ b/src/lib-sieve/plugins/Makefile.am Sun Nov 29 11:50:59 2015 +0100 @@ -24,6 +24,7 @@ duplicate \ index \ metadata \ + mime \ vnd.dovecot \ $(UNFINISHED) diff -r 7c516d25ad50 -r feca1b976393 src/lib-sieve/plugins/mime/Makefile.am --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/mime/Makefile.am Sun Nov 29 11:50:59 2015 +0100 @@ -0,0 +1,24 @@ +noinst_LTLIBRARIES = libsieve_ext_mime.la + +AM_CPPFLAGS = \ + -I$(srcdir)/../.. \ + -I$(srcdir)/../../util \ + $(LIBDOVECOT_INCLUDE) + +commands = \ + cmd-foreverypart.c \ + cmd-break.c + +tags = \ + tag-mime.c + +libsieve_ext_mime_la_SOURCES = \ + ext-mime.c \ + ext-foreverypart.c \ + ext-mime-common.c \ + $(commands) \ + $(tags) + +noinst_HEADERS = \ + ext-mime-common.h + diff -r 7c516d25ad50 -r feca1b976393 src/lib-sieve/plugins/mime/cmd-break.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/mime/cmd-break.c Sun Nov 29 11:50:59 2015 +0100 @@ -0,0 +1,273 @@ +/* Copyright (c) 2002-2015 Pigeonhole authors, see the included COPYING file + */ + +#include "sieve-common.h" +#include "sieve-code.h" +#include "sieve-extensions.h" +#include "sieve-commands.h" +#include "sieve-validator.h" +#include "sieve-generator.h" +#include "sieve-interpreter.h" +#include "sieve-binary.h" +#include "sieve-dump.h" + +#include "ext-mime-common.h" + +#include + +/* break + * + * Syntax: + * break [":name" ] + * + */ + +static bool cmd_break_registered + (struct sieve_validator *valdtr, const struct sieve_extension *ext, + struct sieve_command_registration *cmd_reg); +static bool cmd_break_pre_validate + (struct sieve_validator *valdtr, struct sieve_command *cmd); +static bool cmd_break_validate + (struct sieve_validator *valdtr, struct sieve_command *cmd); +static bool cmd_break_generate + (const struct sieve_codegen_env *cgenv, + struct sieve_command *ctx); + +const struct sieve_command_def cmd_break = { + "break", + SCT_COMMAND, + 0, 0, FALSE, FALSE, + cmd_break_registered, + cmd_break_pre_validate, + cmd_break_validate, + NULL, + cmd_break_generate, + NULL, +}; + +/* + * Tagged arguments + */ + +/* Forward declarations */ + +static bool cmd_break_validate_name_tag + (struct sieve_validator *valdtr, struct sieve_ast_argument **arg, + struct sieve_command *cmd); + +/* Argument objects */ + +static const struct sieve_argument_def break_name_tag = { + "name", + NULL, + cmd_break_validate_name_tag, + NULL, NULL, NULL +}; + +/* + * Break operation + */ + +static bool cmd_break_operation_dump + (const struct sieve_dumptime_env *denv, sieve_size_t *address); +static int cmd_break_operation_execute + (const struct sieve_runtime_env *renv, sieve_size_t *address); + +const struct sieve_operation_def break_operation = { + "break", + &foreverypart_extension, + EXT_FOREVERYPART_OPERATION_BREAK, + cmd_break_operation_dump, + cmd_break_operation_execute +}; + +/* + * Validation data + */ + +struct cmd_break_data { + struct sieve_ast_argument *name; + struct sieve_command *loop_cmd; +}; + +/* + * Tag validation + */ + +static bool cmd_break_validate_name_tag +(struct sieve_validator *valdtr, struct sieve_ast_argument **arg, + struct sieve_command *cmd) +{ + struct cmd_break_data *data = + (struct cmd_break_data *)cmd->data; + struct sieve_ast_argument *tag = *arg; + + /* Detach the tag itself */ + *arg = sieve_ast_arguments_detach(*arg, 1); + + /* Check syntax: + * :name + */ + if ( !sieve_validate_tag_parameter + (valdtr, cmd, tag, *arg, NULL, 0, SAAT_STRING, TRUE) ) + return FALSE; + data->name = *arg; + + /* Skip parameter */ + *arg = sieve_ast_argument_next(*arg); + return TRUE; +} + +/* + * Command registration + */ + +static bool cmd_break_registered +(struct sieve_validator *valdtr, const struct sieve_extension *ext, + struct sieve_command_registration *cmd_reg) +{ + sieve_validator_register_tag + (valdtr, cmd_reg, ext, &break_name_tag, 0); + + return TRUE; +} + +/* + * Command validation + */ + +static bool cmd_break_pre_validate +(struct sieve_validator *valdtr ATTR_UNUSED, struct sieve_command *cmd) +{ + struct cmd_break_data *data; + pool_t pool = sieve_command_pool(cmd); + + data = p_new(pool, struct cmd_break_data, 1); + cmd->data = data; + return TRUE; +} + +static bool cmd_break_validate +(struct sieve_validator *valdtr, struct sieve_command *cmd) +{ + struct cmd_break_data *data = + (struct cmd_break_data *)cmd->data; + struct sieve_ast_node *node = cmd->ast_node; + const char *name = ( data->name == NULL ? + NULL : sieve_ast_argument_strc(data->name) ); + + i_assert(node != NULL); + while ( node != NULL && node->command != NULL ) { + if ( sieve_command_is(node->command, cmd_foreverypart) ) { + struct ext_foreverypart_loop *loop = + (struct ext_foreverypart_loop *)node->command->data; + if ( name == NULL || + (name != NULL && loop->name != NULL && + strcmp(name, loop->name) == 0) ) { + data->loop_cmd = node->command; + break; + } + } + node = sieve_ast_node_parent(node); + } + + if ( data->loop_cmd == NULL ) { + if ( name == NULL ) { + sieve_command_validate_error(valdtr, cmd, + "the break command is not placed inside " + "a foreverypart loop"); + } else { + sieve_command_validate_error(valdtr, cmd, + "the break command is not placed inside " + "a foreverypart loop named `%s'", + name); + } + return FALSE; + } + + return TRUE; +} + From pigeonhole at rename-it.nl Sun Nov 29 10:53:29 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 11:53:29 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: The t_str_trim() function has... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/6f01853a5b35 changeset: 2135:6f01853a5b35 user: Stephan Bosch date: Sun Nov 29 11:47:47 2015 +0100 description: lib-sieve: The t_str_trim() function has moved to Dovecot. diffstat: src/lib-sieve/sieve-settings.c | 31 ++++--------------------------- 1 files changed, 4 insertions(+), 27 deletions(-) diffs (69 lines): diff -r d23225e408af -r 6f01853a5b35 src/lib-sieve/sieve-settings.c --- a/src/lib-sieve/sieve-settings.c Wed Nov 18 22:11:31 2015 +0100 +++ b/src/lib-sieve/sieve-settings.c Sun Nov 29 11:47:47 2015 +0100 @@ -10,29 +10,6 @@ #include -// FIXME: add to dovecot -static const char *t_str_trim(const char *str) -{ - const char *p, *pend, *begin; - - p = str; - pend = str + strlen(str); - if (p == pend) - return ""; - - while (p < pend && (*p == ' ' || *p == '\t')) - p++; - begin = p; - - p = pend - 1; - while (p > begin && (*p == ' ' || *p == '\t')) - p--; - - if (p <= begin) - return ""; - return t_strdup_until(begin, p+1); -} - /* * Access to settings */ @@ -143,7 +120,7 @@ if ( str_value == NULL ) return FALSE; - str_value = t_str_trim(str_value); + str_value = t_str_trim(str_value, "\t "); if ( *str_value == '\0' ) return FALSE; @@ -175,7 +152,7 @@ if ( str_value == NULL ) return FALSE; - str_value = t_str_trim(str_value); + str_value = t_str_trim(str_value, "\t "); if ( *str_value == '\0' ) return FALSE; @@ -228,7 +205,7 @@ if ( str_value == NULL ) return FALSE; - str_value = t_str_trim(str_value); + str_value = t_str_trim(str_value, "\t "); str_value = t_str_lcase(str_value); set_len = strlen(str_value); if ( set_len > 0 ) { @@ -245,7 +222,7 @@ } else if ( str_value[0] == '<' && str_value[set_len-1] == '>') { sender->source = SIEVE_MAIL_SENDER_SOURCE_EXPLICIT; - str_value = t_str_trim(t_strndup(str_value+1, set_len-2)); + str_value = t_str_trim(t_strndup(str_value+1, set_len-2), "\t "); sender->address = NULL; if ( *str_value != '\0' ) sender->address = p_strdup(pool, str_value); From pigeonhole at rename-it.nl Sun Nov 29 10:53:29 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 11:53:29 +0100 Subject: dovecot-2.2-pigeonhole: Merged concurrent changes. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/90de55c880e5 changeset: 2141:90de55c880e5 user: Stephan Bosch date: Sun Nov 29 11:53:24 2015 +0100 description: Merged concurrent changes. diffstat: TODO | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r feca1b976393 -r 90de55c880e5 TODO --- a/TODO Sun Nov 29 11:50:59 2015 +0100 +++ b/TODO Sun Nov 29 11:53:24 2015 +0100 @@ -1,6 +1,7 @@ Current activities: * Rework string matching: + - Give Sieve its own runtime string type, rather than (ab)using string_t. - Add support for stream matching for handling large values, e.g. from the body extension. - Improve efficiency of :matches and :contains match types. From pigeonhole at rename-it.nl Sun Nov 29 10:53:29 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 11:53:29 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Reworked message body parsing... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/0bc3cb5c4c0a changeset: 2136:0bc3cb5c4c0a user: Stephan Bosch date: Sun Nov 29 11:48:21 2015 +0100 description: lib-sieve: Reworked message body parsing to store the message parts in a tree structure. diffstat: src/lib-sieve/plugins/body/ext-body-common.c | 6 +- src/lib-sieve/sieve-message.c | 186 +++++++++++++------------- src/lib-sieve/sieve-message.h | 8 +- tests/extensions/body/content.svtest | 5 +- 4 files changed, 105 insertions(+), 100 deletions(-) diffs (truncated from 470 to 300 lines): diff -r 6f01853a5b35 -r 0bc3cb5c4c0a src/lib-sieve/plugins/body/ext-body-common.c --- a/src/lib-sieve/plugins/body/ext-body-common.c Sun Nov 29 11:47:47 2015 +0100 +++ b/src/lib-sieve/plugins/body/ext-body-common.c Sun Nov 29 11:48:21 2015 +0100 @@ -30,8 +30,8 @@ struct ext_body_stringlist { struct sieve_stringlist strlist; - struct sieve_message_body_part *body_parts; - struct sieve_message_body_part *body_parts_iter; + struct sieve_message_part_data *body_parts; + struct sieve_message_part_data *body_parts_iter; }; int ext_body_get_part_list @@ -40,7 +40,7 @@ { static const char * const _no_content_types[] = { "", NULL }; struct ext_body_stringlist *strlist; - struct sieve_message_body_part *body_parts = NULL; + struct sieve_message_part_data *body_parts = NULL; int ret; *strlist_r = NULL; diff -r 6f01853a5b35 -r 0bc3cb5c4c0a src/lib-sieve/sieve-message.c --- a/src/lib-sieve/sieve-message.c Sun Nov 29 11:47:47 2015 +0100 +++ b/src/lib-sieve/sieve-message.c Sun Nov 29 11:48:21 2015 +0100 @@ -59,7 +59,9 @@ struct edit_mail *edit_mail; }; -struct sieve_message_body_part_cached { +struct sieve_message_part { + struct sieve_message_part *parent, *next, *children; + const char *content_type; const char *content_disposition; @@ -68,7 +70,8 @@ size_t decoded_body_size; size_t text_body_size; - bool have_body; /* there's the empty end-of-headers line */ + unsigned int have_body:1; /* there's the empty end-of-headers line */ + unsigned int epilogue:1; /* this is a multipart epilogue */ }; struct sieve_message_context { @@ -101,8 +104,8 @@ /* Body */ - ARRAY(struct sieve_message_body_part_cached) cached_body_parts; - ARRAY(struct sieve_message_body_part) return_body_parts; + ARRAY(struct sieve_message_part *) cached_body_parts; + ARRAY(struct sieve_message_part_data) return_body_parts; buffer_t *raw_body; unsigned int edit_snapshot:1; @@ -842,33 +845,15 @@ return FALSE; } -static bool _want_multipart_content_type -(const char * const *wanted_types) -{ - for (; *wanted_types != NULL; wanted_types++) { - if (**wanted_types == '\0') { - /* empty string matches everything */ - return TRUE; - } - - /* match only main type */ - if ( strncasecmp(*wanted_types, "multipart", 9) == 0 && - ( strlen(*wanted_types) == 9 || *(*wanted_types+9) == '/' ) ) - return TRUE; - } - - return FALSE; -} - static bool sieve_message_body_get_return_parts (const struct sieve_runtime_env *renv, const char * const *wanted_types, bool extract_text) { struct sieve_message_context *msgctx = renv->msgctx; - const struct sieve_message_body_part_cached *body_parts; + struct sieve_message_part *const *body_parts; unsigned int i, count; - struct sieve_message_body_part *return_part; + struct sieve_message_part_data *return_part; /* Check whether any body parts are cached already */ body_parts = array_get(&msgctx->cached_body_parts, &count); @@ -880,7 +865,7 @@ /* Fill result array with requested content_types */ for (i = 0; i < count; i++) { - if (!body_parts[i].have_body) { + if (!body_parts[i]->have_body) { /* Part has no body; according to RFC this MUST not match to anything and * therefore it is not included in the result. */ @@ -888,37 +873,38 @@ } /* Skip content types that are not requested */ - if (!_is_wanted_content_type(wanted_types, body_parts[i].content_type)) + if (!_is_wanted_content_type + (wanted_types, body_parts[i]->content_type)) continue; /* Add new item to the result */ return_part = array_append_space(&msgctx->return_body_parts); - return_part->content_type = body_parts[i].content_type; - return_part->content_disposition = body_parts[i].content_disposition; + return_part->content_type = body_parts[i]->content_type; + return_part->content_disposition = body_parts[i]->content_disposition; /* Depending on whether a decoded body part is requested, the appropriate * cache item is read. If it is missing, this function fails and the cache - * needs to be completed by sieve_message_body_parts_add_missing(). + * needs to be completed by sieve_message_parts_add_missing(). */ if (extract_text) { - if (body_parts[i].text_body == NULL) + if (body_parts[i]->text_body == NULL) return FALSE; - return_part->content = body_parts[i].text_body; - return_part->size = body_parts[i].text_body_size; + return_part->content = body_parts[i]->text_body; + return_part->size = body_parts[i]->text_body_size; } else { - if (body_parts[i].decoded_body == NULL) + if (body_parts[i]->decoded_body == NULL) return FALSE; - return_part->content = body_parts[i].decoded_body; - return_part->size = body_parts[i].decoded_body_size; + return_part->content = body_parts[i]->decoded_body; + return_part->size = body_parts[i]->decoded_body_size; } } return TRUE; } -static void sieve_message_body_part_save +static void sieve_message_part_save (const struct sieve_runtime_env *renv, buffer_t *buf, - struct sieve_message_body_part_cached *body_part, + struct sieve_message_part *body_part, bool extract_text) { struct sieve_message_context *msgctx = renv->msgctx; @@ -1017,26 +1003,28 @@ return str_c(content_disp); } -/* sieve_message_body_parts_add_missing(): +/* sieve_message_parts_add_missing(): * Add requested message body parts to the cache that are missing. */ -static int sieve_message_body_parts_add_missing +static int sieve_message_parts_add_missing (const struct sieve_runtime_env *renv, - const char *const *content_types, bool extract_text) + const char *const *content_types, + bool extract_text) { struct sieve_message_context *msgctx = renv->msgctx; pool_t pool = msgctx->context_pool; struct mail *mail = sieve_message_get_mail(renv->msgctx); - struct sieve_message_body_part_cached *body_part = NULL, *header_part = NULL; + enum message_parser_flags mparser_flags = + MESSAGE_PARSER_FLAG_INCLUDE_MULTIPART_BLOCKS; + struct sieve_message_part *body_part, *header_part, *last_part; struct message_parser_ctx *parser; struct message_decoder_context *decoder; struct message_block block, decoded; - struct message_part *parts, *prev_part = NULL; - ARRAY(struct message_part *) part_index; + struct message_part *mparts, *prev_mpart = NULL; buffer_t *buf; struct istream *input; unsigned int idx = 0; - bool save_body = FALSE, want_multipart, have_all; + bool save_body = FALSE, have_all; int ret; /* First check whether any are missing */ @@ -1051,71 +1039,91 @@ return sieve_runtime_mail_error(renv, mail, "failed to open input message"); } - if (mail_get_parts(mail, &parts) < 0) { + if (mail_get_parts(mail, &mparts) < 0) { return sieve_runtime_mail_error(renv, mail, "failed to parse input message parts"); } - if ( (want_multipart=_want_multipart_content_type(content_types)) ) { - t_array_init(&part_index, 8); - } - buf = buffer_create_dynamic(default_pool, 4096); + body_part = header_part = last_part = NULL; /* Initialize body decoder */ decoder = message_decoder_init(NULL, 0); - //parser = message_parser_init_from_parts(parts, input, 0, - //MESSAGE_PARSER_FLAG_INCLUDE_MULTIPART_BLOCKS); - parser = message_parser_init(pool_datastack_create(), input, 0, - MESSAGE_PARSER_FLAG_INCLUDE_MULTIPART_BLOCKS); + // FIXME: currently not tested with edit-mail. + //parser = message_parser_init_from_parts(parts, input, 0, + //mparser_flags); + parser = message_parser_init + (pool_datastack_create(), input, 0, mparser_flags); while ( (ret=message_parser_parse_next_block (parser, &block)) > 0 ) { + struct sieve_message_part **body_part_idx; - if ( block.part != prev_part ) { + if ( block.part != prev_mpart ) { bool message_rfc822 = FALSE; /* Save previous body part */ if ( body_part != NULL ) { /* Treat message/rfc822 separately; headers become content */ - if ( block.part->parent == prev_part && + if ( block.part->parent == prev_mpart && strcmp(body_part->content_type, "message/rfc822") == 0 ) { message_rfc822 = TRUE; } else { if ( save_body ) { - sieve_message_body_part_save + sieve_message_part_save (renv, buf, body_part, extract_text); } } } - /* Start processing next */ - body_part = array_idx_modifiable + /* Start processing next part */ + body_part_idx = array_idx_modifiable (&msgctx->cached_body_parts, idx); + if (*body_part_idx == NULL) + *body_part_idx = p_new(pool, struct sieve_message_part, 1); + body_part = *body_part_idx; body_part->content_type = "text/plain"; - /* Check whether this is the epilogue block of a wanted multipart part */ - if ( want_multipart ) { - array_idx_set(&part_index, idx, &block.part); + /* Copy tree structure */ + if ( block.part->context != NULL ) { + struct sieve_message_part *epipart = + (struct sieve_message_part *)block.part->context; + i_assert(epipart != NULL); - if ( prev_part != NULL && prev_part->next != block.part && - block.part->parent != prev_part ) { - struct message_part *const *iparts; - unsigned int count, i; + /* multipart epilogue */ + body_part->content_type = epipart->content_type; + body_part->have_body = TRUE; + body_part->epilogue = TRUE; + save_body = _is_wanted_content_type + (content_types, body_part->content_type); - iparts = array_get(&part_index, &count); - for ( i = 0; i < count; i++ ) { - if ( iparts[i] == block.part ) { - const struct sieve_message_body_part_cached *parent = - array_idx(&msgctx->cached_body_parts, i); - body_part->content_type = parent->content_type; - body_part->have_body = TRUE; - save_body = _is_wanted_content_type - (content_types, body_part->content_type); - break; - } + } else { + struct sieve_message_part *parent = NULL; + + if ( block.part->parent != NULL ) { + body_part->parent = parent = + (struct sieve_message_part *) + block.part->parent->context; + } + + /* new part */ + block.part->context = (void*)body_part; + + if ( last_part != NULL ) { + i_assert( parent != NULL ); + if ( last_part->parent == parent ) { + last_part->next = body_part; From pigeonhole at rename-it.nl Sun Nov 29 10:53:29 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 11:53:29 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Created message body part ite... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/be0971931cfe changeset: 2138:be0971931cfe user: Stephan Bosch date: Sun Nov 29 11:49:47 2015 +0100 description: lib-sieve: Created message body part iterator and mime header list. diffstat: src/lib-sieve/sieve-message.c | 441 ++++++++++++++++++++++++++++++++++++++--- src/lib-sieve/sieve-message.h | 30 ++ 2 files changed, 438 insertions(+), 33 deletions(-) diffs (truncated from 661 to 300 lines): diff -r c107764e6008 -r be0971931cfe src/lib-sieve/sieve-message.c --- a/src/lib-sieve/sieve-message.c Sun Nov 29 11:48:29 2015 +0100 +++ b/src/lib-sieve/sieve-message.c Sun Nov 29 11:49:47 2015 +0100 @@ -12,6 +12,7 @@ #include "message-date.h" #include "message-parser.h" #include "message-decoder.h" +#include "message-header-decode.h" #include "mail-html2text.h" #include "mail-storage.h" #include "mail-user.h" @@ -52,16 +53,18 @@ * Message context */ -struct sieve_message_version { - struct mail *mail; - struct mailbox *box; - struct mailbox_transaction_context *trans; - struct edit_mail *edit_mail; +struct sieve_message_header { + const char *name; + + const unsigned char *value, *utf8_value; + size_t value_len, utf8_value_len; }; struct sieve_message_part { struct sieve_message_part *parent, *next, *children; + ARRAY(struct sieve_message_header) headers; + const char *content_type; const char *content_disposition; @@ -74,6 +77,13 @@ unsigned int epilogue:1; /* this is a multipart epilogue */ }; +struct sieve_message_version { + struct mail *mail; + struct mailbox *box; + struct mailbox_transaction_context *trans; + struct edit_mail *edit_mail; +}; + struct sieve_message_context { pool_t pool; pool_t context_pool; @@ -841,12 +851,28 @@ * Message body */ +static void str_replace_nuls(string_t *str) +{ + char *data = str_c_modifiable(str); + unsigned int i, len = str_len(str); + + for (i = 0; i < len; i++) { + if (data[i] == '\0') + data[i] = ' '; + } +} + static bool _is_wanted_content_type (const char * const *wanted_types, const char *content_type) +ATTR_NULL(1) { - const char *subtype = strchr(content_type, '/'); + const char *subtype; size_t type_len; + if ( wanted_types == NULL ) + return TRUE; + + subtype = strchr(content_type, '/'); type_len = ( subtype == NULL ? strlen(content_type) : (size_t)(subtype - content_type) ); @@ -1039,13 +1065,15 @@ static int sieve_message_parts_add_missing (const struct sieve_runtime_env *renv, const char *const *content_types, - bool extract_text) + bool extract_text, bool iter_all) + ATTR_NULL(2) { struct sieve_message_context *msgctx = renv->msgctx; pool_t pool = msgctx->context_pool; struct mail *mail = sieve_message_get_mail(renv->msgctx); enum message_parser_flags mparser_flags = MESSAGE_PARSER_FLAG_INCLUDE_MULTIPART_BLOCKS; + ARRAY(struct sieve_message_header) headers; struct sieve_message_part *body_part, *header_part, *last_part; struct message_parser_ctx *parser; struct message_decoder_context *decoder; @@ -1055,11 +1083,12 @@ struct istream *input; unsigned int idx = 0; bool save_body = FALSE, have_all; + string_t *hdr_content = NULL; int ret; /* First check whether any are missing */ - if (sieve_message_body_get_return_parts - (renv, content_types, extract_text)) { + if ( !iter_all && sieve_message_body_get_return_parts + (renv, content_types, extract_text) ) { /* Cache hit; all are present */ return SIEVE_EXEC_OK; } @@ -1077,6 +1106,13 @@ buf = buffer_create_dynamic(default_pool, 4096); body_part = header_part = last_part = NULL; + if (iter_all) { + t_array_init(&headers, 64); + hdr_content = t_str_new(512); + } else { + memset(&headers, 0, sizeof(headers)); + } + /* Initialize body decoder */ decoder = message_decoder_init(NULL, 0); @@ -1088,6 +1124,9 @@ while ( (ret=message_parser_parse_next_block (parser, &block)) > 0 ) { struct sieve_message_part **body_part_idx; + struct message_header_line *hdr = block.hdr; + struct sieve_message_header *header; + unsigned char *data; if ( block.part != prev_mpart ) { bool message_rfc822 = FALSE; @@ -1104,15 +1143,23 @@ (renv, buf, body_part, extract_text); } } + if ( iter_all && !array_is_created(&body_part->headers) && + array_count(&headers) > 0 ) { + p_array_init(&body_part->headers, pool, array_count(&headers)); + array_copy(&body_part->headers.arr, 0, + &headers.arr, 0, array_count(&headers)); + } } /* Start processing next part */ body_part_idx = array_idx_modifiable (&msgctx->cached_body_parts, idx); - if (*body_part_idx == NULL) + if ( *body_part_idx == NULL ) *body_part_idx = p_new(pool, struct sieve_message_part, 1); body_part = *body_part_idx; body_part->content_type = "text/plain"; + if ( iter_all ) + array_clear(&headers); /* Copy tree structure */ if ( block.part->context != NULL ) { @@ -1124,7 +1171,7 @@ body_part->content_type = epipart->content_type; body_part->have_body = TRUE; body_part->epilogue = TRUE; - save_body = _is_wanted_content_type + save_body = iter_all || _is_wanted_content_type (content_types, body_part->content_type); } else { @@ -1171,8 +1218,12 @@ idx++; } - if ( block.hdr != NULL || block.size == 0 ) { - bool is_ctype = FALSE; + if ( hdr != NULL || block.size == 0 ) { + enum { + _HDR_CONTENT_TYPE, + _HDR_CONTENT_DISPOSITION, + _HDR_OTHER + } hdr_field; /* Reading headers */ @@ -1181,7 +1232,7 @@ (decoder, &block, &decoded); /* Check for end of headers */ - if ( block.hdr == NULL ) { + if ( hdr == NULL ) { /* Save headers for message/rfc822 part */ if ( header_part != NULL ) { sieve_message_part_save @@ -1191,7 +1242,7 @@ /* Save bodies only if we have a wanted content-type */ i_assert( body_part != NULL ); - save_body = _is_wanted_content_type + save_body = iter_all || _is_wanted_content_type (content_types, body_part->content_type); continue; } @@ -1199,47 +1250,101 @@ /* Encountered the empty line that indicates the end of the headers and * the start of the body */ - if ( block.hdr->eoh ) { + if ( hdr->eoh ) { i_assert( body_part != NULL ); body_part->have_body = TRUE; + continue; } else if ( header_part != NULL ) { /* Save message/rfc822 header as part content */ - if ( block.hdr->continued ) { - buffer_append(buf, block.hdr->value, block.hdr->value_len); + if ( hdr->continued ) { + buffer_append(buf, hdr->value, hdr->value_len); } else { - buffer_append(buf, block.hdr->name, block.hdr->name_len); - buffer_append(buf, block.hdr->middle, block.hdr->middle_len); - buffer_append(buf, block.hdr->value, block.hdr->value_len); + buffer_append(buf, hdr->name, hdr->name_len); + buffer_append(buf, hdr->middle, hdr->middle_len); + buffer_append(buf, hdr->value, hdr->value_len); } - if ( !block.hdr->no_newline ) { + if ( !hdr->no_newline ) { buffer_append(buf, "\r\n", 2); } } - /* We're interested in only the Content-Type: header */ - if ( strcasecmp(block.hdr->name, "Content-Type" ) == 0 ) - is_ctype = TRUE; - else if ( strcasecmp(block.hdr->name, "Content-Disposition" ) != 0 ) + if ( strcasecmp(hdr->name, "Content-Type" ) == 0 ) + hdr_field = _HDR_CONTENT_TYPE; + else if ( strcasecmp(hdr->name, "Content-Disposition" ) != 0 ) + hdr_field = _HDR_CONTENT_DISPOSITION; + else if ( iter_all && !array_is_created(&body_part->headers) ) + hdr_field = _HDR_OTHER; + else { + /* Not interested in this header */ continue; + } /* Header can have folding whitespace. Acquire the full value before * continuing */ - if ( block.hdr->continues ) { - block.hdr->use_full_value = TRUE; + if ( hdr->continues ) { + hdr->use_full_value = TRUE; continue; } + if ( iter_all && !array_is_created(&body_part->headers) ) { + /* Add header */ + header = array_append_space(&headers); + header->name = p_strdup(pool, hdr->name); + + // FIXME: trim header values + + /* Decode MIME encoded-words. */ + str_truncate(hdr_content, 0); + message_header_decode_utf8 + (hdr->full_value, hdr->full_value_len, hdr_content, NULL); + if ( hdr->full_value_len != str_len(hdr_content) || + strncmp(str_c(hdr_content), (const char *)hdr->full_value, + hdr->full_value_len) != 0 ) { + if ( strlen(str_c(hdr_content)) != str_len(hdr_content) ) { + /* replace NULs with spaces */ + str_replace_nuls(hdr_content); + } + /* store raw */ + data = p_malloc(pool, hdr->full_value_len + 1); + data[hdr->full_value_len] = '\0'; + header->value = memcpy(data, + hdr->full_value, hdr->full_value_len); + header->value_len = hdr->full_value_len; + /* store decoded */ + data = p_malloc(pool, str_len(hdr_content) + 1); + data[str_len(hdr_content)] = '\0'; + header->utf8_value = memcpy(data, + str_data(hdr_content), str_len(hdr_content)); + header->utf8_value_len = str_len(hdr_content); + } else { + /* raw == decoded */ + data = p_malloc(pool, hdr->full_value_len + 1); + data[hdr->full_value_len] = '\0'; + header->value = header->utf8_value = + memcpy(data, hdr->full_value, hdr->full_value_len); + header->value_len = header->utf8_value_len = + hdr->full_value_len; + } + + if ( hdr_field == _HDR_OTHER ) + continue; + } + From pigeonhole at rename-it.nl Sun Nov 29 10:53:29 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 11:53:29 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Implemented running code loop... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/7c516d25ad50 changeset: 2139:7c516d25ad50 user: Stephan Bosch date: Sun Nov 29 11:50:44 2015 +0100 description: lib-sieve: Implemented running code loops in the interpreter. diffstat: src/lib-sieve/sieve-interpreter.c | 192 +++++++++++++++++++++++++++++++++++++- src/lib-sieve/sieve-interpreter.h | 32 ++++++ 2 files changed, 221 insertions(+), 3 deletions(-) diffs (270 lines): diff -r be0971931cfe -r 7c516d25ad50 src/lib-sieve/sieve-interpreter.c --- a/src/lib-sieve/sieve-interpreter.c Sun Nov 29 11:49:47 2015 +0100 +++ b/src/lib-sieve/sieve-interpreter.c Sun Nov 29 11:50:44 2015 +0100 @@ -38,6 +38,18 @@ }; /* + * Code loop + */ + +struct sieve_interpreter_loop { + unsigned int level; + sieve_size_t begin, end; + const struct sieve_extension_def *ext_def; + pool_t pool; + void *context; +}; + +/* * Interpreter */ @@ -55,6 +67,10 @@ bool interrupted; /* Interpreter interrupt requested */ bool test_result; /* Result of previous test command */ + /* Loop stack */ + ARRAY(struct sieve_interpreter_loop) loop_stack; + sieve_size_t loop_limit; + /* Runtime environment */ struct sieve_runtime_env runenv; struct sieve_runtime_trace trace; @@ -225,14 +241,21 @@ struct sieve_interpreter *interp = *_interp; struct sieve_runtime_env *renv = &interp->runenv; const struct sieve_interpreter_extension_reg *eregs; - unsigned int ext_count, i; + struct sieve_interpreter_loop *loops; + unsigned int count, i; + + if ( array_is_created(&interp->loop_stack) ) { + loops = array_get_modifiable(&interp->loop_stack, &count); + for ( i = 0; i < count; i++ ) + pool_unref(&loops[i].pool); + } interp->trace.indent = 0; sieve_runtime_trace_end(renv); /* Signal registered extensions that the interpreter is being destroyed */ - eregs = array_get(&interp->extensions, &ext_count); - for ( i = 0; i < ext_count; i++ ) { + eregs = array_get(&interp->extensions, &count); + for ( i = 0; i < count; i++ ) { if ( eregs[i].intext != NULL && eregs[i].intext->free != NULL ) eregs[i].intext->free(eregs[i].ext, interp, eregs[i].context); } @@ -456,6 +479,169 @@ } /* + * Loop handling + */ + +struct sieve_interpreter_loop *sieve_interpreter_loop_start +(struct sieve_interpreter *interp, sieve_size_t loop_end, + const struct sieve_extension_def *ext_def) +{ + const struct sieve_runtime_env *renv = &interp->runenv; + struct sieve_interpreter_loop *loop; + + i_assert( loop_end > interp->runenv.pc ); + + if ( loop_end > sieve_binary_block_get_size(renv->sblock) ) { + sieve_runtime_trace_error(renv, + "loop end offset out of range"); + return NULL; + } + + if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { + unsigned int line = + sieve_runtime_get_source_location(renv, loop_end); + + if ( sieve_runtime_trace_hasflag(renv, SIEVE_TRFLG_ADDRESSES) ) { + sieve_runtime_trace(renv, 0, "loop ends at line %d [%08llx]", + line, (long long unsigned int) loop_end); + } else { + sieve_runtime_trace(renv, 0, "loop ends at line %d", line); + } + } + + if ( !array_is_created(&interp->loop_stack) ) + p_array_init(&interp->loop_stack, interp->pool, 8); + + loop = array_append_space(&interp->loop_stack); + loop->level = array_count(&interp->loop_stack)-1; + loop->ext_def = ext_def; + loop->begin = interp->runenv.pc; + loop->end = loop_end; + loop->pool = pool_alloconly_create("sieve_interpreter", 128); + + return loop; +} + +struct sieve_interpreter_loop *sieve_interpreter_loop_get +(struct sieve_interpreter *interp, sieve_size_t loop_end, + const struct sieve_extension_def *ext_def) +{ + struct sieve_interpreter_loop *loops; + unsigned int count, i; + + if ( !array_is_created(&interp->loop_stack) ) + return NULL; + + loops = array_get_modifiable(&interp->loop_stack, &count); + for ( i = count; i > 0; i-- ) { + /* We're really making sure our loop matches */ + if ( loops[i-1].end == loop_end && + loops[i-1].ext_def == ext_def ) + return &loops[i-1]; + } + return NULL; +} + +void sieve_interpreter_loop_next(struct sieve_interpreter *interp, + struct sieve_interpreter_loop *loop, + sieve_size_t loop_begin) +{ + const struct sieve_runtime_env *renv = &interp->runenv; + struct sieve_interpreter_loop *loops; + unsigned int count; + + if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { + unsigned int line = + sieve_runtime_get_source_location(renv, loop_begin); + + if ( sieve_runtime_trace_hasflag(renv, SIEVE_TRFLG_ADDRESSES) ) { + sieve_runtime_trace(renv, 0, "looping back to line %d [%08llx]", + line, (long long unsigned int) loop_begin); + } else { + sieve_runtime_trace(renv, 0, "looping back to line %d", line); + } + } + + i_assert( loop->begin == loop_begin ); + + i_assert( array_is_created(&interp->loop_stack) ); + loops = array_get_modifiable(&interp->loop_stack, &count); + i_assert( &loops[count-1] == loop ); + + interp->runenv.pc = loop_begin; +} + +void sieve_interpreter_loop_break(struct sieve_interpreter *interp, + struct sieve_interpreter_loop *loop) +{ + const struct sieve_runtime_env *renv = &interp->runenv; + struct sieve_interpreter_loop *loops; + sieve_size_t loop_end = loop->end; + unsigned int count, i; + + i_assert( array_is_created(&interp->loop_stack) ); + loops = array_get_modifiable(&interp->loop_stack, &count); + for ( i = count; i > 0 && &loops[i-1] != loop; i-- ) + pool_unref(&loops[i-1].pool); + i_assert( i > 0 && &loops[i-1] == loop ); + + array_delete(&interp->loop_stack, i-1, count - (i-1)); + + if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { + unsigned int jmp_line = + sieve_runtime_get_source_location(renv, loop_end); + + if ( sieve_runtime_trace_hasflag(renv, SIEVE_TRFLG_ADDRESSES) ) { + sieve_runtime_trace(renv, 0, "ending loop at line %d [%08llx]", + jmp_line, (long long unsigned int) loop_end); + } else { + sieve_runtime_trace(renv, 0, "ending loop at line %d", jmp_line); + } + } + + interp->runenv.pc = loop->end; +} + +struct sieve_interpreter_loop *sieve_interpreter_loop_get_surrounding +(struct sieve_interpreter *interp, + struct sieve_interpreter_loop *loop, + const struct sieve_extension_def *ext_def) +{ + struct sieve_interpreter_loop *loops; + unsigned int count, i; + + if ( !array_is_created(&interp->loop_stack) ) + return NULL; + + loops = array_get_modifiable(&interp->loop_stack, &count); + i_assert(loop == NULL || loop->level < count); + + for ( i = (loop == NULL ? count : loop->level); i > 0; i-- ) { + if ( ext_def == NULL || loops[i-1].ext_def == ext_def ) + return &loops[i-1]; + } + return NULL; +} + +pool_t sieve_interpreter_loop_get_pool +(struct sieve_interpreter_loop *loop) +{ + return loop->pool; +} + +void *sieve_interpreter_loop_get_context +(struct sieve_interpreter_loop *loop) +{ + return loop->context; +} + +void sieve_interpreter_loop_set_context +(struct sieve_interpreter_loop *loop, void *context) +{ + loop->context = context; +} + +/* * Program flow */ diff -r be0971931cfe -r 7c516d25ad50 src/lib-sieve/sieve-interpreter.h --- a/src/lib-sieve/sieve-interpreter.h Sun Nov 29 11:49:47 2015 +0100 +++ b/src/lib-sieve/sieve-interpreter.h Sun Nov 29 11:50:44 2015 +0100 @@ -46,6 +46,38 @@ (struct sieve_interpreter *interp, struct sieve_result *result); /* + * Loop handling + */ + +struct sieve_interpreter_loop; + +struct sieve_interpreter_loop *sieve_interpreter_loop_start + (struct sieve_interpreter *interp, sieve_size_t loop_end, + const struct sieve_extension_def *ext_def); +struct sieve_interpreter_loop *sieve_interpreter_loop_get + (struct sieve_interpreter *interp, sieve_size_t loop_end, + const struct sieve_extension_def *ext_def); +void sieve_interpreter_loop_next + (struct sieve_interpreter *interp, + struct sieve_interpreter_loop *loop, + sieve_size_t loop_begin); +void sieve_interpreter_loop_break + (struct sieve_interpreter *interp, + struct sieve_interpreter_loop *loop); + +struct sieve_interpreter_loop *sieve_interpreter_loop_get_surrounding +(struct sieve_interpreter *interp, + struct sieve_interpreter_loop *loop, + const struct sieve_extension_def *ext_def) ATTR_NULL(2, 3); + +pool_t sieve_interpreter_loop_get_pool + (struct sieve_interpreter_loop *loop) ATTR_PURE; +void *sieve_interpreter_loop_get_context + (struct sieve_interpreter_loop *loop) ATTR_PURE; +void sieve_interpreter_loop_set_context + (struct sieve_interpreter_loop *loop, void *context); + +/* * Program flow */ From pigeonhole at rename-it.nl Sun Nov 29 10:53:29 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 11:53:29 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Created special stringlist cl... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/c107764e6008 changeset: 2137:c107764e6008 user: Stephan Bosch date: Sun Nov 29 11:48:29 2015 +0100 description: lib-sieve: Created special stringlist class for message headers, so that header names can also be returned by the iterator. diffstat: src/lib-sieve/sieve-message.c | 128 +++++++++++++++++++++++++---------------- src/lib-sieve/sieve-message.h | 39 ++++++++++++- 2 files changed, 116 insertions(+), 51 deletions(-) diffs (275 lines): diff -r 0bc3cb5c4c0a -r c107764e6008 src/lib-sieve/sieve-message.c --- a/src/lib-sieve/sieve-message.c Sun Nov 29 11:48:21 2015 +0100 +++ b/src/lib-sieve/sieve-message.c Sun Nov 29 11:48:29 2015 +0100 @@ -491,44 +491,50 @@ } /* - * Header stringlist + * Message header list */ /* Forward declarations */ -static int sieve_message_header_stringlist_next_item - (struct sieve_stringlist *_strlist, string_t **str_r); -static void sieve_message_header_stringlist_reset +static int sieve_message_header_list_next_item + (struct sieve_header_list *_hdrlist, const char **name_r, + string_t **value_r); +static int sieve_message_header_list_next_value + (struct sieve_stringlist *_strlist, string_t **value_r); +static void sieve_message_header_list_reset (struct sieve_stringlist *_strlist); /* String list object */ -struct sieve_message_header_stringlist { - struct sieve_stringlist strlist; +struct sieve_message_header_list { + struct sieve_header_list hdrlist; struct sieve_stringlist *field_names; + const char *header_name; const char *const *headers; int headers_index; unsigned int mime_decode:1; }; -struct sieve_stringlist *sieve_message_header_stringlist_create -(const struct sieve_runtime_env *renv, struct sieve_stringlist *field_names, +struct sieve_header_list *sieve_message_header_list_create +(const struct sieve_runtime_env *renv, + struct sieve_stringlist *field_names, bool mime_decode) { - struct sieve_message_header_stringlist *strlist; + struct sieve_message_header_list *hdrlist; - strlist = t_new(struct sieve_message_header_stringlist, 1); - strlist->strlist.runenv = renv; - strlist->strlist.exec_status = SIEVE_EXEC_OK; - strlist->strlist.next_item = sieve_message_header_stringlist_next_item; - strlist->strlist.reset = sieve_message_header_stringlist_reset; - strlist->field_names = field_names; - strlist->mime_decode = mime_decode; + hdrlist = t_new(struct sieve_message_header_list, 1); + hdrlist->hdrlist.strlist.runenv = renv; + hdrlist->hdrlist.strlist.exec_status = SIEVE_EXEC_OK; + hdrlist->hdrlist.strlist.next_item = sieve_message_header_list_next_value; + hdrlist->hdrlist.strlist.reset = sieve_message_header_list_reset; + hdrlist->hdrlist.next_item = sieve_message_header_list_next_item; + hdrlist->field_names = field_names; + hdrlist->mime_decode = mime_decode; - return &strlist->strlist; + return &hdrlist->hdrlist; } static inline string_t *_header_right_trim(const char *raw) @@ -547,73 +553,93 @@ /* String list implementation */ -static int sieve_message_header_stringlist_next_item -(struct sieve_stringlist *_strlist, string_t **str_r) +static int sieve_message_header_list_next_item +(struct sieve_header_list *_hdrlist, const char **name_r, + string_t **value_r) { - struct sieve_message_header_stringlist *strlist = - (struct sieve_message_header_stringlist *) _strlist; - const struct sieve_runtime_env *renv = _strlist->runenv; + struct sieve_message_header_list *hdrlist = + (struct sieve_message_header_list *) _hdrlist; + const struct sieve_runtime_env *renv = _hdrlist->strlist.runenv; struct mail *mail = sieve_message_get_mail(renv->msgctx); - *str_r = NULL; + if ( name_r != NULL ) + *name_r = NULL; + *value_r = NULL; /* Check for end of current header list */ - if ( strlist->headers == NULL ) { - strlist->headers_index = 0; - } else if ( strlist->headers[strlist->headers_index] == NULL ) { - strlist->headers = NULL; - strlist->headers_index = 0; + if ( hdrlist->headers == NULL ) { + hdrlist->headers_index = 0; + } else if ( hdrlist->headers[hdrlist->headers_index] == NULL ) { + hdrlist->headers = NULL; + hdrlist->headers_index = 0; } /* Fetch next header */ - while ( strlist->headers == NULL ) { + while ( hdrlist->headers == NULL ) { string_t *hdr_item = NULL; int ret; /* Read next header name from source list */ - if ( (ret=sieve_stringlist_next_item(strlist->field_names, &hdr_item)) - <= 0 ) + if ( (ret=sieve_stringlist_next_item + (hdrlist->field_names, &hdr_item)) <= 0 ) return ret; - if ( _strlist->trace ) { + hdrlist->header_name = str_c(hdr_item); + + if ( _hdrlist->strlist.trace ) { sieve_runtime_trace(renv, 0, "extracting `%s' headers from message", str_sanitize(str_c(hdr_item), 80)); } /* Fetch all matching headers from the e-mail */ - if ( strlist->mime_decode ) { - ret = mail_get_headers_utf8(mail, str_c(hdr_item), &strlist->headers); + if ( hdrlist->mime_decode ) { + ret = mail_get_headers_utf8(mail, + str_c(hdr_item), &hdrlist->headers); } else { - ret = mail_get_headers(mail, str_c(hdr_item), &strlist->headers); + ret = mail_get_headers(mail, + str_c(hdr_item), &hdrlist->headers); } if (ret < 0) { - _strlist->exec_status = sieve_runtime_mail_error - (renv, mail, "failed to read header field `%s'", str_c(hdr_item)); + _hdrlist->strlist.exec_status = + sieve_runtime_mail_error(renv, mail, + "failed to read header field `%s'", str_c(hdr_item)); return -1; } - if ( strlist->headers == NULL || strlist->headers[0] == NULL ) { + if ( hdrlist->headers == NULL || hdrlist->headers[0] == NULL ) { /* Try next item when no headers found */ - strlist->headers = NULL; + hdrlist->headers = NULL; } } /* Return next item */ - *str_r = _header_right_trim(strlist->headers[strlist->headers_index++]); + if ( name_r != NULL ) + *name_r = hdrlist->header_name; + *value_r = _header_right_trim(hdrlist->headers[hdrlist->headers_index++]); return 1; } -static void sieve_message_header_stringlist_reset -(struct sieve_stringlist *_strlist) +static int sieve_message_header_list_next_value +(struct sieve_stringlist *_strlist, string_t **value_r) { - struct sieve_message_header_stringlist *strlist = - (struct sieve_message_header_stringlist *) _strlist; + struct sieve_header_list *hdrlist = + (struct sieve_header_list *) _strlist; - strlist->headers = NULL; - strlist->headers_index = 0; - sieve_stringlist_reset(strlist->field_names); + return sieve_message_header_list_next_item + (hdrlist, NULL, value_r); +} + +static void sieve_message_header_list_reset +(struct sieve_stringlist *strlist) +{ + struct sieve_message_header_list *hdrlist = + (struct sieve_message_header_list *) strlist; + + hdrlist->headers = NULL; + hdrlist->headers_index = 0; + sieve_stringlist_reset(hdrlist->field_names); } /* @@ -784,8 +810,10 @@ if ( svmos == NULL || !array_is_created(svmos) || array_count(svmos) == 0 ) { - *fields_r = sieve_message_header_stringlist_create + struct sieve_header_list *headers; + headers = sieve_message_header_list_create (renv, field_names, mime_decode); + *fields_r = &headers->strlist; return SIEVE_EXEC_OK; } @@ -794,8 +822,10 @@ svmo[0].def->header_override != NULL ) { *fields_r = field_names; } else { - *fields_r = sieve_message_header_stringlist_create + struct sieve_header_list *headers; + headers = sieve_message_header_list_create (renv, field_names, mime_decode); + *fields_r = &headers->strlist; } for ( i = 0; i < count; i++ ) { diff -r 0bc3cb5c4c0a -r c107764e6008 src/lib-sieve/sieve-message.h --- a/src/lib-sieve/sieve-message.h Sun Nov 29 11:48:21 2015 +0100 +++ b/src/lib-sieve/sieve-message.h Sun Nov 29 11:48:29 2015 +0100 @@ -5,6 +5,7 @@ #define __SIEVE_MESSAGE_H #include "sieve-common.h" +#include "sieve-stringlist.h" #include "sieve-objects.h" /* @@ -74,8 +75,42 @@ * Header stringlist */ -struct sieve_stringlist *sieve_message_header_stringlist_create - (const struct sieve_runtime_env *renv, struct sieve_stringlist *field_names, +struct sieve_header_list { + struct sieve_stringlist strlist; + + int (*next_item) + (struct sieve_header_list *_hdrlist, const char **name_r, + string_t **value_r) ATTR_NULL(2); +}; + +static inline int sieve_header_list_next_item +(struct sieve_header_list *hdrlist, const char **name_r, + string_t **value_r) ATTR_NULL(2) +{ + return hdrlist->next_item(hdrlist, name_r, value_r); +} + +static inline void sieve_header_list_reset +(struct sieve_header_list *hdrlist) +{ + sieve_stringlist_reset(&hdrlist->strlist); +} + +static inline int sieve_header_list_get_length +(struct sieve_header_list *hdrlist) +{ + return sieve_stringlist_get_length(&hdrlist->strlist); +} + +static inline void sieve_header_list_set_trace +(struct sieve_header_list *hdrlist, bool trace) +{ + sieve_stringlist_set_trace(&hdrlist->strlist, trace); +} + +struct sieve_header_list *sieve_message_header_list_create + (const struct sieve_runtime_env *renv, + struct sieve_stringlist *field_names, bool mime_decode); /* From pigeonhole at rename-it.nl Sun Nov 29 11:17:36 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 12:17:36 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: mime/foreverypart: Fixed code... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/146ca83393c6 changeset: 2143:146ca83393c6 user: Stephan Bosch date: Sun Nov 29 12:17:32 2015 +0100 description: lib-sieve: mime/foreverypart: Fixed code dumping of `:param' argument. It erroneously reported a code corruption. diffstat: src/lib-sieve/plugins/mime/tag-mime.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 8fe5ffd62a84 -r 146ca83393c6 src/lib-sieve/plugins/mime/tag-mime.c --- a/src/lib-sieve/plugins/mime/tag-mime.c Sun Nov 29 12:16:25 2015 +0100 +++ b/src/lib-sieve/plugins/mime/tag-mime.c Sun Nov 29 12:17:32 2015 +0100 @@ -581,6 +581,7 @@ if ( !sieve_opr_stringlist_dump(denv, address, "param-list") ) return FALSE; sieve_code_ascend(denv); + break; default: return FALSE; } From pigeonhole at rename-it.nl Sun Nov 29 11:17:36 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 12:17:36 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: mime/foreverypart: Implemente... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/8fe5ffd62a84 changeset: 2142:8fe5ffd62a84 user: Stephan Bosch date: Sun Nov 29 12:16:25 2015 +0100 description: lib-sieve: mime/foreverypart: Implemented basic execution tests. diffstat: Makefile.am | 1 + tests/extensions/mime/execute.svtest | 67 +++++++++++++++++++++- tests/extensions/mime/execute/foreverypart.sieve | 15 +++- tests/extensions/mime/execute/mime.sieve | 69 ++++++++++++++++++++++++ 4 files changed, 144 insertions(+), 8 deletions(-) diffs (189 lines): diff -r 90de55c880e5 -r 8fe5ffd62a84 Makefile.am --- a/Makefile.am Sun Nov 29 11:53:24 2015 +0100 +++ b/Makefile.am Sun Nov 29 12:16:25 2015 +0100 @@ -167,6 +167,7 @@ tests/extensions/metadata/execute.svtest \ tests/extensions/metadata/errors.svtest \ tests/extensions/mime/errors.svtest \ + tests/extensions/mime/execute.svtest \ tests/extensions/mime/content-header.svtest \ tests/extensions/vnd.dovecot/debug/execute.svtest \ tests/extensions/vnd.dovecot/environment/basic.svtest \ diff -r 90de55c880e5 -r 8fe5ffd62a84 tests/extensions/mime/execute.svtest --- a/tests/extensions/mime/execute.svtest Sun Nov 29 11:53:24 2015 +0100 +++ b/tests/extensions/mime/execute.svtest Sun Nov 29 12:16:25 2015 +0100 @@ -4,8 +4,51 @@ * Execution testing (currently just meant to trigger any segfaults) */ -test "Basic" { - if not test_script_compile "execute/ihave.sieve" { +test_set "message" text: +From: Whomever +To: Someone +Date: Sat, 10 Oct 2009 00:30:04 +0200 +Subject: whatever +Content-Type: multipart/mixed; boundary=outer + +This is a multi-part message in MIME format. + +--outer +Content-Type: multipart/alternative; boundary=inner + +This is a nested multi-part message in MIME format. + +--inner +Content-Type: text/plain; charset="us-ascii" + +Hello + +--inner +Content-Type: text/html; charset="us-ascii" + +Hello + +--inner-- + +This is the end of the inner MIME multipart. + +--outer +Content-Type: message/rfc822 + +From: Someone Else +Subject: Hello, this is an elaborate request for you to finally say hello + already! + +Please say Hello + +--outer-- + +This is the end of the outer MIME multipart. +. +; + +test "Basic - foreverypart" { + if not test_script_compile "execute/foreverypart.sieve" { test_fail "script compile failed"; } @@ -17,7 +60,23 @@ test_fail "result execute failed"; } - test_binary_save "ihave-basic"; - test_binary_load "ihave-basic"; + test_binary_save "ihave-basic"; + test_binary_load "ihave-basic"; } +test "Basic - mime" { + if not test_script_compile "execute/mime.sieve" { + test_fail "script compile failed"; + } + + if not test_script_run { + test_fail "script run failed"; + } + + if not test_result_execute { + test_fail "result execute failed"; + } + + test_binary_save "ihave-basic"; + test_binary_load "ihave-basic"; +} diff -r 90de55c880e5 -r 8fe5ffd62a84 tests/extensions/mime/execute/foreverypart.sieve --- a/tests/extensions/mime/execute/foreverypart.sieve Sun Nov 29 11:53:24 2015 +0100 +++ b/tests/extensions/mime/execute/foreverypart.sieve Sun Nov 29 12:16:25 2015 +0100 @@ -1,7 +1,14 @@ -require "ihave"; +require "foreverypart"; +require "variables"; -if ihave "nonsense-extension" { - nonsense_command "Frop!"; +foreverypart { + foreverypart { + foreverypart { + foreverypart { + set "a" "a${a}"; + } + } + } } -redirect "frop at example.com"; + diff -r 90de55c880e5 -r 8fe5ffd62a84 tests/extensions/mime/execute/mime.sieve --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/extensions/mime/execute/mime.sieve Sun Nov 29 12:16:25 2015 +0100 @@ -0,0 +1,69 @@ +require "mime"; +require "foreverypart"; +require "variables"; + +if header :contains :mime "Content-Type" "text/plain" { + discard; +} +if header :mime :type "Content-Type" "text" { + discard; +} +if header :mime :subtype "Content-Type" "plain" { + discard; +} +if header :mime :contenttype "Content-Type" "text/plain" { + discard; +} +if header :mime :param ["frop", "friep"] "Content-Type" "frml" { + discard; +} +if header :anychild :contains :mime "Content-Type" "text/plain" { + discard; +} +if header :mime :anychild :type "Content-Type" "text" { + discard; +} +if header :mime :subtype :anychild "Content-Type" "plain" { + discard; +} +if header :anychild :mime :contenttype "Content-Type" "text/plain" { + discard; +} +if header :mime :param ["frop", "friep"] :anychild "Content-Type" "frml" { + discard; +} + +foreverypart { + foreverypart { + if header :contains :mime "Content-Type" "text/plain" { + discard; + } + if header :mime :type "Content-Type" "text" { + discard; + } + if header :mime :subtype "Content-Type" "plain" { + discard; + } + if header :mime :contenttype "Content-Type" "text/plain" { + discard; + } + if header :mime :param ["frop", "friep"] "Content-Type" "frml" { + discard; + } + if header :anychild :contains :mime "Content-Type" "text/plain" { + discard; + } + if header :mime :anychild :type "Content-Type" "text" { + discard; + } + if header :mime :subtype :anychild "Content-Type" "plain" { + discard; + } + if header :anychild :mime :contenttype "Content-Type" "text/plain" { + discard; + } + if header :mime :param ["frop", "friep"] :anychild "Content-Type" "frml" { + discard; + } + } +} From pigeonhole at rename-it.nl Sun Nov 29 12:00:43 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 13:00:43 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: mime/foreverypart: Implemente... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/5607f866fd94 changeset: 2144:5607f866fd94 user: Stephan Bosch date: Sun Nov 29 13:00:38 2015 +0100 description: lib-sieve: mime/foreverypart: Implemented loop nesting limit. diffstat: src/lib-sieve/plugins/mime/cmd-foreverypart.c | 43 +++++++++++++++++++++++--- src/lib-sieve/sieve-interpreter.c | 20 ++++++++++-- src/lib-sieve/sieve-interpreter.h | 5 +- src/lib-sieve/sieve-limits.h | 2 + tests/extensions/mime/errors.svtest | 10 ++++++ tests/extensions/mime/errors/limits.sieve | 13 ++++++++ 6 files changed, 82 insertions(+), 11 deletions(-) diffs (202 lines): diff -r 146ca83393c6 -r 5607f866fd94 src/lib-sieve/plugins/mime/cmd-foreverypart.c --- a/src/lib-sieve/plugins/mime/cmd-foreverypart.c Sun Nov 29 12:17:32 2015 +0100 +++ b/src/lib-sieve/plugins/mime/cmd-foreverypart.c Sun Nov 29 13:00:38 2015 +0100 @@ -2,6 +2,7 @@ */ #include "sieve-common.h" +#include "sieve-limits.h" #include "sieve-code.h" #include "sieve-extensions.h" #include "sieve-commands.h" @@ -28,6 +29,8 @@ struct sieve_command_registration *cmd_reg); static bool cmd_foreverypart_pre_validate (struct sieve_validator *valdtr, struct sieve_command *cmd); +static bool cmd_foreverypart_validate + (struct sieve_validator *valdtr, struct sieve_command *cmd); static bool cmd_foreverypart_generate (const struct sieve_codegen_env *cgenv, struct sieve_command *ctx); @@ -38,7 +41,8 @@ 0, 0, TRUE, TRUE, cmd_foreverypart_registered, cmd_foreverypart_pre_validate, - NULL, NULL, + cmd_foreverypart_validate, + NULL, cmd_foreverypart_generate, NULL, }; @@ -149,6 +153,36 @@ return TRUE; } +static bool cmd_foreverypart_validate +(struct sieve_validator *valdtr, struct sieve_command *cmd) +{ + struct sieve_ast_node *node = cmd->ast_node; + unsigned int nesting = 0; + + /* Determine nesting depth of foreverypart commands at this point. */ + i_assert(node != NULL); + node = sieve_ast_node_parent(node); + while ( node != NULL && node->command != NULL ) { + if ( sieve_command_is(node->command, cmd_foreverypart) ) + nesting++; + node = sieve_ast_node_parent(node); + } + + /* Enforce nesting limit + NOTE: this only recognizes the foreverypart command as a loop; if + new loop commands are introduced in the future, these must be + recognized somehow. */ + if ( nesting + 1 > SIEVE_MAX_LOOP_DEPTH ) { + sieve_command_validate_error(valdtr, cmd, + "the nested foreverypart loop exceeds " + "the nesting limit (<= %u levels)", + SIEVE_MAX_LOOP_DEPTH); + return FALSE; + } + + return TRUE; +} + /* * Code generation */ @@ -262,10 +296,9 @@ sfploop = ext_foreverypart_runtime_loop_get_current(renv); - loop = sieve_interpreter_loop_start - (renv->interp, loop_end, &foreverypart_extension); - if ( loop == NULL ) - return SIEVE_EXEC_BIN_CORRUPT; + if ( (ret=sieve_interpreter_loop_start(renv->interp, + loop_end, &foreverypart_extension, &loop)) <= 0 ) + return ret; pool = sieve_interpreter_loop_get_pool(loop); fploop = p_new(pool, struct ext_foreverypart_runtime_loop, 1); diff -r 146ca83393c6 -r 5607f866fd94 src/lib-sieve/sieve-interpreter.c --- a/src/lib-sieve/sieve-interpreter.c Sun Nov 29 12:17:32 2015 +0100 +++ b/src/lib-sieve/sieve-interpreter.c Sun Nov 29 13:00:38 2015 +0100 @@ -9,6 +9,7 @@ #include "mail-storage.h" #include "sieve-common.h" +#include "sieve-limits.h" #include "sieve-script.h" #include "sieve-error.h" #include "sieve-extensions.h" @@ -482,9 +483,10 @@ * Loop handling */ -struct sieve_interpreter_loop *sieve_interpreter_loop_start +int sieve_interpreter_loop_start (struct sieve_interpreter *interp, sieve_size_t loop_end, - const struct sieve_extension_def *ext_def) + const struct sieve_extension_def *ext_def, + struct sieve_interpreter_loop **loop_r) { const struct sieve_runtime_env *renv = &interp->runenv; struct sieve_interpreter_loop *loop; @@ -494,7 +496,7 @@ if ( loop_end > sieve_binary_block_get_size(renv->sblock) ) { sieve_runtime_trace_error(renv, "loop end offset out of range"); - return NULL; + return SIEVE_EXEC_BIN_CORRUPT; } if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { @@ -511,6 +513,15 @@ if ( !array_is_created(&interp->loop_stack) ) p_array_init(&interp->loop_stack, interp->pool, 8); + else if ( array_count(&interp->loop_stack) + >= SIEVE_MAX_LOOP_DEPTH ) { + /* Should normally be caught at compile time */ + sieve_runtime_error(renv, NULL, + "new program loop exceeds " + "the nesting limit (<= %u levels)", + SIEVE_MAX_LOOP_DEPTH); + return SIEVE_EXEC_FAILURE; + } loop = array_append_space(&interp->loop_stack); loop->level = array_count(&interp->loop_stack)-1; @@ -519,7 +530,8 @@ loop->end = loop_end; loop->pool = pool_alloconly_create("sieve_interpreter", 128); - return loop; + *loop_r = loop; + return SIEVE_EXEC_OK; } struct sieve_interpreter_loop *sieve_interpreter_loop_get diff -r 146ca83393c6 -r 5607f866fd94 src/lib-sieve/sieve-interpreter.h --- a/src/lib-sieve/sieve-interpreter.h Sun Nov 29 12:17:32 2015 +0100 +++ b/src/lib-sieve/sieve-interpreter.h Sun Nov 29 13:00:38 2015 +0100 @@ -51,9 +51,10 @@ struct sieve_interpreter_loop; -struct sieve_interpreter_loop *sieve_interpreter_loop_start +int sieve_interpreter_loop_start (struct sieve_interpreter *interp, sieve_size_t loop_end, - const struct sieve_extension_def *ext_def); + const struct sieve_extension_def *ext_def, + struct sieve_interpreter_loop **loop_r); struct sieve_interpreter_loop *sieve_interpreter_loop_get (struct sieve_interpreter *interp, sieve_size_t loop_end, const struct sieve_extension_def *ext_def); diff -r 146ca83393c6 -r 5607f866fd94 src/lib-sieve/sieve-limits.h --- a/src/lib-sieve/sieve-limits.h Sun Nov 29 12:17:32 2015 +0100 +++ b/src/lib-sieve/sieve-limits.h Sun Nov 29 13:00:38 2015 +0100 @@ -12,6 +12,8 @@ #define SIEVE_DEFAULT_MAX_SCRIPT_SIZE (1 << 20) +#define SIEVE_MAX_LOOP_DEPTH 4 + /* * Lexer */ diff -r 146ca83393c6 -r 5607f866fd94 tests/extensions/mime/errors.svtest --- a/tests/extensions/mime/errors.svtest Sun Nov 29 12:17:32 2015 +0100 +++ b/tests/extensions/mime/errors.svtest Sun Nov 29 13:00:38 2015 +0100 @@ -52,3 +52,13 @@ test_fail "incorrect number of compile errors reported"; } } + +test "Limits" { + if test_script_compile "errors/limits.sieve" { + test_fail "compile should have failed"; + } + + if test_error :count "ne" :comparator "i;ascii-numeric" "2" { + test_fail "incorrect number of compile errors reported"; + } +} diff -r 146ca83393c6 -r 5607f866fd94 tests/extensions/mime/errors/limits.sieve --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/extensions/mime/errors/limits.sieve Sun Nov 29 13:00:38 2015 +0100 @@ -0,0 +1,13 @@ +require "foreverypart"; + +foreverypart :name "frop" { + foreverypart :name "friep" { + foreverypart :name "frml" { + foreverypart { + foreverypart { + break; + } + } + } + } +} From pigeonhole at rename-it.nl Sun Nov 29 12:46:08 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 13:46:08 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: mime/foreverypart: Enforced l... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/8b9147c83fd9 changeset: 2145:8b9147c83fd9 user: Stephan Bosch date: Sun Nov 29 13:46:03 2015 +0100 description: lib-sieve: mime/foreverypart: Enforced loop nesting limit across includes. diffstat: src/lib-sieve/plugins/include/ext-include-common.c | 7 +- src/lib-sieve/sieve-interpreter.c | 45 ++++++++++++++------ src/lib-sieve/sieve-interpreter.h | 22 +++++++--- src/lib-sieve/sieve.c | 4 +- src/testsuite/testsuite-script.c | 4 +- src/testsuite/testsuite.c | 2 +- tests/extensions/mime/errors.svtest | 13 +++++- tests/extensions/mime/errors/limits-include.sieve | 6 ++ tests/extensions/mime/included/include-loop-2.sieve | 6 ++ tests/extensions/mime/included/include-loop-3.sieve | 6 ++ tests/extensions/mime/included/include-loop-4.sieve | 6 ++ tests/extensions/mime/included/include-loop-5.sieve | 9 ++++ 12 files changed, 101 insertions(+), 29 deletions(-) diffs (276 lines): diff -r 5607f866fd94 -r 8b9147c83fd9 src/lib-sieve/plugins/include/ext-include-common.c --- a/src/lib-sieve/plugins/include/ext-include-common.c Sun Nov 29 13:00:38 2015 +0100 +++ b/src/lib-sieve/plugins/include/ext-include-common.c Sun Nov 29 13:46:03 2015 +0100 @@ -708,8 +708,8 @@ * (first sub-interpreter) */ subinterp = sieve_interpreter_create_for_block - (included->block, included->script, renv->msgdata, renv->scriptenv, - ehandler, rtflags); + (included->block, included->script, renv->interp, + renv->msgdata, renv->scriptenv, ehandler, rtflags); if ( subinterp != NULL ) { curctx = ext_include_interpreter_context_init_child @@ -767,7 +767,8 @@ /* Create sub-interpreter */ subinterp = sieve_interpreter_create_for_block - (curctx->include->block, curctx->include->script, renv->msgdata, + (curctx->include->block, curctx->include->script, + curctx->interp, renv->msgdata, renv->scriptenv, ehandler, rtflags); if ( subinterp != NULL ) { diff -r 5607f866fd94 -r 8b9147c83fd9 src/lib-sieve/sieve-interpreter.c --- a/src/lib-sieve/sieve-interpreter.c Sun Nov 29 13:00:38 2015 +0100 +++ b/src/lib-sieve/sieve-interpreter.c Sun Nov 29 13:46:03 2015 +0100 @@ -71,6 +71,7 @@ /* Loop stack */ ARRAY(struct sieve_interpreter_loop) loop_stack; sieve_size_t loop_limit; + unsigned int parent_loop_level; /* Runtime environment */ struct sieve_runtime_env runenv; @@ -85,10 +86,15 @@ }; static struct sieve_interpreter *_sieve_interpreter_create -(struct sieve_binary *sbin, struct sieve_binary_block *sblock, - struct sieve_script *script, const struct sieve_message_data *msgdata, - const struct sieve_script_env *senv, struct sieve_error_handler *ehandler, +(struct sieve_binary *sbin, + struct sieve_binary_block *sblock, + struct sieve_script *script, + struct sieve_interpreter *parent, + const struct sieve_message_data *msgdata, + const struct sieve_script_env *senv, + struct sieve_error_handler *ehandler, enum sieve_runtime_flags flags) + ATTR_NULL(3, 4) { unsigned int i, ext_count; struct sieve_interpreter *interp; @@ -143,6 +149,12 @@ p_array_init(&interp->extensions, pool, sieve_extensions_get_count(svinst)); + interp->parent_loop_level = 0; + if ( parent != NULL && array_is_created(&parent->loop_stack) ) { + interp->parent_loop_level = parent->parent_loop_level + + array_count(&parent->loop_stack); + } + /* Pre-load core language features implemented as 'extensions' */ ext_preloaded = sieve_extensions_get_preloaded(svinst, &ext_count); for ( i = 0; i < ext_count; i++ ) { @@ -211,8 +223,11 @@ } struct sieve_interpreter *sieve_interpreter_create -(struct sieve_binary *sbin, const struct sieve_message_data *msgdata, - const struct sieve_script_env *senv, struct sieve_error_handler *ehandler, +(struct sieve_binary *sbin, + struct sieve_interpreter *parent, + const struct sieve_message_data *msgdata, + const struct sieve_script_env *senv, + struct sieve_error_handler *ehandler, enum sieve_runtime_flags flags) { struct sieve_binary_block *sblock; @@ -221,20 +236,24 @@ == NULL ) return NULL; - return _sieve_interpreter_create - (sbin, sblock, NULL, msgdata, senv, ehandler, flags); + return _sieve_interpreter_create(sbin, sblock, NULL, + parent, msgdata, senv, ehandler, flags); } struct sieve_interpreter *sieve_interpreter_create_for_block -(struct sieve_binary_block *sblock, struct sieve_script *script, - const struct sieve_message_data *msgdata, const struct sieve_script_env *senv, - struct sieve_error_handler *ehandler, enum sieve_runtime_flags flags) +(struct sieve_binary_block *sblock, + struct sieve_script *script, + struct sieve_interpreter *parent, + const struct sieve_message_data *msgdata, + const struct sieve_script_env *senv, + struct sieve_error_handler *ehandler, + enum sieve_runtime_flags flags) { if ( sblock == NULL ) return NULL; return _sieve_interpreter_create - (sieve_binary_block_get_binary(sblock), sblock, script, msgdata, senv, - ehandler, flags); + (sieve_binary_block_get_binary(sblock), sblock, script, + parent, msgdata, senv, ehandler, flags); } void sieve_interpreter_free(struct sieve_interpreter **_interp) @@ -513,7 +532,7 @@ if ( !array_is_created(&interp->loop_stack) ) p_array_init(&interp->loop_stack, interp->pool, 8); - else if ( array_count(&interp->loop_stack) + if ( (interp->parent_loop_level + array_count(&interp->loop_stack)) >= SIEVE_MAX_LOOP_DEPTH ) { /* Should normally be caught at compile time */ sieve_runtime_error(renv, NULL, diff -r 5607f866fd94 -r 8b9147c83fd9 src/lib-sieve/sieve-interpreter.h --- a/src/lib-sieve/sieve-interpreter.h Sun Nov 29 13:00:38 2015 +0100 +++ b/src/lib-sieve/sieve-interpreter.h Sun Nov 29 13:46:03 2015 +0100 @@ -16,14 +16,22 @@ */ struct sieve_interpreter *sieve_interpreter_create - (struct sieve_binary *sbin, const struct sieve_message_data *msgdata, - const struct sieve_script_env *senv, struct sieve_error_handler *ehandler, + (struct sieve_binary *sbin, + struct sieve_interpreter *parent, + const struct sieve_message_data *msgdata, + const struct sieve_script_env *senv, + struct sieve_error_handler *ehandler, + enum sieve_runtime_flags flags) + ATTR_NULL(2); +struct sieve_interpreter *sieve_interpreter_create_for_block + (struct sieve_binary_block *sblock, + struct sieve_script *script, + struct sieve_interpreter *parent, + const struct sieve_message_data *msgdata, + const struct sieve_script_env *senv, + struct sieve_error_handler *ehandler, enum sieve_runtime_flags flags); -struct sieve_interpreter *sieve_interpreter_create_for_block - (struct sieve_binary_block *sblock, struct sieve_script *script, - const struct sieve_message_data *msgdata, - const struct sieve_script_env *senv, struct sieve_error_handler *ehandler, - enum sieve_runtime_flags flags); + ATTR_NULL(3); void sieve_interpreter_free(struct sieve_interpreter **_interp); /* diff -r 5607f866fd94 -r 8b9147c83fd9 src/lib-sieve/sieve.c --- a/src/lib-sieve/sieve.c Sun Nov 29 13:00:38 2015 +0100 +++ b/src/lib-sieve/sieve.c Sun Nov 29 13:46:03 2015 +0100 @@ -323,8 +323,8 @@ int ret = 0; /* Create the interpreter */ - if ( (interp=sieve_interpreter_create(sbin, msgdata, senv, ehandler, flags)) - == NULL ) + if ( (interp=sieve_interpreter_create + (sbin, NULL, msgdata, senv, ehandler, flags)) == NULL ) return SIEVE_EXEC_BIN_CORRUPT; /* Reset execution status */ diff -r 5607f866fd94 -r 8b9147c83fd9 src/testsuite/testsuite-script.c --- a/src/testsuite/testsuite-script.c Sun Nov 29 13:00:38 2015 +0100 +++ b/src/testsuite/testsuite-script.c Sun Nov 29 13:46:03 2015 +0100 @@ -122,8 +122,8 @@ result = testsuite_result_get(); /* Execute the script */ - interp=sieve_interpreter_create(ictx->compiled_script, renv->msgdata, - &scriptenv, testsuite_log_ehandler, 0); + interp=sieve_interpreter_create(ictx->compiled_script, + NULL, renv->msgdata, &scriptenv, testsuite_log_ehandler, 0); if ( interp == NULL ) return SIEVE_EXEC_BIN_CORRUPT; diff -r 5607f866fd94 -r 8b9147c83fd9 src/testsuite/testsuite.c --- a/src/testsuite/testsuite.c Sun Nov 29 13:00:38 2015 +0100 +++ b/src/testsuite/testsuite.c Sun Nov 29 13:46:03 2015 +0100 @@ -65,7 +65,7 @@ /* Create the interpreter */ if ( (interp=sieve_interpreter_create - (sbin, msgdata, senv, ehandler, 0)) == NULL ) + (sbin, NULL, msgdata, senv, ehandler, 0)) == NULL ) return SIEVE_EXEC_BIN_CORRUPT; /* Run the interpreter */ diff -r 5607f866fd94 -r 8b9147c83fd9 tests/extensions/mime/errors.svtest --- a/tests/extensions/mime/errors.svtest Sun Nov 29 13:00:38 2015 +0100 +++ b/tests/extensions/mime/errors.svtest Sun Nov 29 13:46:03 2015 +0100 @@ -3,7 +3,7 @@ require "relational"; require "comparator-i;ascii-numeric"; -test "Foreverypart command" { +/*test "Foreverypart command" { if test_script_compile "errors/foreverypart.sieve" { test_fail "compile should have failed"; } @@ -61,4 +61,15 @@ if test_error :count "ne" :comparator "i;ascii-numeric" "2" { test_fail "incorrect number of compile errors reported"; } +}*/ + +test "Limits - include" { + if not test_script_compile "errors/limits-include.sieve" { + test_fail "script compile failed"; + } + + if test_script_run { + test_fail "script run should have failed"; + } } + diff -r 5607f866fd94 -r 8b9147c83fd9 tests/extensions/mime/errors/limits-include.sieve --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/extensions/mime/errors/limits-include.sieve Sun Nov 29 13:46:03 2015 +0100 @@ -0,0 +1,6 @@ +require "foreverypart"; +require "include"; + +foreverypart :name "frop" { + include "include-loop-2"; +} diff -r 5607f866fd94 -r 8b9147c83fd9 tests/extensions/mime/included/include-loop-2.sieve --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/extensions/mime/included/include-loop-2.sieve Sun Nov 29 13:46:03 2015 +0100 @@ -0,0 +1,6 @@ +require "foreverypart"; +require "include"; + +foreverypart :name "friep" { + include "include-loop-3"; +} diff -r 5607f866fd94 -r 8b9147c83fd9 tests/extensions/mime/included/include-loop-3.sieve --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/extensions/mime/included/include-loop-3.sieve Sun Nov 29 13:46:03 2015 +0100 @@ -0,0 +1,6 @@ +require "foreverypart"; +require "include"; + +foreverypart :name "frml" { + include "include-loop-4"; +} diff -r 5607f866fd94 -r 8b9147c83fd9 tests/extensions/mime/included/include-loop-4.sieve --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/extensions/mime/included/include-loop-4.sieve Sun Nov 29 13:46:03 2015 +0100 @@ -0,0 +1,6 @@ +require "foreverypart"; +require "include"; + +foreverypart { + include "include-loop-5"; +} diff -r 5607f866fd94 -r 8b9147c83fd9 tests/extensions/mime/included/include-loop-5.sieve --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/extensions/mime/included/include-loop-5.sieve Sun Nov 29 13:46:03 2015 +0100 @@ -0,0 +1,9 @@ +require "foreverypart"; +require "include"; +require "mime"; + +foreverypart { + if header :mime :subtype "content-type" "plain" { + break; + } +} From pigeonhole at rename-it.nl Sun Nov 29 13:59:48 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 14:59:48 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: interpreter: Improved robustn... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/3e5d2ed2052b changeset: 2146:3e5d2ed2052b user: Stephan Bosch date: Sun Nov 29 14:58:20 2015 +0100 description: lib-sieve: interpreter: Improved robustness of code loop handling against binary corruption. When the program crosses the boundary of the current jump somehow, the program is terminated and a corruption error is returned. diffstat: src/lib-sieve/plugins/mime/cmd-foreverypart.c | 8 +- src/lib-sieve/sieve-interpreter.c | 68 +++++++++++++++++++++----- src/lib-sieve/sieve-interpreter.h | 4 +- 3 files changed, 60 insertions(+), 20 deletions(-) diffs (234 lines): diff -r 8b9147c83fd9 -r 3e5d2ed2052b src/lib-sieve/plugins/mime/cmd-foreverypart.c --- a/src/lib-sieve/plugins/mime/cmd-foreverypart.c Sun Nov 29 13:46:03 2015 +0100 +++ b/src/lib-sieve/plugins/mime/cmd-foreverypart.c Sun Nov 29 14:58:20 2015 +0100 @@ -362,11 +362,9 @@ i_assert(fploop->part != NULL); fploop->part = sieve_message_part_iter_next(&fploop->part_iter); if ( fploop->part == NULL ) - sieve_interpreter_loop_break(renv->interp, loop); - else - sieve_interpreter_loop_next(renv->interp, loop, loop_begin); - - return SIEVE_EXEC_OK; + return sieve_interpreter_loop_break(renv->interp, loop); + + return sieve_interpreter_loop_next(renv->interp, loop, loop_begin); } diff -r 8b9147c83fd9 -r 3e5d2ed2052b src/lib-sieve/sieve-interpreter.c --- a/src/lib-sieve/sieve-interpreter.c Sun Nov 29 13:46:03 2015 +0100 +++ b/src/lib-sieve/sieve-interpreter.c Sun Nov 29 14:58:20 2015 +0100 @@ -512,12 +512,14 @@ i_assert( loop_end > interp->runenv.pc ); + /* Check supplied end offset */ if ( loop_end > sieve_binary_block_get_size(renv->sblock) ) { sieve_runtime_trace_error(renv, "loop end offset out of range"); return SIEVE_EXEC_BIN_CORRUPT; } + /* Trace */ if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { unsigned int line = sieve_runtime_get_source_location(renv, loop_end); @@ -530,6 +532,7 @@ } } + /* Check loop nesting limit */ if ( !array_is_created(&interp->loop_stack) ) p_array_init(&interp->loop_stack, interp->pool, 8); if ( (interp->parent_loop_level + array_count(&interp->loop_stack)) @@ -542,12 +545,16 @@ return SIEVE_EXEC_FAILURE; } + /* Create new loop */ loop = array_append_space(&interp->loop_stack); loop->level = array_count(&interp->loop_stack)-1; loop->ext_def = ext_def; loop->begin = interp->runenv.pc; loop->end = loop_end; loop->pool = pool_alloconly_create("sieve_interpreter", 128); + + /* Set new loop limit */ + interp->loop_limit = loop_end; *loop_r = loop; return SIEVE_EXEC_OK; @@ -573,7 +580,7 @@ return NULL; } -void sieve_interpreter_loop_next(struct sieve_interpreter *interp, +int sieve_interpreter_loop_next(struct sieve_interpreter *interp, struct sieve_interpreter_loop *loop, sieve_size_t loop_begin) { @@ -581,6 +588,7 @@ struct sieve_interpreter_loop *loops; unsigned int count; + /* Trace */ if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { unsigned int line = sieve_runtime_get_source_location(renv, loop_begin); @@ -593,16 +601,24 @@ } } - i_assert( loop->begin == loop_begin ); + /* Check the code for corruption */ + if ( loop->begin != loop_begin ) { + sieve_runtime_trace_error(renv, + "loop begin offset invalid"); + return SIEVE_EXEC_BIN_CORRUPT; + } + /* Check invariants */ i_assert( array_is_created(&interp->loop_stack) ); loops = array_get_modifiable(&interp->loop_stack, &count); i_assert( &loops[count-1] == loop ); + /* Return to beginning */ interp->runenv.pc = loop_begin; + return SIEVE_EXEC_OK; } -void sieve_interpreter_loop_break(struct sieve_interpreter *interp, +int sieve_interpreter_loop_break(struct sieve_interpreter *interp, struct sieve_interpreter_loop *loop) { const struct sieve_runtime_env *renv = &interp->runenv; @@ -610,27 +626,38 @@ sieve_size_t loop_end = loop->end; unsigned int count, i; + /* Find the loop */ i_assert( array_is_created(&interp->loop_stack) ); loops = array_get_modifiable(&interp->loop_stack, &count); for ( i = count; i > 0 && &loops[i-1] != loop; i-- ) pool_unref(&loops[i-1].pool); i_assert( i > 0 && &loops[i-1] == loop ); + /* Delete it and all deeper loops */ array_delete(&interp->loop_stack, i-1, count - (i-1)); + /* Set new loop limit */ + if ( --i > 0 ) + interp->loop_limit = loops[i-1].end; + else + interp->loop_limit =- 0; + + /* Trace */ if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { unsigned int jmp_line = sieve_runtime_get_source_location(renv, loop_end); if ( sieve_runtime_trace_hasflag(renv, SIEVE_TRFLG_ADDRESSES) ) { - sieve_runtime_trace(renv, 0, "ending loop at line %d [%08llx]", + sieve_runtime_trace(renv, 0, "exiting loops at line %d [%08llx]", jmp_line, (long long unsigned int) loop_end); } else { - sieve_runtime_trace(renv, 0, "ending loop at line %d", jmp_line); + sieve_runtime_trace(renv, 0, "exiting loops at line %d", jmp_line); } } + /* Exit loop */ interp->runenv.pc = loop->end; + return SIEVE_EXEC_OK; } struct sieve_interpreter_loop *sieve_interpreter_loop_get_surrounding @@ -700,6 +727,8 @@ const struct sieve_runtime_env *renv = &interp->runenv; sieve_size_t *address = &(interp->runenv.pc); sieve_size_t jmp_start = *address; + sieve_size_t jmp_limit = ( interp->loop_limit == 0 ? + sieve_binary_block_get_size(renv->sblock) : interp->loop_limit ); sieve_offset_t jmp_offset; if ( !sieve_binary_read_offset(renv->sblock, address, &jmp_offset) ) @@ -708,7 +737,7 @@ return SIEVE_EXEC_BIN_CORRUPT; } - if ( jmp_start + jmp_offset <= sieve_binary_block_get_size(renv->sblock) && + if ( (jmp_start + jmp_offset) <= jmp_limit && jmp_start + jmp_offset > 0 ) { if ( jump ) { @@ -734,7 +763,13 @@ return SIEVE_EXEC_OK; } - sieve_runtime_trace_error(renv, "jump offset out of range"); + if ( interp->loop_limit != 0 ) { + sieve_runtime_trace_error(renv, + "jump offset crosses loop boundary"); + } else { + sieve_runtime_trace_error(renv, + "jump offset out of range"); + } return SIEVE_EXEC_BIN_CORRUPT; } @@ -796,24 +831,31 @@ int sieve_interpreter_continue (struct sieve_interpreter *interp, bool *interrupted) { + const struct sieve_runtime_env *renv = &interp->runenv; sieve_size_t *address = &(interp->runenv.pc); int ret = SIEVE_EXEC_OK; - sieve_result_ref(interp->runenv.result); + sieve_result_ref(renv->result); interp->interrupted = FALSE; if ( interrupted != NULL ) *interrupted = FALSE; while ( ret == SIEVE_EXEC_OK && !interp->interrupted && - *address < sieve_binary_block_get_size(interp->runenv.sblock) ) { + *address < sieve_binary_block_get_size(renv->sblock) ) { + if ( interp->loop_limit != 0 && *address > interp->loop_limit ) { + sieve_runtime_trace_error(renv, + "program crossed loop boundary"); + ret = SIEVE_EXEC_BIN_CORRUPT; + break; + } ret = sieve_interpreter_operation_execute(interp); + } - if ( ret != SIEVE_EXEC_OK ) { - sieve_runtime_trace(&interp->runenv, SIEVE_TRLVL_NONE, - "[[EXECUTION ABORTED]]"); - } + if ( ret != SIEVE_EXEC_OK ) { + sieve_runtime_trace(&interp->runenv, SIEVE_TRLVL_NONE, + "[[EXECUTION ABORTED]]"); } if ( interrupted != NULL ) diff -r 8b9147c83fd9 -r 3e5d2ed2052b src/lib-sieve/sieve-interpreter.h --- a/src/lib-sieve/sieve-interpreter.h Sun Nov 29 13:46:03 2015 +0100 +++ b/src/lib-sieve/sieve-interpreter.h Sun Nov 29 14:58:20 2015 +0100 @@ -66,11 +66,11 @@ struct sieve_interpreter_loop *sieve_interpreter_loop_get (struct sieve_interpreter *interp, sieve_size_t loop_end, const struct sieve_extension_def *ext_def); -void sieve_interpreter_loop_next +int sieve_interpreter_loop_next (struct sieve_interpreter *interp, struct sieve_interpreter_loop *loop, sieve_size_t loop_begin); -void sieve_interpreter_loop_break +int sieve_interpreter_loop_break (struct sieve_interpreter *interp, struct sieve_interpreter_loop *loop); From pigeonhole at rename-it.nl Sun Nov 29 14:33:30 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 15:33:30 +0100 Subject: dovecot-2.2-pigeonhole: testsuite: Accidentally committed disabl... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/cfa3025bb14f changeset: 2147:cfa3025bb14f user: Stephan Bosch date: Sun Nov 29 15:28:18 2015 +0100 description: testsuite: Accidentally committed disabled tests. diffstat: tests/extensions/mime/errors.svtest | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 3e5d2ed2052b -r cfa3025bb14f tests/extensions/mime/errors.svtest --- a/tests/extensions/mime/errors.svtest Sun Nov 29 14:58:20 2015 +0100 +++ b/tests/extensions/mime/errors.svtest Sun Nov 29 15:28:18 2015 +0100 @@ -3,7 +3,7 @@ require "relational"; require "comparator-i;ascii-numeric"; -/*test "Foreverypart command" { +test "Foreverypart command" { if test_script_compile "errors/foreverypart.sieve" { test_fail "compile should have failed"; } @@ -61,7 +61,7 @@ if test_error :count "ne" :comparator "i;ascii-numeric" "2" { test_fail "incorrect number of compile errors reported"; } -}*/ +} test "Limits - include" { if not test_script_compile "errors/limits-include.sieve" { From pigeonhole at rename-it.nl Sun Nov 29 14:33:30 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 15:33:30 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Fixed memory leak occurring i... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/d9d6a8e4d0b9 changeset: 2148:d9d6a8e4d0b9 user: Stephan Bosch date: Sun Nov 29 15:33:25 2015 +0100 description: lib-sieve: Fixed memory leak occurring in interpreter loop handling. diffstat: src/lib-sieve/sieve-interpreter.c | 23 ++++++++++++++--------- 1 files changed, 14 insertions(+), 9 deletions(-) diffs (36 lines): diff -r cfa3025bb14f -r d9d6a8e4d0b9 src/lib-sieve/sieve-interpreter.c --- a/src/lib-sieve/sieve-interpreter.c Sun Nov 29 15:28:18 2015 +0100 +++ b/src/lib-sieve/sieve-interpreter.c Sun Nov 29 15:33:25 2015 +0100 @@ -629,18 +629,23 @@ /* Find the loop */ i_assert( array_is_created(&interp->loop_stack) ); loops = array_get_modifiable(&interp->loop_stack, &count); - for ( i = count; i > 0 && &loops[i-1] != loop; i-- ) + i_assert( count > 0 ); + + i = count; + do { pool_unref(&loops[i-1].pool); - i_assert( i > 0 && &loops[i-1] == loop ); + i--; + } while ( i > 0 && &loops[i] != loop ); + i_assert( &loops[i] == loop ); + + /* Set new loop limit */ + if ( i > 0 ) + interp->loop_limit = loops[i].end; + else + interp->loop_limit = 0; /* Delete it and all deeper loops */ - array_delete(&interp->loop_stack, i-1, count - (i-1)); - - /* Set new loop limit */ - if ( --i > 0 ) - interp->loop_limit = loops[i-1].end; - else - interp->loop_limit =- 0; + array_delete(&interp->loop_stack, i, count - i); /* Trace */ if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { From pigeonhole at rename-it.nl Sun Nov 29 15:17:55 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 16:17:55 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: interpreter; Added support fo... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/d9bc8186e02e changeset: 2149:d9bc8186e02e user: Stephan Bosch date: Sun Nov 29 16:17:15 2015 +0100 description: lib-sieve: interpreter; Added support for program jumps that explicitly cross loop boundaries. Needed for test suite. diffstat: src/lib-sieve/sieve-code.c | 6 ++-- src/lib-sieve/sieve-interpreter.c | 45 ++++++++++++++++++++++++++++++-------- src/lib-sieve/sieve-interpreter.h | 2 +- 3 files changed, 39 insertions(+), 14 deletions(-) diffs (126 lines): diff -r d9d6a8e4d0b9 -r d9bc8186e02e src/lib-sieve/sieve-code.c --- a/src/lib-sieve/sieve-code.c Sun Nov 29 15:33:25 2015 +0100 +++ b/src/lib-sieve/sieve-code.c Sun Nov 29 16:17:15 2015 +0100 @@ -1145,7 +1145,7 @@ static int opc_jmp_execute (const struct sieve_runtime_env *renv, sieve_size_t *address ATTR_UNUSED) { - return sieve_interpreter_program_jump(renv->interp, TRUE); + return sieve_interpreter_program_jump(renv->interp, TRUE, FALSE); } static int opc_jmptrue_execute @@ -1156,7 +1156,7 @@ sieve_runtime_trace(renv, SIEVE_TRLVL_COMMANDS, "jump if result is true"); sieve_runtime_trace_descend(renv); - return sieve_interpreter_program_jump(renv->interp, result); + return sieve_interpreter_program_jump(renv->interp, result, FALSE); } static int opc_jmpfalse_execute @@ -1167,5 +1167,5 @@ sieve_runtime_trace(renv, SIEVE_TRLVL_COMMANDS, "jump if result is false"); sieve_runtime_trace_descend(renv); - return sieve_interpreter_program_jump(renv->interp, !result); + return sieve_interpreter_program_jump(renv->interp, !result, FALSE); } diff -r d9d6a8e4d0b9 -r d9bc8186e02e src/lib-sieve/sieve-interpreter.c --- a/src/lib-sieve/sieve-interpreter.c Sun Nov 29 15:33:25 2015 +0100 +++ b/src/lib-sieve/sieve-interpreter.c Sun Nov 29 16:17:15 2015 +0100 @@ -665,6 +665,28 @@ return SIEVE_EXEC_OK; } +static int +sieve_interpreter_loop_break_out(struct sieve_interpreter *interp, + sieve_size_t target) +{ + struct sieve_interpreter_loop *loops; + unsigned int count, i; + + if ( !array_is_created(&interp->loop_stack) ) + return SIEVE_EXEC_OK; + + loops = array_get_modifiable(&interp->loop_stack, &count); + for ( i = count; i > 0; i-- ) { + /* We're really making sure our loop matches */ + if ( loops[i-1].end > target ) + break; + } + if ( i == count ) + return SIEVE_EXEC_OK; + + return sieve_interpreter_loop_break(interp, &loops[i]); +} + struct sieve_interpreter_loop *sieve_interpreter_loop_get_surrounding (struct sieve_interpreter *interp, struct sieve_interpreter_loop *loop, @@ -727,14 +749,14 @@ } int sieve_interpreter_program_jump -(struct sieve_interpreter *interp, bool jump) +(struct sieve_interpreter *interp, bool jump, bool break_loops) { const struct sieve_runtime_env *renv = &interp->runenv; sieve_size_t *address = &(interp->runenv.pc); - sieve_size_t jmp_start = *address; - sieve_size_t jmp_limit = ( interp->loop_limit == 0 ? - sieve_binary_block_get_size(renv->sblock) : interp->loop_limit ); + sieve_size_t jmp_start = *address, jmp_target; + sieve_size_t loop_limit = ( break_loops ? 0 : interp->loop_limit ); sieve_offset_t jmp_offset; + int ret; if ( !sieve_binary_read_offset(renv->sblock, address, &jmp_offset) ) { @@ -742,25 +764,28 @@ return SIEVE_EXEC_BIN_CORRUPT; } - if ( (jmp_start + jmp_offset) <= jmp_limit && + jmp_target = jmp_start + jmp_offset; + if ( jmp_target <= sieve_binary_block_get_size(renv->sblock) && + (loop_limit == 0 || jmp_target < loop_limit) && jmp_start + jmp_offset > 0 ) { if ( jump ) { - sieve_size_t jmp_addr = jmp_start + jmp_offset; - if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { unsigned int jmp_line = - sieve_runtime_get_source_location(renv, jmp_addr); + sieve_runtime_get_source_location(renv, jmp_target); if ( sieve_runtime_trace_hasflag(renv, SIEVE_TRFLG_ADDRESSES) ) { sieve_runtime_trace(renv, 0, "jumping to line %d [%08llx]", - jmp_line, (long long unsigned int) jmp_addr); + jmp_line, (long long unsigned int) jmp_target); } else { sieve_runtime_trace(renv, 0, "jumping to line %d", jmp_line); } } - *address = jmp_addr; + if ( break_loops && + (ret=sieve_interpreter_loop_break_out(interp, jmp_target)) <= 0 ) + return ret; + *address = jmp_target; } else { sieve_runtime_trace(renv, 0, "not jumping"); } diff -r d9d6a8e4d0b9 -r d9bc8186e02e src/lib-sieve/sieve-interpreter.h --- a/src/lib-sieve/sieve-interpreter.h Sun Nov 29 15:33:25 2015 +0100 +++ b/src/lib-sieve/sieve-interpreter.h Sun Nov 29 16:17:15 2015 +0100 @@ -98,7 +98,7 @@ (struct sieve_interpreter *interp); int sieve_interpreter_program_jump - (struct sieve_interpreter *interp, bool jump); + (struct sieve_interpreter *interp, bool jump, bool break_loops); /* * Test results From pigeonhole at rename-it.nl Sun Nov 29 15:17:55 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 16:17:55 +0100 Subject: dovecot-2.2-pigeonhole: testsuite: Fixed the test_fail command t... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/b77785770cd7 changeset: 2150:b77785770cd7 user: Stephan Bosch date: Sun Nov 29 16:17:41 2015 +0100 description: testsuite: Fixed the test_fail command to also work from within a foreverypart loop. diffstat: src/testsuite/cmd-test-fail.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d9bc8186e02e -r b77785770cd7 src/testsuite/cmd-test-fail.c --- a/src/testsuite/cmd-test-fail.c Sun Nov 29 16:17:15 2015 +0100 +++ b/src/testsuite/cmd-test-fail.c Sun Nov 29 16:17:41 2015 +0100 @@ -142,7 +142,7 @@ testsuite_test_fail(reason); - return sieve_interpreter_program_jump(renv->interp, TRUE); + return sieve_interpreter_program_jump(renv->interp, TRUE, TRUE); } From pigeonhole at rename-it.nl Sun Nov 29 17:33:45 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 18:33:45 +0100 Subject: dovecot-2.2-pigeonhole: testsuite: mime: Created tests for some ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/b803686a0731 changeset: 2151:b803686a0731 user: Stephan Bosch date: Sun Nov 29 18:33:40 2015 +0100 description: testsuite: mime: Created tests for some basic foreverypart loops. diffstat: Makefile.am | 1 + tests/extensions/mime/foreverypart.svtest | 140 ++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 0 deletions(-) diffs (155 lines): diff -r b77785770cd7 -r b803686a0731 Makefile.am --- a/Makefile.am Sun Nov 29 16:17:41 2015 +0100 +++ b/Makefile.am Sun Nov 29 18:33:40 2015 +0100 @@ -169,6 +169,7 @@ tests/extensions/mime/errors.svtest \ tests/extensions/mime/execute.svtest \ tests/extensions/mime/content-header.svtest \ + tests/extensions/mime/foreverypart.svtest \ tests/extensions/vnd.dovecot/debug/execute.svtest \ tests/extensions/vnd.dovecot/environment/basic.svtest \ tests/extensions/vnd.dovecot/environment/variables.svtest \ diff -r b77785770cd7 -r b803686a0731 tests/extensions/mime/foreverypart.svtest --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/extensions/mime/foreverypart.svtest Sun Nov 29 18:33:40 2015 +0100 @@ -0,0 +1,140 @@ +require "vnd.dovecot.testsuite"; +require "relational"; +require "foreverypart"; +require "mime"; +require "variables"; + +test_set "message" text: +From: Hendrik +To: Harrie +Date: Sat, 11 Oct 2010 00:31:44 +0200 +Subject: Harrie is een prutser +Content-Type: multipart/mixed; boundary=AA +X-Test: AA + +This is a multi-part message in MIME format. +--AA +Content-Type: multipart/mixed; boundary=BB +X-Test: BB + +This is a multi-part message in MIME format. +--BB +Content-Type: text/plain; charset="us-ascii" +X-Test: CC + +Hello + +--BB +Content-Type: text/plain; charset="us-ascii" +X-Test: DD + +Hello again + +--BB-- +This is the end of MIME multipart. + +--AA +Content-Type: text/plain; charset="us-ascii" +X-Test: EE + +And again + +--AA-- +This is the end of MIME multipart. +. +; + +test "Single loop" { + set "a" "a"; + foreverypart { + set :length "la" "${a}"; + + if string "${a}" "a" { + if not header :mime "X-Test" "AA" { + test_fail "wrong header extracted (${la})"; + } + } elsif string "${a}" "aa" { + if not header :mime "X-Test" "BB" { + test_fail "wrong header extracted (${la})"; + } + } elsif string "${a}" "aaa" { + if not header :mime "X-Test" "CC" { + test_fail "wrong header extracted (${la})"; + } + } elsif string "${a}" "aaaa" { + if not header :mime "X-Test" "DD" { + test_fail "wrong header extracted (${la})"; + } + } elsif string "${a}" "aaaaa" { + if not header :mime "X-Test" "EE" { + test_fail "wrong header extracted (${la})"; + } + } + set "a" "a${a}"; + } +} + +test "Double loop" { + set "a" "a"; + foreverypart { + set :length "la" "${a}"; + + if string "${a}" "a" { + if not header :mime "X-Test" "AA" { + test_fail "wrong header extracted (${la})"; + } + } elsif string "${a}" "aaaaaa" { + if not header :mime "X-Test" "BB" { + test_fail "wrong header extracted (${la})"; + } + } elsif string "${a}" "aaaaaaaaa" { + if not header :mime "X-Test" "CC" { + test_fail "wrong header extracted (${la})"; + } + } elsif string "${a}" "aaaaaaaaaa" { + if not header :mime "X-Test" "DD" { + test_fail "wrong header extracted (${la})"; + } + } elsif string "${a}" "aaaaaaaaaaa" { + if not header :mime "X-Test" "EE" { + test_fail "wrong header extracted (${la})"; + } + } + + set "a" "a${a}"; + + foreverypart { + set :length "la" "${a}"; + + if string "${a}" "aa" { + if not header :mime "X-Test" "BB" { + test_fail "wrong header extracted (${la})"; + } + } elsif string "${a}" "aaa" { + if not header :mime "X-Test" "CC" { + test_fail "wrong header extracted (${la})"; + } + } elsif string "${a}" "aaaa" { + if not header :mime "X-Test" "DD" { + test_fail "wrong header extracted (${la})"; + } + } elsif string "${a}" "aaaaa" { + if not header :mime "X-Test" "EE" { + test_fail "wrong header extracted (${la})"; + } + } elsif string "${a}" "aaaaaaa" { + if not header :mime "X-Test" "CC" { + test_fail "wrong header extracted (${la})"; + } + } elsif string "${a}" "aaaaaaaa" { + if not header :mime "X-Test" "DD" { + test_fail "wrong header extracted (${la})"; + } + } + set "a" "a${a}"; + } + } +} + + + From pigeonhole at rename-it.nl Sun Nov 29 19:01:14 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 20:01:14 +0100 Subject: dovecot-2.2-pigeonhole: testsuite: mime extension: Added iCalend... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/e4f181199457 changeset: 2153:e4f181199457 user: Stephan Bosch date: Sun Nov 29 20:01:09 2015 +0100 description: testsuite: mime extension: Added iCalendar example test case. diffstat: Makefile.am | 1 + tests/extensions/mime/calendar-example.svtest | 129 ++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 0 deletions(-) diffs (144 lines): diff -r 737323455d25 -r e4f181199457 Makefile.am --- a/Makefile.am Sun Nov 29 20:00:37 2015 +0100 +++ b/Makefile.am Sun Nov 29 20:01:09 2015 +0100 @@ -170,6 +170,7 @@ tests/extensions/mime/execute.svtest \ tests/extensions/mime/content-header.svtest \ tests/extensions/mime/foreverypart.svtest \ + tests/extensions/mime/calendar-example.svtest \ tests/extensions/vnd.dovecot/debug/execute.svtest \ tests/extensions/vnd.dovecot/environment/basic.svtest \ tests/extensions/vnd.dovecot/environment/variables.svtest \ diff -r 737323455d25 -r e4f181199457 tests/extensions/mime/calendar-example.svtest --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/extensions/mime/calendar-example.svtest Sun Nov 29 20:01:09 2015 +0100 @@ -0,0 +1,129 @@ +require "vnd.dovecot.testsuite"; +require "mime"; +require "foreverypart"; +require "editheader"; +require "relational"; +require "variables"; + +# Example from RFC 6047, Section 2.5: +test_set "message" text: +From: user1 at example.com +To: user2 at example.com +Subject: Phone Conference +Mime-Version: 1.0 +Date: Wed, 07 May 2008 21:30:25 +0400 +Message-ID: <4821E731.5040506 at laptop1.example.com> +Content-Type: text/calendar; method=REQUEST; charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + +BEGIN:VCALENDAR +PRODID:-//Example/ExampleCalendarClient//EN +METHOD:REQUEST +VERSION:2.0 +BEGIN:VEVENT +ORGANIZER:mailto:user1 at example.com +ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:user1 at example.com +ATTENDEE;RSVP=YES;CUTYPE=INDIVIDUAL:mailto:user2 at example.com +DTSTAMP:20080507T170000Z +DTSTART:20080701T160000Z +DTEND:20080701T163000Z +SUMMARY:Phone call to discuss your last visit +DESCRIPTION:=D1=82=D1=8B =D0=BA=D0=B0=D0=BA - =D0=B4=D0=BE=D0= + =B2=D0=BE=D0=BB=D0=B5=D0=BD =D0=BF=D0=BE=D0=B5=D0=B7=D0=B4=D0=BA=D0 + =BE=D0=B9? +UID:calsvr.example.com-8739701987387998 +SEQUENCE:0 +STATUS:TENTATIVE +END:VEVENT +END:VCALENDAR +. +; + +test "Calendar only" { + foreverypart { + if allof( + header :mime :count "eq" "Content-Type" "1", + header :mime :contenttype "Content-Type" "text/calendar", + header :mime :param "method" :matches "Content-Type" "*", + header :mime :param "charset" :is "Content-Type" "UTF-8" ) { + addheader "X-ICAL" "${1}"; + break; + } + } + + if not header "x-ical" "request" { + test_fail "Failed to parse message correctly"; + } +} + +# Modified example +test_set "message" text: +From: user1 at example.com +To: user2 at example.com +Subject: Phone Conference +Mime-Version: 1.0 +Date: Wed, 07 May 2008 21:30:25 +0400 +Message-ID: <4821E731.5040506 at laptop1.example.com> +Content-Type: multipart/mixed; boundary=AA + +This is a multi-part message in MIME format. + +--AA +Content-Type: text/plain + +Hello, + +I'd like to discuss your last visit. A tentative meeting schedule is +attached. + +Regards, + +User1 + +--AA +Content-Type: text/calendar; method=REQUEST; charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + +BEGIN:VCALENDAR +PRODID:-//Example/ExampleCalendarClient//EN +METHOD:REQUEST +VERSION:2.0 +BEGIN:VEVENT +ORGANIZER:mailto:user1 at example.com +ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:user1 at example.com +ATTENDEE;RSVP=YES;CUTYPE=INDIVIDUAL:mailto:user2 at example.com +DTSTAMP:20080507T170000Z +DTSTART:20080701T160000Z +DTEND:20080701T163000Z +SUMMARY:Phone call to discuss your last visit +DESCRIPTION:=D1=82=D1=8B =D0=BA=D0=B0=D0=BA - =D0=B4=D0=BE=D0= + =B2=D0=BE=D0=BB=D0=B5=D0=BD =D0=BF=D0=BE=D0=B5=D0=B7=D0=B4=D0=BA=D0 + =BE=D0=B9? +UID:calsvr.example.com-8739701987387998 +SEQUENCE:0 +STATUS:TENTATIVE +END:VEVENT +END:VCALENDAR + +--AA-- +. +; + +test "Multipart message" { + foreverypart { + if allof( + header :mime :count "eq" "Content-Type" "1", + header :mime :contenttype "Content-Type" "text/calendar", + header :mime :param "method" :matches "Content-Type" "*", + header :mime :param "charset" :is "Content-Type" "UTF-8" ) { + addheader "X-ICAL" "${1}"; + break; + } + } + + if not header "x-ical" "request" { + test_fail "Failed to parse message correctly"; + } +} + + From pigeonhole at rename-it.nl Sun Nov 29 19:01:14 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 20:01:14 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: mime extension: Improved trac... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/737323455d25 changeset: 2152:737323455d25 user: Stephan Bosch date: Sun Nov 29 20:00:37 2015 +0100 description: lib-sieve: mime extension: Improved trace output. diffstat: src/lib-sieve/plugins/mime/cmd-foreverypart.c | 12 +++- src/lib-sieve/plugins/mime/tag-mime.c | 73 +++++++++++++++++++++++--- src/lib-sieve/sieve-message.c | 9 ++- 3 files changed, 79 insertions(+), 15 deletions(-) diffs (205 lines): diff -r b803686a0731 -r 737323455d25 src/lib-sieve/plugins/mime/cmd-foreverypart.c --- a/src/lib-sieve/plugins/mime/cmd-foreverypart.c Sun Nov 29 18:33:40 2015 +0100 +++ b/src/lib-sieve/plugins/mime/cmd-foreverypart.c Sun Nov 29 20:00:37 2015 +0100 @@ -291,7 +291,8 @@ * Perform operation */ - sieve_runtime_trace(renv, SIEVE_TRLVL_ACTIONS, "foreverypart loop begin"); + sieve_runtime_trace(renv, SIEVE_TRLVL_COMMANDS, + "foreverypart loop begin"); sieve_runtime_trace_descend(renv); sfploop = ext_foreverypart_runtime_loop_get_current(renv); @@ -347,7 +348,7 @@ */ sieve_runtime_trace(renv, - SIEVE_TRLVL_ACTIONS, "foreverypart loop end"); + SIEVE_TRLVL_COMMANDS, "foreverypart loop end"); sieve_runtime_trace_descend(renv); loop = sieve_interpreter_loop_get @@ -361,9 +362,14 @@ sieve_interpreter_loop_get_context(loop); i_assert(fploop->part != NULL); fploop->part = sieve_message_part_iter_next(&fploop->part_iter); - if ( fploop->part == NULL ) + if ( fploop->part == NULL ) { + sieve_runtime_trace(renv, + SIEVE_TRLVL_COMMANDS, "no more message parts"); return sieve_interpreter_loop_break(renv->interp, loop); + } + sieve_runtime_trace(renv, + SIEVE_TRLVL_COMMANDS, "switched to next message part"); return sieve_interpreter_loop_next(renv->interp, loop, loop_begin); } diff -r b803686a0731 -r 737323455d25 src/lib-sieve/plugins/mime/tag-mime.c --- a/src/lib-sieve/plugins/mime/tag-mime.c Sun Nov 29 18:33:40 2015 +0100 +++ b/src/lib-sieve/plugins/mime/tag-mime.c Sun Nov 29 20:00:37 2015 +0100 @@ -357,16 +357,20 @@ static string_t * content_type_param_next(struct content_header_stringlist *strlist) -{ +{ + const struct sieve_runtime_env *renv = strlist->strlist.runenv; const char *const *values = strlist->param_values; + bool trace = strlist->strlist.trace; i_assert( strlist->params != NULL ); + /* Iterate over all parsed parameter values */ for ( ; *values != NULL; values += 2 ) { const char *const *params = strlist->params; const char *name = values[0], *value = values[1]; size_t nlen = strlen(name); + /* Iterate over all interesting parameter names */ for ( ; *params != NULL; params++ ) { size_t plen = strlen(*params); @@ -376,15 +380,29 @@ if ( plen == nlen ) { if ( strcasecmp(name, *params) == 0 ) { + /* Return raw value */ + if ( trace ) { + sieve_runtime_trace(renv, 0, + "found mime parameter `%s' in mime header", + *params); + } + strlist->param_values = values + 2; return t_str_new_const(value, strlen(value)); } } else { + if ( trace ) { + sieve_runtime_trace(renv, 0, + "found encoded parameter `%s' in mime header", + *params); + } + if ( strncasecmp(name, *params, plen) == 0 ) { string_t *result = NULL; strlist->param_values = values + 2; + /* Decode value first */ // FIXME: transcode charset value = strchr(value, '\''); if (value != NULL) @@ -409,6 +427,8 @@ content_header_parse(struct content_header_stringlist *strlist, const char *hdr_name, string_t *str) { + const struct sieve_runtime_env *renv = strlist->strlist.runenv; + bool trace = strlist->strlist.trace; struct rfc822_parser_context parser; const char *type, *p; bool is_ctype = FALSE; @@ -419,8 +439,13 @@ if ( strcasecmp(hdr_name, "content-type") == 0 ) is_ctype = TRUE; - else if ( strcasecmp(hdr_name, "content-disposition") != 0 ) + else if ( strcasecmp(hdr_name, "content-disposition") != 0 ) { + if ( trace ) { + sieve_runtime_trace(renv, 0, + "non-mime header yields empty string"); + } return t_str_new(0); + } /* Initialize parsing */ rfc822_parser_init(&parser, str_data(str), str_len(str), NULL); @@ -450,6 +475,7 @@ if ( strlist->option == EXT_MIME_OPTION_PARAM ) { string_t *param_val; + /* MIME parameter */ i_assert( strlist->params != NULL ); // FIXME: not very nice when multiple parameters in the same header @@ -461,17 +487,44 @@ if ( param_val != NULL ) content = param_val; } else { + /* Get :type/:subtype:/:contenttype value */ type = str_c(content); - if ( (p=strchr(type, '/')) == NULL ) { - i_assert( !is_ctype ); - if ( strlist->option == EXT_MIME_OPTION_SUBTYPE ) + p = strchr(type, '/'); + switch ( strlist->option ) { + case EXT_MIME_OPTION_TYPE: + if ( trace ) { + sieve_runtime_trace(renv, 0, + "extracted MIME type"); + } + if ( p != NULL ) { + i_assert( is_ctype ); + str_truncate(content, (p - type)); + } + break; + case EXT_MIME_OPTION_SUBTYPE: + if ( p == NULL ) { + i_assert( !is_ctype ); + if ( trace ) { + sieve_runtime_trace(renv, 0, + "no MIME sub-type for content-disposition"); + } str_truncate(content, 0); - } else { + break; + } + i_assert( is_ctype ); - if ( strlist->option == EXT_MIME_OPTION_TYPE ) - str_truncate(content, (p - type)); - else if ( strlist->option == EXT_MIME_OPTION_SUBTYPE ) - str_delete(content, 0, (p - type) + 1); + if ( trace ) { + sieve_runtime_trace(renv, 0, + "extracted MIME sub-type"); + } + str_delete(content, 0, (p - type) + 1); + break; + case EXT_MIME_OPTION_CONTENTTYPE: + sieve_runtime_trace(renv, 0, + "extracted full MIME contenttype"); + break; + default: + break; } } diff -r b803686a0731 -r 737323455d25 src/lib-sieve/sieve-message.c --- a/src/lib-sieve/sieve-message.c Sun Nov 29 18:33:40 2015 +0100 +++ b/src/lib-sieve/sieve-message.c Sun Nov 29 20:00:37 2015 +0100 @@ -1722,8 +1722,13 @@ (&mpart->headers, &hdrlist->headers_count); hdrlist->headers_index = 0; } - if ( hdrlist->headers_count > 0 ) + if ( hdrlist->headers_count > 0 ) { + if ( _hdrlist->strlist.trace ) { + sieve_runtime_trace(renv, 0, + "moving to next message part"); + } break; + } } /* Read next header name from source list */ @@ -1735,7 +1740,7 @@ if ( _hdrlist->strlist.trace ) { sieve_runtime_trace(renv, 0, - "extracting `%s' headers from message", + "extracting `%s' headers from message part", str_sanitize(str_c(hdr_item), 80)); } From pigeonhole at rename-it.nl Sun Nov 29 20:33:39 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 21:33:39 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Fixed memory leak in code dum... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/168b6eb4671e changeset: 2154:168b6eb4671e user: Stephan Bosch date: Sun Nov 29 21:33:33 2015 +0100 description: lib-sieve: Fixed memory leak in code dumper. The free handler of code dumper extensions was never executed. diffstat: src/lib-sieve/plugins/variables/ext-variables-dump.c | 6 ++++-- src/lib-sieve/sieve-code-dumper.c | 19 +++++++++++++++---- src/lib-sieve/sieve-code-dumper.h | 2 +- 3 files changed, 20 insertions(+), 7 deletions(-) diffs (66 lines): diff -r e4f181199457 -r 168b6eb4671e src/lib-sieve/plugins/variables/ext-variables-dump.c --- a/src/lib-sieve/plugins/variables/ext-variables-dump.c Sun Nov 29 20:01:09 2015 +0100 +++ b/src/lib-sieve/plugins/variables/ext-variables-dump.c Sun Nov 29 21:33:33 2015 +0100 @@ -19,7 +19,8 @@ static void ext_variables_code_dumper_free (struct sieve_code_dumper *dumper, void *context); -const struct sieve_code_dumper_extension variables_dump_extension = { +static const struct sieve_code_dumper_extension +variables_dump_extension = { &variables_extension, ext_variables_code_dumper_free }; @@ -62,7 +63,8 @@ p_array_init(&dctx->ext_scopes, pool, sieve_extensions_get_count(this_ext->svinst)); - sieve_dump_extension_set_context(dumper, this_ext, dctx); + sieve_dump_extension_register + (dumper, this_ext, &variables_dump_extension, dctx); } return dctx; diff -r e4f181199457 -r 168b6eb4671e src/lib-sieve/sieve-code-dumper.c --- a/src/lib-sieve/sieve-code-dumper.c Sun Nov 29 20:01:09 2015 +0100 +++ b/src/lib-sieve/sieve-code-dumper.c Sun Nov 29 21:33:33 2015 +0100 @@ -67,12 +67,23 @@ return cdumper; } -void sieve_code_dumper_free(struct sieve_code_dumper **cdumper) +void sieve_code_dumper_free(struct sieve_code_dumper **_cdumper) { - sieve_binary_debug_reader_deinit(&(*cdumper)->dreader); + struct sieve_code_dumper *cdumper = *_cdumper; + const struct sieve_code_dumper_extension_reg *eregs; + unsigned int count, i; - pool_unref(&((*cdumper)->pool)); - *cdumper = NULL; + sieve_binary_debug_reader_deinit(&cdumper->dreader); + + /* Signal registered extensions that the dumper is being destroyed */ + eregs = array_get(&cdumper->extensions, &count); + for ( i = 0; i < count; i++ ) { + if ( eregs[i].cdmpext != NULL && eregs[i].cdmpext->free != NULL ) + eregs[i].cdmpext->free(cdumper, eregs[i].context); + } + + pool_unref(&cdumper->pool); + *_cdumper = NULL; } pool_t sieve_code_dumper_pool(struct sieve_code_dumper *cdumper) diff -r e4f181199457 -r 168b6eb4671e src/lib-sieve/sieve-code-dumper.h --- a/src/lib-sieve/sieve-code-dumper.h Sun Nov 29 20:01:09 2015 +0100 +++ b/src/lib-sieve/sieve-code-dumper.h Sun Nov 29 21:33:33 2015 +0100 @@ -11,7 +11,7 @@ struct sieve_code_dumper *sieve_code_dumper_create (struct sieve_dumptime_env *denv); void sieve_code_dumper_free - (struct sieve_code_dumper **dumper); + (struct sieve_code_dumper **_dumper); pool_t sieve_code_dumper_pool (struct sieve_code_dumper *dumper); From pigeonhole at rename-it.nl Sun Nov 29 20:47:19 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 29 Nov 2015 21:47:19 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: variables extension: Fixed ha... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/7e9392cc0ed5 changeset: 2155:7e9392cc0ed5 user: Stephan Bosch date: Sun Nov 29 21:47:14 2015 +0100 description: lib-sieve: variables extension: Fixed handling of empty string by the `:length' set modifier. An empty string yielded an empty string rather than "0". diffstat: src/lib-sieve/plugins/enotify/vmodf-encodeurl.c | 5 ++ src/lib-sieve/plugins/variables/cmd-set.c | 32 +++++++------- src/lib-sieve/plugins/variables/ext-variables-modifiers.c | 25 +++++++++++ tests/extensions/variables/modifiers.svtest | 10 ++++- 4 files changed, 54 insertions(+), 18 deletions(-) diffs (143 lines): diff -r 168b6eb4671e -r 7e9392cc0ed5 src/lib-sieve/plugins/enotify/vmodf-encodeurl.c --- a/src/lib-sieve/plugins/enotify/vmodf-encodeurl.c Sun Nov 29 21:33:33 2015 +0100 +++ b/src/lib-sieve/plugins/enotify/vmodf-encodeurl.c Sun Nov 29 21:47:14 2015 +0100 @@ -67,6 +67,11 @@ unsigned int i; const unsigned char *c; + if ( str_len(in) == 0 ) { + *result = in; + return TRUE; + } + *result = t_str_new(2*str_len(in)); c = str_data(in); diff -r 168b6eb4671e -r 7e9392cc0ed5 src/lib-sieve/plugins/variables/cmd-set.c --- a/src/lib-sieve/plugins/variables/cmd-set.c Sun Nov 29 21:33:33 2015 +0100 +++ b/src/lib-sieve/plugins/variables/cmd-set.c Sun Nov 29 21:47:14 2015 +0100 @@ -326,26 +326,24 @@ break; } - if ( str_len(value) > 0 ) { - if ( modf.def != NULL && modf.def->modify != NULL ) { - if ( !modf.def->modify(value, &new_value) ) { - value = NULL; - ret = SIEVE_EXEC_FAILURE; - break; - } + if ( modf.def != NULL && modf.def->modify != NULL ) { + if ( !modf.def->modify(value, &new_value) ) { + value = NULL; + ret = SIEVE_EXEC_FAILURE; + break; + } - sieve_runtime_trace_here - (renv, SIEVE_TRLVL_COMMANDS, "modify :%s \"%s\" => \"%s\"", - sieve_variables_modifier_name(&modf), str_c(value), str_c(new_value)); + sieve_runtime_trace_here + (renv, SIEVE_TRLVL_COMMANDS, "modify :%s \"%s\" => \"%s\"", + sieve_variables_modifier_name(&modf), str_c(value), str_c(new_value)); - value = new_value; - if ( value == NULL ) - break; + value = new_value; + if ( value == NULL ) + break; - /* Hold value within limits */ - if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) - str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); - } + /* Hold value within limits */ + if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) + str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); } } diff -r 168b6eb4671e -r 7e9392cc0ed5 src/lib-sieve/plugins/variables/ext-variables-modifiers.c --- a/src/lib-sieve/plugins/variables/ext-variables-modifiers.c Sun Nov 29 21:33:33 2015 +0100 +++ b/src/lib-sieve/plugins/variables/ext-variables-modifiers.c Sun Nov 29 21:47:14 2015 +0100 @@ -190,6 +190,11 @@ { char *content; + if ( str_len(in) == 0 ) { + *result = in; + return TRUE; + } + *result = t_str_new(str_len(in)); str_append_str(*result, in); @@ -203,6 +208,11 @@ { char *content; + if ( str_len(in) == 0 ) { + *result = in; + return TRUE; + } + *result = t_str_new(str_len(in)); str_append_str(*result, in); @@ -216,6 +226,11 @@ { char *content; + if ( str_len(in) == 0 ) { + *result = in; + return TRUE; + } + *result = t_str_new(str_len(in)); str_append_str(*result, in); @@ -229,6 +244,11 @@ { char *content; + if ( str_len(in) == 0 ) { + *result = in; + return TRUE; + } + *result = t_str_new(str_len(in)); str_append_str(*result, in); @@ -251,6 +271,11 @@ unsigned int i; const char *content; + if ( str_len(in) == 0 ) { + *result = in; + return TRUE; + } + *result = t_str_new(str_len(in) * 2); content = (const char *) str_data(in); diff -r 168b6eb4671e -r 7e9392cc0ed5 tests/extensions/variables/modifiers.svtest --- a/tests/extensions/variables/modifiers.svtest Sun Nov 29 21:33:33 2015 +0100 +++ b/tests/extensions/variables/modifiers.svtest Sun Nov 29 21:47:14 2015 +0100 @@ -70,7 +70,15 @@ } } -test "Modifier :length" { +test "Modifier :length (empty)" { + set :length "test" ""; + + if not string :is "${test}" "0" { + test_fail "modified variable assignment failed"; + } +} + +test "Modifier :length (simple)" { set :length "test" "VaLuE"; if not string :is "${test}" "5" { From dovecot at dovecot.org Mon Nov 30 10:22:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 30 Nov 2015 10:22:34 +0000 Subject: dovecot-2.2: doveadm fetch: Added date.sent/received/saved.unixtime Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9a15b989cc41 changeset: 19434:9a15b989cc41 user: Timo Sirainen date: Mon Nov 30 12:22:22 2015 +0200 description: doveadm fetch: Added date.sent/received/saved.unixtime diffstat: src/doveadm/doveadm-mail-fetch.c | 35 +++++++++++++++++++++++++++++++++++ 1 files changed, 35 insertions(+), 0 deletions(-) diffs (52 lines): diff -r 922de4bc4777 -r 9a15b989cc41 src/doveadm/doveadm-mail-fetch.c --- a/src/doveadm/doveadm-mail-fetch.c Sat Nov 28 23:50:14 2015 +0200 +++ b/src/doveadm/doveadm-mail-fetch.c Mon Nov 30 12:22:22 2015 +0200 @@ -327,6 +327,38 @@ return 0; } +static int fetch_date_received_unixtime(struct fetch_cmd_context *ctx) +{ + time_t t; + + if (mail_get_received_date(ctx->mail, &t) < 0) + return -1; + doveadm_print(dec2str(t)); + return 0; +} + +static int fetch_date_sent_unixtime(struct fetch_cmd_context *ctx) +{ + time_t t; + int tz; + + if (mail_get_date(ctx->mail, &t, &tz) < 0) + return -1; + + doveadm_print(dec2str(t)); + return 0; +} + +static int fetch_date_saved_unixtime(struct fetch_cmd_context *ctx) +{ + time_t t; + + if (mail_get_save_date(ctx->mail, &t) < 0) + return -1; + doveadm_print(dec2str(t)); + return 0; +} + static int fetch_imap_envelope(struct fetch_cmd_context *ctx) { const char *value; @@ -417,6 +449,9 @@ { "date.received", MAIL_FETCH_RECEIVED_DATE, fetch_date_received }, { "date.sent", MAIL_FETCH_DATE, fetch_date_sent }, { "date.saved", MAIL_FETCH_SAVE_DATE, fetch_date_saved }, + { "date.received.unixtime", MAIL_FETCH_RECEIVED_DATE, fetch_date_received_unixtime }, + { "date.sent.unixtime", MAIL_FETCH_DATE, fetch_date_sent_unixtime }, + { "date.saved.unixtime", MAIL_FETCH_SAVE_DATE, fetch_date_saved_unixtime }, { "imap.envelope", MAIL_FETCH_IMAP_ENVELOPE, fetch_imap_envelope }, { "imap.body", MAIL_FETCH_IMAP_BODY, fetch_imap_body }, { "imap.bodystructure", MAIL_FETCH_IMAP_BODYSTRUCTURE, fetch_imap_bodystructure }, From dovecot at dovecot.org Mon Nov 30 11:15:58 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 30 Nov 2015 11:15:58 +0000 Subject: dovecot-2.2: lib-mail: rfc822_parse_content_param() was unescapi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9eab653f5b94 changeset: 19435:9eab653f5b94 user: Timo Sirainen date: Mon Nov 30 13:14:29 2015 +0200 description: lib-mail: rfc822_parse_content_param() was unescaping already unescaped parameters This caused all Content-* parameter parsing to be unescaped once too many times, resulting in somewhat broken BODY and BODYSTRUCTURE replies if any <\> characters were used. Also MIME boundaries were parsed in case <\> was used in them, but this probably didn't practically happen. diffstat: src/lib-mail/Makefile.am | 7 +++- src/lib-mail/rfc822-parser.c | 1 - src/lib-mail/test-rfc822-parser.c | 74 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) diffs (113 lines): diff -r 9a15b989cc41 -r 9eab653f5b94 src/lib-mail/Makefile.am --- a/src/lib-mail/Makefile.am Mon Nov 30 12:22:22 2015 +0200 +++ b/src/lib-mail/Makefile.am Mon Nov 30 13:14:29 2015 +0200 @@ -95,7 +95,8 @@ test-ostream-dot \ test-qp-decoder \ test-quoted-printable \ - test-rfc2231-parser + test-rfc2231-parser \ + test-rfc822-parser noinst_PROGRAMS = $(test_programs) @@ -196,6 +197,10 @@ test_rfc2231_parser_LDADD = rfc2231-parser.lo rfc822-parser.lo $(test_libs) test_rfc2231_parser_DEPENDENCIES = $(test_deps) +test_rfc822_parser_SOURCES = test-rfc822-parser.c +test_rfc822_parser_LDADD = rfc822-parser.lo $(test_libs) +test_rfc822_parser_DEPENDENCIES = $(test_deps) + check: check-am check-test check-test: all-am for bin in $(test_programs); do \ diff -r 9a15b989cc41 -r 9eab653f5b94 src/lib-mail/rfc822-parser.c --- a/src/lib-mail/rfc822-parser.c Mon Nov 30 12:22:22 2015 +0200 +++ b/src/lib-mail/rfc822-parser.c Mon Nov 30 13:14:29 2015 +0200 @@ -409,7 +409,6 @@ /* broken / no value */ } else if (*ctx->data == '"') { ret = rfc822_parse_quoted_string(ctx, tmp); - (void)str_unescape(str_c_modifiable(tmp) + value_pos); } else if (ctx->data != ctx->end && *ctx->data == '=') { /* workaround for broken input: name==?utf-8?b?...?= */ diff -r 9a15b989cc41 -r 9eab653f5b94 src/lib-mail/test-rfc822-parser.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-mail/test-rfc822-parser.c Mon Nov 30 13:14:29 2015 +0200 @@ -0,0 +1,74 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "rfc822-parser.h" +#include "test-common.h" + +static void test_rfc822_parse_quoted_string(void) +{ + const struct { + const char *input, *output; + int ret; + } tests[] = { + { "\"", "", -1 }, + { "\"\"", "", 0 }, + { "\"foo\"", "foo", 0 }, + { "\"\"foo", "", 1 }, + { "\"\"\"", "", 1 }, + { "\"\\\"\"", "\"", 0 }, + { "\"\\\\\"", "\\", 0 }, + { "\"\\\\foo\\\\foo\\\\\"", "\\foo\\foo\\", 0 } + }; + struct rfc822_parser_context parser; + string_t *str = t_str_new(64); + unsigned int i = 0; + + test_begin("rfc822 parse quoted string"); + for (i = 0; i < N_ELEMENTS(tests); i++) { + rfc822_parser_init(&parser, (const void *)tests[i].input, + strlen(tests[i].input), NULL); + test_assert_idx(rfc822_parse_quoted_string(&parser, str) == tests[i].ret, i); + test_assert_idx(tests[i].ret < 0 || + strcmp(tests[i].output, str_c(str)) == 0, i); + str_truncate(str, 0); + } + test_end(); +} + +static void test_rfc822_parse_content_param(void) +{ + const char *input = + "; key1=value1#$!%&'*+-.^_`{|}~" + "; key2=\" \\\"(),/:;<=>?@[\\\\]\""; + const struct { + const char *key, *value; + } output[] = { + { "key1", "value1#$!%&'*+-.^_`{|}~" }, + { "key2", " \"(),/:;<=>?@[\\]" } + }; + struct rfc822_parser_context parser; + const char *key, *value; + unsigned int i = 0; + int ret; + + test_begin("rfc822 parse content param"); + rfc822_parser_init(&parser, (const void *)input, strlen(input), NULL); + while ((ret = rfc822_parse_content_param(&parser, &key, &value)) > 0) { + test_assert_idx(strcmp(output[i].key, key) == 0, i); + test_assert_idx(strcmp(output[i].value, value) == 0, i); + i++; + } + test_assert(ret == 0); + test_end(); +} + +int main(void) +{ + static void (*test_functions[])(void) = { + test_rfc822_parse_quoted_string, + test_rfc822_parse_content_param, + NULL + }; + return test_run(test_functions); +} From dovecot at dovecot.org Mon Nov 30 11:15:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 30 Nov 2015 11:15:59 +0000 Subject: dovecot-2.2: lib-imap: Added unit tests for imap_append_[an]stri... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/326711633532 changeset: 19436:326711633532 user: Timo Sirainen date: Mon Nov 30 13:15:44 2015 +0200 description: lib-imap: Added unit tests for imap_append_[an]string() diffstat: src/lib-imap/test-imap-quote.c | 66 ++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 66 insertions(+), 0 deletions(-) diffs (80 lines): diff -r 9eab653f5b94 -r 326711633532 src/lib-imap/test-imap-quote.c --- a/src/lib-imap/test-imap-quote.c Mon Nov 30 13:14:29 2015 +0200 +++ b/src/lib-imap/test-imap-quote.c Mon Nov 30 13:15:44 2015 +0200 @@ -37,10 +37,76 @@ test_end(); } +static void test_imap_append_astring(void) +{ + static struct { + const char *input, *output; + } tests[] = { + { "", "\"\"" }, + { "NIL", "\"NIL\"" }, + { "niL", "\"niL\"" }, + { "ni", "ni" }, + { "\\", "\"\\\\\"" }, + { "\\\\", "\"\\\\\\\\\"" }, + { "\\\\\\", "\"\\\\\\\\\\\\\"" }, + { "\\\\\\\\", "\"\\\\\\\\\\\\\\\\\"" }, + { "\\\\\\\\\\", "{5}\r\n\\\\\\\\\\" }, + { "\\\\\\\\\\\\", "{6}\r\n\\\\\\\\\\\\" }, + { "\"", "\"\\\"\"" }, + { "\"\"", "\"\\\"\\\"\"" }, + { "\"\"\"", "\"\\\"\\\"\\\"\"" }, + { "\"\"\"\"", "\"\\\"\\\"\\\"\\\"\"" }, + { "\"\"\"\"\"", "{5}\r\n\"\"\"\"\"" }, + { "\"\"\"\"\"\"", "{6}\r\n\"\"\"\"\"\"" }, + { "\r", "{1}\r\n\r" }, + { "\n", "{1}\r\n\n" }, + { "\r\n", "{2}\r\n\r\n" }, + { "\x7f", "\"\x7f\"" }, + { "\x80", "{1}\r\n\x80" }, + { "\xff", "{1}\r\n\xff" }, + }; + string_t *str = t_str_new(128); + unsigned int i; + + test_begin("test_imap_append_astring()"); + + for (i = 0; i < N_ELEMENTS(tests); i++) { + str_truncate(str, 0); + imap_append_astring(str, tests[i].input); + test_assert_idx(strcmp(tests[i].output, str_c(str)) == 0, i); + } + test_end(); +} + +static void test_imap_append_nstring(void) +{ + static struct { + const char *input, *output; + } tests[] = { + { "", "\"\"" }, + { NULL, "NIL" }, + { "NIL", "\"NIL\"" }, + { "ni", "\"ni\"" } + }; + string_t *str = t_str_new(128); + unsigned int i; + + test_begin("test_imap_append_nstring()"); + + for (i = 0; i < N_ELEMENTS(tests); i++) { + str_truncate(str, 0); + imap_append_nstring(str, tests[i].input); + test_assert_idx(strcmp(tests[i].output, str_c(str)) == 0, i); + } + test_end(); +} + int main(void) { static void (*test_functions[])(void) = { test_imap_append_string_for_humans, + test_imap_append_astring, + test_imap_append_nstring, NULL }; return test_run(test_functions); From dovecot at dovecot.org Mon Nov 30 19:34:47 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 30 Nov 2015 19:34:47 +0000 Subject: dovecot-2.2: director: Fixed "doveadm director status user@domai... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ac30316bed82 changeset: 19437:ac30316bed82 user: Timo Sirainen date: Mon Nov 30 21:32:26 2015 +0200 description: director: Fixed "doveadm director status user at domain" not to hang due to missing LF. diffstat: src/director/doveadm-connection.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 326711633532 -r ac30316bed82 src/director/doveadm-connection.c --- a/src/director/doveadm-connection.c Mon Nov 30 13:15:44 2015 +0200 +++ b/src/director/doveadm-connection.c Mon Nov 30 21:32:26 2015 +0200 @@ -510,7 +510,7 @@ host = mail_host_get_by_hash(conn->dir->orig_config_hosts, username_hash, tag); if (host == NULL) - str_append(str, "\t"); + str_append(str, "\t\n"); else str_printfa(str, "\t%s\n", net_ip2addr(&host->ip)); o_stream_nsend(conn->output, str_data(str), str_len(str)); From dovecot at dovecot.org Mon Nov 30 19:40:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 30 Nov 2015 19:40:09 +0000 Subject: dovecot-2.2: director: Don't treat empty hostname the same as ha... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f643e85f42ec changeset: 19438:f643e85f42ec user: Timo Sirainen date: Mon Nov 30 21:36:51 2015 +0200 description: director: Don't treat empty hostname the same as having it. This could have caused "host not given" errors if an empty "host=" was sent to login process, even though hostip was also sent. Fixed this in two places, although either one should have been enough. diffstat: src/director/login-connection.c | 2 +- src/director/mail-host.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diffs (25 lines): diff -r ac30316bed82 -r f643e85f42ec src/director/login-connection.c --- a/src/director/login-connection.c Mon Nov 30 21:32:26 2015 +0200 +++ b/src/director/login-connection.c Mon Nov 30 21:36:51 2015 +0200 @@ -153,7 +153,7 @@ secs = dir->set->director_user_expire / 2; str_printfa(str, "%s\tproxy_refresh=%u\t", request->line, secs); - if (hostname == NULL) + if (hostname == NULL || hostname[0] == '\0') str_printfa(str, "host=%s", net_ip2addr(ip)); else { str_printfa(str, "host=%s\thostip=%s", diff -r ac30316bed82 -r f643e85f42ec src/director/mail-host.c --- a/src/director/mail-host.c Mon Nov 30 21:32:26 2015 +0200 +++ b/src/director/mail-host.c Mon Nov 30 21:36:51 2015 +0200 @@ -211,7 +211,8 @@ struct mail_host *host; host = mail_host_add_ip(list, ip, tag_name); - host->hostname = i_strdup(hostname); + if (hostname != NULL && hostname[0] != '\0') + host->hostname = i_strdup(hostname); return host; } From dovecot at dovecot.org Mon Nov 30 19:40:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 30 Nov 2015 19:40:10 +0000 Subject: dovecot-2.2: director: Fixed recent tag reimplementation to actu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a36239ce0f78 changeset: 19439:a36239ce0f78 user: Timo Sirainen date: Mon Nov 30 21:39:07 2015 +0200 description: director: Fixed recent tag reimplementation to actually work. We still just created one large vhosts pool for all tags containing all the hosts. diffstat: src/director/mail-host.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r f643e85f42ec -r a36239ce0f78 src/director/mail-host.c --- a/src/director/mail-host.c Mon Nov 30 21:36:51 2015 +0200 +++ b/src/director/mail-host.c Mon Nov 30 21:39:07 2015 +0200 @@ -67,7 +67,7 @@ char num_str[MAX_INT_STRLEN]; unsigned int i, j; - if (host->down) + if (host->down || host->tag != tag) return; ip_str = net_ip2addr(&host->ip); @@ -110,7 +110,7 @@ /* rebuild vhosts */ array_clear(&tag->vhosts); array_foreach(&list->hosts, hostp) { - if ((*hostp)->down) + if ((*hostp)->down || (*hostp)->tag != tag) continue; for (i = 0; i < (*hostp)->vhost_count; i++) { vhost = array_append_space(&tag->vhosts); From dovecot at dovecot.org Mon Nov 30 19:40:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 30 Nov 2015 19:40:10 +0000 Subject: dovecot-2.2: director: Include tag also in the hosts_hash. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1c826c476dbb changeset: 19440:1c826c476dbb user: Timo Sirainen date: Mon Nov 30 21:39:34 2015 +0200 description: director: Include tag also in the hosts_hash. diffstat: src/director/mail-host.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r a36239ce0f78 -r 1c826c476dbb src/director/mail-host.c --- a/src/director/mail-host.c Mon Nov 30 21:39:07 2015 +0200 +++ b/src/director/mail-host.c Mon Nov 30 21:39:34 2015 +0200 @@ -148,6 +148,8 @@ num = net_ip_hash(&(*hostp)->ip); list->hosts_hash = crc32_data_more(list->hosts_hash, &num, sizeof(num)); + list->hosts_hash = crc32_str_more(list->hosts_hash, + (*hostp)->tag->name); } } From dovecot at dovecot.org Mon Nov 30 19:40:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 30 Nov 2015 19:40:11 +0000 Subject: dovecot-2.2: director: Minor code cleanup - removed unnecessary ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b0e2a14d5a40 changeset: 19441:b0e2a14d5a40 user: Timo Sirainen date: Mon Nov 30 21:39:56 2015 +0200 description: director: Minor code cleanup - removed unnecessary code. diffstat: src/director/mail-host.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diffs (11 lines): diff -r 1c826c476dbb -r b0e2a14d5a40 src/director/mail-host.c --- a/src/director/mail-host.c Mon Nov 30 21:39:34 2015 +0200 +++ b/src/director/mail-host.c Mon Nov 30 21:39:56 2015 +0200 @@ -547,7 +547,6 @@ dest = i_new(struct mail_host, 1); *dest = *src; - dest->tag = src->tag; dest->hostname = i_strdup(src->hostname); return dest; }