From dovecot at dovecot.org Fri Nov 1 16:44:38 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 01 Nov 2013 16:44:38 +0200 Subject: dovecot-2.2: lib-storage: Hooks weren't called exactly in the or... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aa8f6832a5e9 changeset: 16900:aa8f6832a5e9 user: Timo Sirainen date: Fri Nov 01 16:43:53 2013 +0200 description: lib-storage: Hooks weren't called exactly in the order they were intended. The comparison didn't skip the paths, so the "lib" prefix skipping didn't work. diffstat: src/lib-storage/mail-storage-hooks.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (16 lines): diff -r 70f1d2bb0bd9 -r aa8f6832a5e9 src/lib-storage/mail-storage-hooks.c --- a/src/lib-storage/mail-storage-hooks.c Wed Oct 30 12:11:54 2013 +0200 +++ b/src/lib-storage/mail-storage-hooks.c Fri Nov 01 16:43:53 2013 +0200 @@ -121,6 +121,12 @@ const struct mail_storage_module_hooks *h2) { const char *s1 = h1->module->path, *s2 = h2->module->path; + const char *p; + + p = strrchr(s1, '/'); + if (p != NULL) s1 = p+1; + p = strrchr(s2, '/'); + if (p != NULL) s2 = p+1; if (strncmp(s1, "lib", 3) == 0) s1 += 3; From dovecot at dovecot.org Fri Nov 1 16:44:38 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 01 Nov 2013 16:44:38 +0200 Subject: dovecot-2.2: lib-storage: If hook is forced, don't bother checki... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/59979be53b9f changeset: 16901:59979be53b9f user: Timo Sirainen date: Fri Nov 01 16:44:25 2013 +0200 description: lib-storage: If hook is forced, don't bother checking if it's in user's mail_plugins. diffstat: src/lib-storage/mail-storage-hooks.c | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diffs (19 lines): diff -r aa8f6832a5e9 -r 59979be53b9f src/lib-storage/mail-storage-hooks.c --- a/src/lib-storage/mail-storage-hooks.c Fri Nov 01 16:43:53 2013 +0200 +++ b/src/lib-storage/mail-storage-hooks.c Fri Nov 01 16:44:25 2013 +0200 @@ -146,9 +146,12 @@ t_array_init(&tmp_hooks, array_count(&module_hooks)); plugins = t_strsplit_spaces(user->set->mail_plugins, ", "); array_foreach(&module_hooks, module_hook) { - name = module_get_plugin_name(module_hook->module); - if (str_array_find(plugins, name) || module_hook->forced) - array_append(&tmp_hooks, module_hook, 1); + if (!module_hook->forced) { + name = module_get_plugin_name(module_hook->module); + if (!str_array_find(plugins, name)) + continue; + } + array_append(&tmp_hooks, module_hook, 1); } /* next we have to sort them by the modules' priority (based on name) */ From dovecot at dovecot.org Sat Nov 2 12:14:28 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 12:14:28 +0200 Subject: dovecot-2.2: doveadm import: Fixed translating different source ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1936d50f01c4 changeset: 16902:1936d50f01c4 user: Timo Sirainen date: Sat Nov 02 12:14:17 2013 +0200 description: doveadm import: Fixed translating different source and dest hierarchy separators in mailbox names. diffstat: src/doveadm/doveadm-mail-import.c | 35 +++++++++++++++++++++++++++++------ 1 files changed, 29 insertions(+), 6 deletions(-) diffs (72 lines): diff -r 59979be53b9f -r 1936d50f01c4 src/doveadm/doveadm-mail-import.c --- a/src/doveadm/doveadm-mail-import.c Fri Nov 01 16:44:25 2013 +0200 +++ b/src/doveadm/doveadm-mail-import.c Sat Nov 02 12:14:17 2013 +0200 @@ -2,6 +2,7 @@ #include "lib.h" #include "array.h" +#include "str.h" #include "mail-storage.h" #include "mail-storage-service.h" #include "mail-namespace.h" @@ -17,23 +18,46 @@ bool subscribe; }; +static const char * +convert_vname_separators(const char *vname, char src_sep, char dest_sep) +{ + string_t *str = t_str_new(128); + + for (; *vname != '\0'; vname++) { + if (*vname == src_sep) + str_append_c(str, dest_sep); + else if (*vname == dest_sep) + str_append_c(str, '_'); + else + str_append_c(str, *vname); + } + return str_c(str); +} + static int dest_mailbox_open_or_create(struct import_cmd_context *ctx, - struct mail_user *user, const char *name, + struct mail_user *user, + const struct mailbox_info *info, struct mailbox **box_r) { struct mail_namespace *ns; struct mailbox *box; enum mail_error error; - const char *errstr; + const char *name, *errstr; if (*ctx->dest_parent != '\0') { /* prefix destination mailbox name with given parent mailbox */ ns = mail_namespace_find(user->namespaces, ctx->dest_parent); + } else { + ns = mail_namespace_find(user->namespaces, info->vname); + } + name = convert_vname_separators(info->vname, + mail_namespace_get_sep(info->ns), + mail_namespace_get_sep(ns)); + + if (*ctx->dest_parent != '\0') { name = t_strdup_printf("%s%c%s", ctx->dest_parent, mail_namespace_get_sep(ns), name); - } else { - ns = mail_namespace_find(user->namespaces, name); } box = mailbox_alloc(ns->list, name, MAILBOX_FLAG_SAVEONLY); @@ -113,8 +137,7 @@ if (doveadm_mail_iter_next(iter, &mail)) { /* at least one mail matches in this mailbox */ - if (dest_mailbox_open_or_create(ctx, dest_user, info->vname, - &box) < 0) + if (dest_mailbox_open_or_create(ctx, dest_user, info, &box) < 0) ret = -1; else { if (cmd_import_box_contents(iter, mail, box) < 0) { From dovecot at dovecot.org Sat Nov 2 12:25:17 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 12:25:17 +0200 Subject: dovecot-2.2: mdbox: When mail's refcount is too high, don't prev... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8cdd4124de0b changeset: 16903:8cdd4124de0b user: Timo Sirainen date: Sat Nov 02 12:25:09 2013 +0200 description: mdbox: When mail's refcount is too high, don't prevent shrinking the refcount. diffstat: src/lib-storage/index/dbox-multi/mdbox-map.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 1936d50f01c4 -r 8cdd4124de0b src/lib-storage/index/dbox-multi/mdbox-map.c --- a/src/lib-storage/index/dbox-multi/mdbox-map.c Sat Nov 02 12:14:17 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c Sat Nov 02 12:25:09 2013 +0200 @@ -649,7 +649,7 @@ map_uid); return -1; } - if (old_diff + new_diff >= 32768) { + if (old_diff + new_diff >= 32768 && new_diff > 0) { /* we're getting close to the 64k limit. fail early to make it less likely that two processes increase the refcount enough times to cross the limit */ From dovecot at dovecot.org Sat Nov 2 12:25:33 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 12:25:33 +0200 Subject: dovecot-2.1: mdbox: When mail's refcount is too high, don't prev... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/f572fbafb445 changeset: 15006:f572fbafb445 user: Timo Sirainen date: Sat Nov 02 12:25:09 2013 +0200 description: mdbox: When mail's refcount is too high, don't prevent shrinking the refcount. diffstat: src/lib-storage/index/dbox-multi/mdbox-map.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d16e212531ec -r f572fbafb445 src/lib-storage/index/dbox-multi/mdbox-map.c --- a/src/lib-storage/index/dbox-multi/mdbox-map.c Sat Oct 26 18:00:07 2013 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c Sat Nov 02 12:25:09 2013 +0200 @@ -670,7 +670,7 @@ map_uid); return -1; } - if (old_diff + new_diff >= 32768) { + if (old_diff + new_diff >= 32768 && new_diff > 0) { /* we're getting close to the 64k limit. fail early to make it less likely that two processes increase the refcount enough times to cross the limit */ From dovecot at dovecot.org Sat Nov 2 12:37:21 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 12:37:21 +0200 Subject: dovecot-2.2: auth: passdb passwd supports now lookup_credentials... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/47848e9fc622 changeset: 16904:47848e9fc622 user: Timo Sirainen date: Sat Nov 02 12:37:10 2013 +0200 description: auth: passdb passwd supports now lookup_credentials() API It's not very useful, but at least it allows the lookup itself to succeed. diffstat: src/auth/passdb-passwd.c | 74 ++++++++++++++++++++++++++++++++--------------- 1 files changed, 50 insertions(+), 24 deletions(-) diffs (107 lines): diff -r 8cdd4124de0b -r 47848e9fc622 src/auth/passdb-passwd.c --- a/src/auth/passdb-passwd.c Sat Nov 02 12:25:09 2013 +0200 +++ b/src/auth/passdb-passwd.c Sat Nov 02 12:37:10 2013 +0200 @@ -11,38 +11,46 @@ #define PASSWD_CACHE_KEY "%u" #define PASSWD_PASS_SCHEME "CRYPT" +static enum passdb_result +passwd_lookup(struct auth_request *request, struct passwd *pw_r) +{ + auth_request_log_debug(request, "passwd", "lookup"); + + switch (i_getpwnam(request->user, pw_r)) { + case -1: + auth_request_log_error(request, "passwd", + "getpwnam() failed: %m"); + return PASSDB_RESULT_INTERNAL_FAILURE; + case 0: + auth_request_log_unknown_user(request, "passwd"); + return PASSDB_RESULT_USER_UNKNOWN; + } + + if (!IS_VALID_PASSWD(pw_r->pw_passwd)) { + auth_request_log_info(request, "passwd", + "invalid password field '%s'", pw_r->pw_passwd); + return PASSDB_RESULT_USER_DISABLED; + } + + /* save the password so cache can use it */ + auth_request_set_field(request, "password", pw_r->pw_passwd, + PASSWD_PASS_SCHEME); + return PASSDB_RESULT_OK; +} + static void passwd_verify_plain(struct auth_request *request, const char *password, verify_plain_callback_t *callback) { struct passwd pw; + enum passdb_result res; int ret; - auth_request_log_debug(request, "passwd", "lookup"); - - switch (i_getpwnam(request->user, &pw)) { - case -1: - auth_request_log_error(request, "passwd", - "getpwnam() failed: %m"); - callback(PASSDB_RESULT_INTERNAL_FAILURE, request); - return; - case 0: - auth_request_log_unknown_user(request, "passwd"); - callback(PASSDB_RESULT_USER_UNKNOWN, request); + res = passwd_lookup(request, &pw); + if (res != PASSDB_RESULT_OK) { + callback(res, request); return; } - - if (!IS_VALID_PASSWD(pw.pw_passwd)) { - auth_request_log_info(request, "passwd", - "invalid password field '%s'", pw.pw_passwd); - callback(PASSDB_RESULT_USER_DISABLED, request); - return; - } - - /* save the password so cache can use it */ - auth_request_set_field(request, "password", pw.pw_passwd, - PASSWD_PASS_SCHEME); - /* check if the password is valid */ ret = auth_request_password_verify(request, password, pw.pw_passwd, PASSWD_PASS_SCHEME, "passwd"); @@ -61,6 +69,24 @@ callback(PASSDB_RESULT_OK, request); } +static void +passwd_lookup_credentials(struct auth_request *request, + lookup_credentials_callback_t *callback) +{ + struct passwd pw; + enum passdb_result res; + + res = passwd_lookup(request, &pw); + if (res != PASSDB_RESULT_OK) { + callback(res, NULL, 0, request); + return; + } + /* make sure we're using the username exactly as it's in the database */ + auth_request_set_field(request, "user", pw.pw_name, NULL); + passdb_handle_credentials(PASSDB_RESULT_OK, pw.pw_passwd, + PASSWD_PASS_SCHEME, callback, request); +} + static struct passdb_module * passwd_preinit(pool_t pool, const char *args) { @@ -91,7 +117,7 @@ passwd_deinit, passwd_verify_plain, - NULL, + passwd_lookup_credentials, NULL }; From dovecot at dovecot.org Sat Nov 2 12:43:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 12:43:06 +0200 Subject: dovecot-2.2: auth: Return a reason string if PASS lookup fails b... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9b45f6d20d9d changeset: 16905:9b45f6d20d9d user: Timo Sirainen date: Sat Nov 02 12:42:55 2013 +0200 description: auth: Return a reason string if PASS lookup fails because passdbs don't support it. diffstat: src/auth/auth-master-connection.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (16 lines): diff -r 47848e9fc622 -r 9b45f6d20d9d src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Sat Nov 02 12:37:10 2013 +0200 +++ b/src/auth/auth-master-connection.c Sat Nov 02 12:42:55 2013 +0200 @@ -340,8 +340,11 @@ break; case PASSDB_RESULT_PASSWORD_MISMATCH: case PASSDB_RESULT_INTERNAL_FAILURE: + str_printfa(str, "FAIL\t%u", auth_request->id); + break; case PASSDB_RESULT_SCHEME_NOT_AVAILABLE: - str_printfa(str, "FAIL\t%u", auth_request->id); + str_printfa(str, "FAIL\t%u\treason=Configured passdbs don't support crentials lookups", + auth_request->id); break; } From dovecot at dovecot.org Sat Nov 2 12:46:13 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 12:46:13 +0200 Subject: dovecot-2.2: auth: passdb shadow supports now lookup_credentials... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8eec76bbde28 changeset: 16906:8eec76bbde28 user: Timo Sirainen date: Sat Nov 02 12:46:08 2013 +0200 description: auth: passdb shadow supports now lookup_credentials() API diffstat: src/auth/passdb-shadow.c | 64 ++++++++++++++++++++++++++++++++++------------- 1 files changed, 46 insertions(+), 18 deletions(-) diffs (98 lines): diff -r 9b45f6d20d9d -r 8eec76bbde28 src/auth/passdb-shadow.c --- a/src/auth/passdb-shadow.c Sat Nov 02 12:42:55 2013 +0200 +++ b/src/auth/passdb-shadow.c Sat Nov 02 12:46:08 2013 +0200 @@ -12,33 +12,43 @@ #define SHADOW_CACHE_KEY "%u" #define SHADOW_PASS_SCHEME "CRYPT" +static enum passdb_result +shadow_lookup(struct auth_request *request, struct spwd **spw_r) +{ + auth_request_log_debug(request, "shadow", "lookup"); + + *spw_r = getspnam(request->user); + if (*spw_r == NULL) { + auth_request_log_unknown_user(request, "shadow"); + return PASSDB_RESULT_USER_UNKNOWN; + } + + if (!IS_VALID_PASSWD((*spw_r)->sp_pwdp)) { + auth_request_log_info(request, "shadow", + "invalid password field"); + return PASSDB_RESULT_USER_DISABLED; + } + + /* save the password so cache can use it */ + auth_request_set_field(request, "password", (*spw_r)->sp_pwdp, + SHADOW_PASS_SCHEME); + return PASSDB_RESULT_OK; +} + static void shadow_verify_plain(struct auth_request *request, const char *password, verify_plain_callback_t *callback) { struct spwd *spw; + enum passdb_result res; int ret; - auth_request_log_debug(request, "shadow", "lookup"); - - spw = getspnam(request->user); - if (spw == NULL) { - auth_request_log_unknown_user(request, "shadow"); - callback(PASSDB_RESULT_USER_UNKNOWN, request); + res = shadow_lookup(request, &spw); + if (res != PASSDB_RESULT_OK) { + callback(res, request); return; } - if (!IS_VALID_PASSWD(spw->sp_pwdp)) { - auth_request_log_info(request, "shadow", - "invalid password field"); - callback(PASSDB_RESULT_USER_DISABLED, request); - return; - } - - /* save the password so cache can use it */ - auth_request_set_field(request, "password", spw->sp_pwdp, - SHADOW_PASS_SCHEME); - /* check if the password is valid */ ret = auth_request_password_verify(request, password, spw->sp_pwdp, SHADOW_PASS_SCHEME, "shadow"); @@ -57,6 +67,24 @@ callback(PASSDB_RESULT_OK, request); } +static void +shadow_lookup_credentials(struct auth_request *request, + lookup_credentials_callback_t *callback) +{ + struct spwd *spw; + enum passdb_result res; + + res = shadow_lookup(request, &spw); + if (res != PASSDB_RESULT_OK) { + callback(res, NULL, 0, request); + return; + } + /* make sure we're using the username exactly as it's in the database */ + auth_request_set_field(request, "user", spw->sp_namp, NULL); + passdb_handle_credentials(PASSDB_RESULT_OK, spw->sp_pwdp, + SHADOW_PASS_SCHEME, callback, request); +} + static struct passdb_module * shadow_preinit(pool_t pool, const char *args) { @@ -87,7 +115,7 @@ shadow_deinit, shadow_verify_plain, - NULL, + shadow_lookup_credentials, NULL }; #else From dovecot at dovecot.org Sat Nov 2 13:15:02 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 13:15:02 +0200 Subject: dovecot-2.2: mbox: Added assert to try to catch corruption. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d3062d066593 changeset: 16907:d3062d066593 user: Timo Sirainen date: Sat Nov 02 13:14:16 2013 +0200 description: mbox: Added assert to try to catch corruption. diffstat: src/lib-storage/index/mbox/mbox-sync.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 8eec76bbde28 -r d3062d066593 src/lib-storage/index/mbox/mbox-sync.c --- a/src/lib-storage/index/mbox/mbox-sync.c Sat Nov 02 12:46:08 2013 +0200 +++ b/src/lib-storage/index/mbox/mbox-sync.c Sat Nov 02 13:14:16 2013 +0200 @@ -651,6 +651,8 @@ /* read the From-line before rewriting overwrites it */ if (mbox_read_from_line(mail_ctx) < 0) return -1; + i_assert(mail_ctx->mail.from_offset + move_diff != 1 && + mail_ctx->mail.from_offset + move_diff != 2); mbox_sync_update_header(mail_ctx); ret = mbox_sync_try_rewrite(mail_ctx, move_diff); From dovecot at dovecot.org Sat Nov 2 13:34:28 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 13:34:28 +0200 Subject: dovecot-2.2: maildir: If filename begins with ':', rename it imm... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/58dfed53ab43 changeset: 16908:58dfed53ab43 user: Timo Sirainen date: Sat Nov 02 13:34:17 2013 +0200 description: maildir: If filename begins with ':', rename it immediately instead of causing errors. diffstat: src/lib-storage/index/maildir/maildir-sync.c | 29 ++++++++++++++++++++++++++++ 1 files changed, 29 insertions(+), 0 deletions(-) diffs (46 lines): diff -r d3062d066593 -r 58dfed53ab43 src/lib-storage/index/maildir/maildir-sync.c --- a/src/lib-storage/index/maildir/maildir-sync.c Sat Nov 02 13:14:16 2013 +0200 +++ b/src/lib-storage/index/maildir/maildir-sync.c Sat Nov 02 13:34:17 2013 +0200 @@ -362,6 +362,27 @@ } static int +maildir_rename_empty_basename(struct maildir_sync_context *ctx, + const char *dir, const char *fname) +{ + const char *old_path, *new_fname, *new_path; + + old_path = t_strconcat(dir, "/", fname, NULL); + new_fname = maildir_filename_generate(); + new_path = t_strconcat(mailbox_get_path(&ctx->mbox->box), + "/new/", new_fname, NULL); + if (rename(old_path, new_path) == 0) + i_warning("Fixed broken filename: %s -> %s", old_path, new_fname); + else if (errno != ENOENT) { + mail_storage_set_critical(&ctx->mbox->storage->storage, + "Couldn't fix a broken filename: rename(%s, %s) failed: %m", + old_path, new_path); + return -1; + } + return 0; +} + +static int maildir_stat(struct maildir_mailbox *mbox, const char *path, struct stat *st_r) { struct mailbox *box = &mbox->box; @@ -457,6 +478,14 @@ if (dp->d_name[0] == '.') continue; + if (dp->d_name[0] == MAILDIR_INFO_SEP) { + /* don't even try to use file with empty base name */ + if (maildir_rename_empty_basename(ctx, path, + dp->d_name) < 0) + break; + continue; + } + flags = 0; if (move_new) { i_assert(dp->d_name[0] != '\0'); From dovecot at dovecot.org Sat Nov 2 13:52:45 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 13:52:45 +0200 Subject: dovecot-2.2: auth: Added %{session_pid} variable for userdb look... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9ef23179892a changeset: 16909:9ef23179892a user: Timo Sirainen date: Sat Nov 02 13:52:30 2013 +0200 description: auth: Added %{session_pid} variable for userdb lookups in login requests. session_pid is now always sent to auth process. A new request_auth_token parameter was added to specify if auth_token should be returned or not. diffstat: src/auth/auth-request-handler.c | 3 ++- src/auth/auth-request.c | 7 ++++++- src/auth/auth-request.h | 3 ++- src/lib-master/master-login-auth.c | 3 ++- 4 files changed, 12 insertions(+), 4 deletions(-) diffs (78 lines): diff -r 58dfed53ab43 -r 9ef23179892a src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Sat Nov 02 13:34:17 2013 +0200 +++ b/src/auth/auth-request-handler.c Sat Nov 02 13:52:30 2013 +0200 @@ -669,7 +669,8 @@ str_append(str, "\tanonymous"); } /* generate auth_token when master service provided session_pid */ - if (request->session_pid != (pid_t)-1) { + if (request->request_auth_token && + request->session_pid != (pid_t)-1) { const char *auth_token = auth_token_get(request->service, dec2str(request->session_pid), diff -r 58dfed53ab43 -r 9ef23179892a src/auth/auth-request.c --- a/src/auth/auth-request.c Sat Nov 02 13:34:17 2013 +0200 +++ b/src/auth/auth-request.c Sat Nov 02 13:52:30 2013 +0200 @@ -331,7 +331,9 @@ if (strcmp(key, "session_pid") == 0) { if (str_to_pid(value, &pid) == 0) request->session_pid = pid; - } else + } else if (strcmp(key, "request_auth_token") == 0) + request->request_auth_token = TRUE; + else return FALSE; return TRUE; } @@ -1950,6 +1952,7 @@ { '\0', NULL, "domain_first" }, { '\0', NULL, "domain_last" }, { '\0', NULL, "master_user" }, + { '\0', NULL, "session_pid" }, /* be sure to update AUTH_REQUEST_VAR_TAB_COUNT */ { '\0', NULL, NULL } }; @@ -2037,6 +2040,8 @@ tab[24].value = escape_func(tab[24].value+1, auth_request); tab[25].value = auth_request->master_user == NULL ? NULL : escape_func(auth_request->master_user, auth_request); + tab[26].value = auth_request->session_pid == (pid_t)-1 ? NULL : + dec2str(auth_request->session_pid); return ret_tab; } diff -r 58dfed53ab43 -r 9ef23179892a src/auth/auth-request.h --- a/src/auth/auth-request.h Sat Nov 02 13:34:17 2013 +0200 +++ b/src/auth/auth-request.h Sat Nov 02 13:52:30 2013 +0200 @@ -108,6 +108,7 @@ unsigned int no_penalty:1; unsigned int valid_client_cert:1; unsigned int cert_username:1; + unsigned int request_auth_token:1; /* success/failure states: */ unsigned int successful:1; @@ -143,7 +144,7 @@ #define AUTH_REQUEST_VAR_TAB_USER_IDX 0 #define AUTH_REQUEST_VAR_TAB_USERNAME_IDX 1 #define AUTH_REQUEST_VAR_TAB_DOMAIN_IDX 2 -#define AUTH_REQUEST_VAR_TAB_COUNT 26 +#define AUTH_REQUEST_VAR_TAB_COUNT 27 extern const struct var_expand_table auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT+1]; diff -r 58dfed53ab43 -r 9ef23179892a src/lib-master/master-login-auth.c --- a/src/lib-master/master-login-auth.c Sat Nov 02 13:34:17 2013 +0200 +++ b/src/lib-master/master-login-auth.c Sat Nov 02 13:52:30 2013 +0200 @@ -440,8 +440,9 @@ str_printfa(str, "REQUEST\t%u\t%u\t%u\t", req->id, req->client_pid, req->auth_id); binary_to_hex_append(str, req->cookie, sizeof(req->cookie)); + str_printfa(str, "\tsession_pid=%s", my_pid); if (auth->request_auth_token) - str_printfa(str, "\tsession_pid=%s", my_pid); + str_append(str, "\trequest_auth_token"); str_append_c(str, '\n'); o_stream_nsend(auth->output, str_data(str), str_len(str)); } From dovecot at dovecot.org Sat Nov 2 14:19:39 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 14:19:39 +0200 Subject: dovecot-2.2: dsync: Don't track too new changes from transaction... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dbe7c5a0021e changeset: 16910:dbe7c5a0021e user: Timo Sirainen date: Sat Nov 02 14:19:23 2013 +0200 description: dsync: Don't track too new changes from transaction log. This mattered only during race conditions while there were changes being done to the mailbox during syncing. diffstat: src/doveadm/dsync/dsync-transaction-log-scan.c | 41 +++++++++++++++++++------ 1 files changed, 30 insertions(+), 11 deletions(-) diffs (64 lines): diff -r 9ef23179892a -r dbe7c5a0021e src/doveadm/dsync/dsync-transaction-log-scan.c --- a/src/doveadm/dsync/dsync-transaction-log-scan.c Sat Nov 02 13:52:30 2013 +0200 +++ b/src/doveadm/dsync/dsync-transaction-log-scan.c Sat Nov 02 14:19:23 2013 +0200 @@ -353,17 +353,16 @@ bool reset; int ret; - if (modseq == 0 || - !mail_index_modseq_get_next_log_offset(view, modseq, - &log_seq, &log_offset)) - ret = 0; - else { + end_seq = view->log_file_head_seq; + end_offset = view->log_file_head_offset; + + if (modseq != 0 && + mail_index_modseq_get_next_log_offset(view, modseq, + &log_seq, &log_offset)) { /* scan the view only up to end of the current view. if there are more changes, we don't care about them until the next sync. the modseq may however already point to beyond the current view's end (FIXME: why?) */ - end_seq = view->log_file_head_seq; - end_offset = view->log_file_head_offset; if (log_seq > end_seq || (log_seq == end_seq && log_offset > end_offset)) { end_seq = log_seq; @@ -373,12 +372,32 @@ log_seq, log_offset, end_seq, end_offset, &reset); + if (ret != 0) + return ret; } + + /* return everything we've got (until the end of the view) */ + if (!pvt_scan) + ctx->returned_all_changes = TRUE; + if (mail_transaction_log_view_set_all(log_view) < 0) + return -1; + + mail_transaction_log_view_get_prev_pos(log_view, &log_seq, &log_offset); + if (log_seq > end_seq || + (log_seq == end_seq && log_offset > end_offset)) { + end_seq = log_seq; + end_offset = log_offset; + } + ret = mail_transaction_log_view_set(log_view, + log_seq, log_offset, + end_seq, end_offset, &reset); if (ret == 0) { - /* return everything we've got */ - if (!pvt_scan) - ctx->returned_all_changes = TRUE; - return mail_transaction_log_view_set_all(log_view); + /* we shouldn't get here. _view_set_all() already + reserved all the log files, the _view_set() only + removed unwanted ones. */ + i_error("%s: Couldn't set transaction log view (seq %u..%u)", + view->index->filepath, log_seq, end_seq); + ret = -1; } return ret < 0 ? -1 : 0; } From dovecot at dovecot.org Sat Nov 2 15:18:30 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 15:18:30 +0200 Subject: dovecot-2.2: ssl-params: Fixed closing listener fds for regenera... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/07c083496665 changeset: 16911:07c083496665 user: Timo Sirainen date: Sat Nov 02 15:18:15 2013 +0200 description: ssl-params: Fixed closing listener fds for regeneration process. diffstat: src/ssl-params/ssl-params.c | 19 ++++++++++++++++++- 1 files changed, 18 insertions(+), 1 deletions(-) diffs (43 lines): diff -r dbe7c5a0021e -r 07c083496665 src/ssl-params/ssl-params.c --- a/src/ssl-params/ssl-params.c Sat Nov 02 14:19:23 2013 +0200 +++ b/src/ssl-params/ssl-params.c Sat Nov 02 15:18:15 2013 +0200 @@ -5,6 +5,7 @@ #include "buffer.h" #include "file-lock.h" #include "read-full.h" +#include "master-interface.h" #include "master-service.h" #include "master-service-settings.h" #include "ssl-params-settings.h" @@ -110,6 +111,22 @@ i_info("SSL parameters regeneration completed"); } +static void ssl_params_close_listeners(void) +{ + unsigned int i; + + /* we have forked, but the fds are still shared. we can't go + io_remove()ing the fds from ioloop, because with many ioloops + (e.g. epoll) the fds get removed from the main process's ioloop + as well. so we'll just do the closing here manually. */ + for (i = 0; i < master_service_get_socket_count(master_service); i++) { + int fd = MASTER_LISTEN_FD_FIRST + i; + + if (close(fd) < 0) + i_error("close(listener %d) failed: %m", fd); + } +} + static void ssl_params_rebuild(struct ssl_params *param) { if (param->to_rebuild != NULL) @@ -121,7 +138,7 @@ case 0: /* child - close listener fds so a long-running ssl-params doesn't cause Dovecot restart to fail */ - master_service_stop_new_connections(master_service); + ssl_params_close_listeners(); ssl_params_if_unchanged(param->path, param->last_mtime); exit(0); default: From dovecot at dovecot.org Sat Nov 2 15:27:59 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 15:27:59 +0200 Subject: dovecot-2.2: ssl-params: Added ssl_dh_parameters_length & remove... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/43ab5abeb8f0 changeset: 16912:43ab5abeb8f0 user: Timo Sirainen date: Sat Nov 02 15:27:28 2013 +0200 description: ssl-params: Added ssl_dh_parameters_length & removed ssl_parameters_regenerate setting. ssl_parameters_regenerate was based on some text from GNUTLS documentation a long time ago, but there's really not much point in doing it. Ideally we should also support "openssl dhparam" input files, but for now there's the ssl_dh_parameters_length setting that can be used to specify the wanted DH parameters length. If the current ssl-parameters.dat has a different length, it's regenerated. We should probably at some point support also built-in DH parameters which are returned while the ssl-params runs. diffstat: doc/example-config/conf.d/10-ssl.conf | 6 +-- src/login-common/ssl-proxy-openssl.c | 32 ++++++++-------- src/ssl-params/ssl-params-openssl.c | 9 +--- src/ssl-params/ssl-params-settings.c | 4 +- src/ssl-params/ssl-params-settings.h | 1 + src/ssl-params/ssl-params.c | 66 +++++++++++++++++++--------------- src/ssl-params/ssl-params.h | 2 +- 7 files changed, 63 insertions(+), 57 deletions(-) diffs (truncated from 324 to 300 lines): diff -r 07c083496665 -r 43ab5abeb8f0 doc/example-config/conf.d/10-ssl.conf --- a/doc/example-config/conf.d/10-ssl.conf Sat Nov 02 15:18:15 2013 +0200 +++ b/doc/example-config/conf.d/10-ssl.conf Sat Nov 02 15:27:28 2013 +0200 @@ -42,10 +42,8 @@ # auth_ssl_username_from_cert=yes. #ssl_cert_username_field = commonName -# How often to regenerate the SSL parameters file. Generation is quite CPU -# intensive operation. The value is in hours, 0 disables regeneration -# entirely. -#ssl_parameters_regenerate = 168 +# DH parameters length to use. +#ssl_dh_parameters_length = 1024 # SSL protocols to use #ssl_protocols = !SSLv2 diff -r 07c083496665 -r 43ab5abeb8f0 src/login-common/ssl-proxy-openssl.c --- a/src/login-common/ssl-proxy-openssl.c Sat Nov 02 15:18:15 2013 +0200 +++ b/src/login-common/ssl-proxy-openssl.c Sat Nov 02 15:27:28 2013 +0200 @@ -86,7 +86,7 @@ time_t last_refresh; int fd; - DH *dh_512, *dh_1024; + DH *dh_512, *dh_default; }; struct ssl_server_context { @@ -163,9 +163,9 @@ return ctx1->verify_client_cert == ctx2->verify_client_cert ? 0 : 1; } -static void ssl_params_corrupted(void) +static void ssl_params_corrupted(const char *reason) { - i_fatal("Corrupted SSL parameters file in state_dir: ssl-parameters.dat"); + i_fatal("Corrupted SSL ssl-parameters.dat in state_dir: %s", reason); } static void read_next(struct ssl_parameters *params, void *data, size_t size) @@ -175,7 +175,7 @@ if ((ret = read_full(params->fd, data, size)) < 0) i_fatal("read(%s) failed: %m", params->path); if (ret == 0) - ssl_params_corrupted(); + ssl_params_corrupted("Truncated file"); } static bool read_dh_parameters_next(struct ssl_parameters *params) @@ -194,7 +194,7 @@ /* read data size. */ read_next(params, &len, sizeof(len)); if (len > 1024*100) /* should be enough? */ - ssl_params_corrupted(); + ssl_params_corrupted("File too large"); buf = i_malloc(len); read_next(params, buf, len); @@ -202,13 +202,15 @@ cbuf = buf; switch (bits) { case 512: + if (params->dh_512 != NULL) + ssl_params_corrupted("Duplicate 512bit parameters"); params->dh_512 = d2i_DHparams(NULL, &cbuf, len); break; - case 1024: - params->dh_1024 = d2i_DHparams(NULL, &cbuf, len); + default: + if (params->dh_default != NULL) + ssl_params_corrupted("Duplicate default parameters"); + params->dh_default = d2i_DHparams(NULL, &cbuf, len); break; - default: - ssl_params_corrupted(); } i_free(buf); @@ -221,9 +223,9 @@ DH_free(params->dh_512); params->dh_512 = NULL; } - if (params->dh_1024 != NULL) { - DH_free(params->dh_1024); - params->dh_1024 = NULL; + if (params->dh_default != NULL) { + DH_free(params->dh_default); + params->dh_default = NULL; } } @@ -250,7 +252,7 @@ i_fatal("read(%s) failed: %m", params->path); else if (ret != 0) { /* more data than expected */ - ssl_params_corrupted(); + ssl_params_corrupted("More data than expected"); } if (close(params->fd) < 0) @@ -840,12 +842,10 @@ static DH *ssl_tmp_dh_callback(SSL *ssl ATTR_UNUSED, int is_export, int keylength) { - /* Well, I'm not exactly sure why the logic in here is this. - It's the same as in Postfix, so it can't be too wrong. */ if (is_export && keylength == 512 && ssl_params.dh_512 != NULL) return ssl_params.dh_512; - return ssl_params.dh_1024; + return ssl_params.dh_default; } static void ssl_info_callback(const SSL *ssl, int where, int ret) diff -r 07c083496665 -r 43ab5abeb8f0 src/ssl-params/ssl-params-openssl.c --- a/src/ssl-params/ssl-params-openssl.c Sat Nov 02 15:18:15 2013 +0200 +++ b/src/ssl-params/ssl-params-openssl.c Sat Nov 02 15:27:28 2013 +0200 @@ -13,8 +13,6 @@ default.. */ #define DH_GENERATOR 2 -static int dh_param_bitsizes[] = { 512, 1024 }; - static const char *ssl_last_error(void) { unsigned long err; @@ -56,13 +54,12 @@ i_free(buf); } -void ssl_generate_parameters(int fd, const char *fname) +void ssl_generate_parameters(int fd, unsigned int dh_length, const char *fname) { - unsigned int i; int bits; - for (i = 0; i < N_ELEMENTS(dh_param_bitsizes); i++) - generate_dh_parameters(dh_param_bitsizes[i], fd, fname); + generate_dh_parameters(512, fd, fname); + generate_dh_parameters(dh_length, fd, fname); bits = 0; if (write_full(fd, &bits, sizeof(bits)) < 0) i_fatal("write_full() failed for file %s: %m", fname); diff -r 07c083496665 -r 43ab5abeb8f0 src/ssl-params/ssl-params-settings.c --- a/src/ssl-params/ssl-params-settings.c Sat Nov 02 15:18:15 2013 +0200 +++ b/src/ssl-params/ssl-params-settings.c Sat Nov 02 15:27:28 2013 +0200 @@ -61,12 +61,14 @@ static const struct setting_define ssl_params_setting_defines[] = { DEF(SET_TIME, ssl_parameters_regenerate), + DEF(SET_UINT, ssl_dh_parameters_length), SETTING_DEFINE_LIST_END }; static const struct ssl_params_settings ssl_params_default_settings = { - .ssl_parameters_regenerate = 3600*24*7 + .ssl_parameters_regenerate = 0, + .ssl_dh_parameters_length = 1024 }; const struct setting_parser_info ssl_params_setting_parser_info = { diff -r 07c083496665 -r 43ab5abeb8f0 src/ssl-params/ssl-params-settings.h --- a/src/ssl-params/ssl-params-settings.h Sat Nov 02 15:18:15 2013 +0200 +++ b/src/ssl-params/ssl-params-settings.h Sat Nov 02 15:27:28 2013 +0200 @@ -5,6 +5,7 @@ struct ssl_params_settings { unsigned int ssl_parameters_regenerate; + unsigned int ssl_dh_parameters_length; }; struct ssl_params_settings * diff -r 07c083496665 -r 43ab5abeb8f0 src/ssl-params/ssl-params.c --- a/src/ssl-params/ssl-params.c Sat Nov 02 15:18:15 2013 +0200 +++ b/src/ssl-params/ssl-params.c Sat Nov 02 15:27:28 2013 +0200 @@ -22,7 +22,7 @@ # include #endif -#define MAX_PARAM_FILE_SIZE 1024 +#define MAX_PARAM_FILE_SIZE 1024*1024 #define SSL_BUILD_PARAM_TIMEOUT_SECS (60*30) #define SSL_PARAMS_PRIORITY 15 @@ -31,11 +31,11 @@ struct ssl_params_settings set; time_t last_mtime; - struct timeout *to_rebuild; ssl_params_callback_t *callback; }; -static void ssl_params_if_unchanged(const char *path, time_t mtime) +static void ssl_params_if_unchanged(const char *path, time_t mtime, + unsigned int ssl_dh_parameters_length) { const char *temp_path; struct file_lock *lock; @@ -99,7 +99,7 @@ i_info("Generating SSL parameters"); #ifdef HAVE_SSL - ssl_generate_parameters(fd, temp_path); + ssl_generate_parameters(fd, ssl_dh_parameters_length, temp_path); #endif if (rename(temp_path, path) < 0) @@ -129,9 +129,6 @@ static void ssl_params_rebuild(struct ssl_params *param) { - if (param->to_rebuild != NULL) - timeout_remove(¶m->to_rebuild); - switch (fork()) { case -1: i_fatal("fork() failed: %m"); @@ -139,7 +136,8 @@ /* child - close listener fds so a long-running ssl-params doesn't cause Dovecot restart to fail */ ssl_params_close_listeners(); - ssl_params_if_unchanged(param->path, param->last_mtime); + ssl_params_if_unchanged(param->path, param->last_mtime, + param->set.ssl_dh_parameters_length); exit(0); default: /* parent */ @@ -147,27 +145,38 @@ } } -static void ssl_params_set_timeout(struct ssl_params *param) +static bool +ssl_params_verify(struct ssl_params *param, + const unsigned char *data, size_t size) { - time_t next_rebuild, diff; + unsigned int bitsize, len; + bool found = FALSE; - if (param->to_rebuild != NULL) - timeout_remove(¶m->to_rebuild); - if (param->set.ssl_parameters_regenerate == 0) - return; + /* ... */ + while (size >= sizeof(bitsize)) { + memcpy(&bitsize, data, sizeof(bitsize)); + if (bitsize == 0) { + if (found) + return TRUE; + i_warning("Regenerating %s for ssl_dh_parameters_length=%u", + param->path, param->set.ssl_dh_parameters_length); + return FALSE; + } + data += sizeof(bitsize); + size -= sizeof(bitsize); + if (bitsize == param->set.ssl_dh_parameters_length) + found = TRUE; - next_rebuild = param->last_mtime + - param->set.ssl_parameters_regenerate; - - if (ioloop_time >= next_rebuild) { - ssl_params_rebuild(param); - return; + if (size < sizeof(len)) + break; + memcpy(&len, data, sizeof(len)); + if (len > size - sizeof(len)) + break; + data += sizeof(bitsize) + len; + size -= sizeof(bitsize) + len; } - - diff = next_rebuild - ioloop_time; - if (diff > INT_MAX / 1000) - diff = INT_MAX / 1000; - param->to_rebuild = timeout_add(diff * 1000, ssl_params_rebuild, param); + i_error("Corrupted %s", param->path); + return FALSE; } static int ssl_params_read(struct ssl_params *param) @@ -188,6 +197,7 @@ i_close_fd(&fd); return -1; } + param->last_mtime = st.st_mtime; if (st.st_size == 0 || st.st_size > MAX_PARAM_FILE_SIZE) { i_error("Corrupted file: %s", param->path); i_close_fd(&fd); @@ -202,9 +212,9 @@ else if (ret == 0) { i_error("File unexpectedly shrank: %s", param->path); ret = -1; + } else if (!ssl_params_verify(param, buffer, st.st_size)) { + ret = -1; } else { - param->last_mtime = st.st_mtime; From dovecot at dovecot.org Sat Nov 2 15:30:55 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 15:30:55 +0200 Subject: dovecot-2.2: ssl-params: Don't fail completely if 512 bit DH par... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c472e0454ee3 changeset: 16913:c472e0454ee3 user: Timo Sirainen date: Sat Nov 02 15:30:47 2013 +0200 description: ssl-params: Don't fail completely if 512 bit DH parameters generation fails. diffstat: src/ssl-params/ssl-params-openssl.c | 17 ++++++++++------- 1 files changed, 10 insertions(+), 7 deletions(-) diffs (45 lines): diff -r 43ab5abeb8f0 -r c472e0454ee3 src/ssl-params/ssl-params-openssl.c --- a/src/ssl-params/ssl-params-openssl.c Sat Nov 02 15:27:28 2013 +0200 +++ b/src/ssl-params/ssl-params-openssl.c Sat Nov 02 15:30:47 2013 +0200 @@ -29,16 +29,14 @@ return buf; } -static void generate_dh_parameters(int bitsize, int fd, const char *fname) +static bool generate_dh_parameters(int bitsize, int fd, const char *fname) { DH *dh = DH_generate_parameters(bitsize, DH_GENERATOR, NULL, NULL); unsigned char *buf, *p; int len; - if (dh == NULL) { - i_fatal("DH_generate_parameters(bits=%d, gen=%d) failed: %s", - bitsize, DH_GENERATOR, ssl_last_error()); - } + if (dh == NULL) + return FALSE; len = i2d_DHparams(dh, NULL); if (len < 0) @@ -52,14 +50,19 @@ write_full(fd, buf, len) < 0) i_fatal("write_full() failed for file %s: %m", fname); i_free(buf); + return TRUE; } void ssl_generate_parameters(int fd, unsigned int dh_length, const char *fname) { int bits; - generate_dh_parameters(512, fd, fname); - generate_dh_parameters(dh_length, fd, fname); + /* this fails in FIPS mode */ + (void)generate_dh_parameters(512, fd, fname); + if (!generate_dh_parameters(dh_length, fd, fname)) { + i_fatal("DH_generate_parameters(bits=%d, gen=%d) failed: %s", + dh_length, DH_GENERATOR, ssl_last_error()); + } bits = 0; if (write_full(fd, &bits, sizeof(bits)) < 0) i_fatal("write_full() failed for file %s: %m", fname); From dovecot at dovecot.org Sat Nov 2 21:30:00 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 21:30:00 +0200 Subject: dovecot-2.2: lib-imap: IMAP protocol parser was parsing NIL case... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/09d5728a69d1 changeset: 16914:09d5728a69d1 user: Timo Sirainen date: Sat Nov 02 20:05:08 2013 +0200 description: lib-imap: IMAP protocol parser was parsing NIL case-sensitively. Luckily no commands currently actually use NILs. diffstat: src/lib-imap/imap-parser.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r c472e0454ee3 -r 09d5728a69d1 src/lib-imap/imap-parser.c --- a/src/lib-imap/imap-parser.c Sat Nov 02 15:30:47 2013 +0200 +++ b/src/lib-imap/imap-parser.c Sat Nov 02 20:05:08 2013 +0200 @@ -233,7 +233,7 @@ switch (parser->cur_type) { case ARG_PARSE_ATOM: case ARG_PARSE_TEXT: - if (size == 3 && memcmp(data, "NIL", 3) == 0) { + if (size == 3 && i_memcasecmp(data, "NIL", 3) == 0) { /* NIL argument */ arg->type = IMAP_ARG_NIL; } else { From dovecot at dovecot.org Sat Nov 2 21:30:00 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 21:30:00 +0200 Subject: dovecot-2.2: lib-imap: Added IMAP_PARSE_FLAG_STOP_AT_LIST for st... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0c4ee3b9fa3b changeset: 16915:0c4ee3b9fa3b user: Timo Sirainen date: Sat Nov 02 20:09:28 2013 +0200 description: lib-imap: Added IMAP_PARSE_FLAG_STOP_AT_LIST for stopping after '(' diffstat: src/lib-imap/imap-parser.c | 10 ++++++++-- src/lib-imap/imap-parser.h | 4 +++- 2 files changed, 11 insertions(+), 3 deletions(-) diffs (48 lines): diff -r 09d5728a69d1 -r 0c4ee3b9fa3b src/lib-imap/imap-parser.c --- a/src/lib-imap/imap-parser.c Sat Nov 02 20:05:08 2013 +0200 +++ b/src/lib-imap/imap-parser.c Sat Nov 02 20:09:28 2013 +0200 @@ -609,6 +609,10 @@ break; case '(': imap_parser_open_list(parser); + if ((parser->flags & IMAP_PARSE_FLAG_STOP_AT_LIST) != 0) { + i_stream_skip(parser->input, 1); + return FALSE; + } break; case ')': if (!imap_parser_close_list(parser)) @@ -690,7 +694,8 @@ /* ARG_PARSE_NONE checks that last argument isn't only partially parsed. */ #define IS_UNFINISHED(parser) \ ((parser)->cur_type != ARG_PARSE_NONE || \ - (parser)->cur_list != &parser->root_list) + ((parser)->cur_list != &parser->root_list && \ + ((parser)->flags & IMAP_PARSE_FLAG_STOP_AT_LIST) == 0)) static int finish_line(struct imap_parser *parser, unsigned int count, const struct imap_arg **args_r) @@ -703,7 +708,8 @@ parser->cur_pos = 0; parser->cur_resp_text = FALSE; - if (parser->list_arg != NULL && !parser->literal_size_return) { + if (parser->list_arg != NULL && !parser->literal_size_return && + (parser->flags & IMAP_PARSE_FLAG_STOP_AT_LIST) == 0) { parser->error = "Missing ')'"; *args_r = NULL; return -1; diff -r 09d5728a69d1 -r 0c4ee3b9fa3b src/lib-imap/imap-parser.h --- a/src/lib-imap/imap-parser.h Sat Nov 02 20:05:08 2013 +0200 +++ b/src/lib-imap/imap-parser.h Sat Nov 02 20:09:28 2013 +0200 @@ -24,7 +24,9 @@ /* We're parsing IMAP server replies. Parse the "text" after OK/NO/BAD/BYE replies as a single atom. We assume that the initial "*" or tag was already skipped over. */ - IMAP_PARSE_FLAG_SERVER_TEXT = 0x80 + IMAP_PARSE_FLAG_SERVER_TEXT = 0x80, + /* Parse until '(' and return it as an empty list */ + IMAP_PARSE_FLAG_STOP_AT_LIST = 0x100 }; struct imap_parser; From dovecot at dovecot.org Sat Nov 2 21:30:00 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 21:30:00 +0200 Subject: dovecot-2.2: imap: Added initial support for METADATA extension. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0a08efeb3f40 changeset: 16916:0a08efeb3f40 user: Timo Sirainen date: Sat Nov 02 21:29:39 2013 +0200 description: imap: Added initial support for METADATA extension. For now this is enabled only when imap_metadata=yes setting is used. The setting will go away once the feature is complete. Also mail_attribute_dict must be set. TODO: - Metadata doesn't work for public namespaces. There should probably be a mail_attribute_public_dict setting for that. - There isn't any kind of quota or other limits - After ENABLE METADATA start sending untagged METADATA entries to clients - /shared/admin should probably return postmaster_address URL - Check if we handle ACLs correctly - RFC says that it SHOULD be possible to set METADATA entries to \NoSelect mailboxes. We probably will never allow this though. diffstat: src/imap/Makefile.am | 4 + src/imap/cmd-getmetadata.c | 392 +++++++++++++++++++++++++++++++++++++++++++++ src/imap/cmd-setmetadata.c | 321 ++++++++++++++++++++++++++++++++++++ src/imap/imap-client.c | 5 + src/imap/imap-client.h | 1 + src/imap/imap-commands.c | 2 + src/imap/imap-commands.h | 2 + src/imap/imap-metadata.c | 85 +++++++++ src/imap/imap-metadata.h | 13 + src/imap/imap-settings.c | 2 + src/imap/imap-settings.h | 1 + 11 files changed, 828 insertions(+), 0 deletions(-) diffs (truncated from 942 to 300 lines): diff -r 0c4ee3b9fa3b -r 0a08efeb3f40 src/imap/Makefile.am --- a/src/imap/Makefile.am Sat Nov 02 20:09:28 2013 +0200 +++ b/src/imap/Makefile.am Sat Nov 02 21:29:39 2013 +0200 @@ -39,6 +39,7 @@ cmd-expunge.c \ cmd-fetch.c \ cmd-genurlauth.c \ + cmd-getmetadata.c \ cmd-id.c \ cmd-idle.c \ cmd-list.c \ @@ -51,6 +52,7 @@ cmd-resetkey.c \ cmd-search.c \ cmd-select.c \ + cmd-setmetadata.c \ cmd-sort.c \ cmd-status.c \ cmd-store.c \ @@ -70,6 +72,7 @@ imap-fetch.c \ imap-fetch-body.c \ imap-list.c \ + imap-metadata.c \ imap-notify.c \ imap-search.c \ imap-search-args.c \ @@ -87,6 +90,7 @@ imap-expunge.h \ imap-fetch.h \ imap-list.h \ + imap-metadata.h \ imap-notify.h \ imap-search.h \ imap-search-args.h \ diff -r 0c4ee3b9fa3b -r 0a08efeb3f40 src/imap/cmd-getmetadata.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/imap/cmd-getmetadata.c Sat Nov 02 21:29:39 2013 +0200 @@ -0,0 +1,392 @@ +/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */ + +#include "imap-common.h" +#include "str.h" +#include "istream.h" +#include "ostream.h" +#include "imap-quote.h" +#include "imap-metadata.h" + +struct imap_getmetadata_context { + struct client_command_context *cmd; + + struct mailbox *box; + struct mailbox_transaction_context *trans; + + ARRAY_TYPE(const_string) entries; + uint32_t maxsize; + uoff_t largest_seen_size; + unsigned int depth; + + struct istream *cur_stream; + uoff_t cur_stream_offset, cur_stream_size; + + struct mailbox_attribute_iter *iter; + string_t *iter_entry_prefix; + + const char *key_prefix; + unsigned int entry_idx; + bool first_entry_sent; + bool failed; +}; + +static bool +cmd_getmetadata_parse_options(struct imap_getmetadata_context *ctx, + const struct imap_arg *options) +{ + const char *value; + + while (!IMAP_ARG_IS_EOL(options)) { + if (imap_arg_atom_equals(options, "MAXSIZE")) { + options++; + if (!imap_arg_get_atom(options, &value) || + str_to_uint32(value, &ctx->maxsize) < 0) { + client_send_command_error(ctx->cmd, + "Invalid value for MAXSIZE option"); + return FALSE; + } + } else if (imap_arg_atom_equals(options, "DEPTH")) { + options++; + if (!imap_arg_get_atom(options, &value)) { + client_send_command_error(ctx->cmd, + "Invalid value for DEPTH option"); + return FALSE; + } + if (strcmp(value, "0") == 0) + ctx->depth = 0; + else if (strcmp(value, "1") == 0) + ctx->depth = 1; + else if (strcmp(value, "infinity") == 0) + ctx->depth = UINT_MAX; + else { + client_send_command_error(ctx->cmd, + "Invalid value for DEPTH option"); + return FALSE; + } + } else { + client_send_command_error(ctx->cmd, "Unknown option"); + return FALSE; + } + options++; + } + return TRUE; +} + +static bool +imap_metadata_parse_entry_names(struct imap_getmetadata_context *ctx, + const struct imap_arg *entries) +{ + const char *value; + + p_array_init(&ctx->entries, ctx->cmd->pool, 4); + for (; !IMAP_ARG_IS_EOL(entries); entries++) { + if (!imap_arg_get_astring(entries, &value)) { + client_send_command_error(ctx->cmd, "Entry isn't astring"); + return FALSE; + } + if (!imap_metadata_verify_entry_name(ctx->cmd, value)) + return FALSE; + + /* names are case-insensitive so we'll always lowercase them */ + value = p_strdup(ctx->cmd->pool, t_str_lcase(value)); + array_append(&ctx->entries, &value, 1); + } + return TRUE; +} + +static void cmd_getmetadata_send_entry(struct imap_getmetadata_context *ctx, + const char *entry) +{ + enum mail_attribute_type type; + struct mail_attribute_value value; + enum mail_error error; + uoff_t value_len; + const char *key; + string_t *str; + + imap_metadata_entry2key(entry, ctx->key_prefix, &type, &key); + if (ctx->key_prefix == NULL && + strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT, + strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT)) == 0) { + /* skip over dovecot's internal attributes. (if key_prefix + isn't NULL, we're getting server metadata, which is handled + inside the private metadata.) */ + return; + } + + if (mailbox_attribute_get_stream(ctx->trans, type, key, &value) < 0) { + (void)mailbox_get_last_error(ctx->box, &error); + if (error != MAIL_ERROR_NOTFOUND && error != MAIL_ERROR_PERM) { + client_send_untagged_storage_error(ctx->cmd->client, + mailbox_get_storage(ctx->box)); + ctx->failed = TRUE; + } + } + if (value.value != NULL) + value_len = strlen(value.value); + else if (value.value_stream != NULL) { + if (i_stream_get_size(value.value_stream, TRUE, &value_len) < 0) { + i_error("GETMETADATA %s: i_stream_get_size(%s) failed: %s", entry, + i_stream_get_name(value.value_stream), + i_stream_get_error(value.value_stream)); + i_stream_unref(&value.value_stream); + ctx->failed = TRUE; + return; + } + } else { + /* skip nonexistent entries */ + return; + } + + if (value_len > ctx->maxsize) { + /* value length is larger than specified MAXSIZE, + skip this entry */ + if (ctx->largest_seen_size < value_len) + ctx->largest_seen_size = value_len; + if (value.value_stream != NULL) + i_stream_unref(&value.value_stream); + return; + } + + str = t_str_new(64); + if (!ctx->first_entry_sent) { + ctx->first_entry_sent = TRUE; + str_append(str, "* METADATA "); + imap_append_astring(str, mailbox_get_vname(ctx->box)); + str_append(str, " ("); + + /* nothing can be sent until untagged METADATA is finished */ + ctx->cmd->client->output_cmd_lock = ctx->cmd; + } else { + str_append_c(str, ' '); + } + imap_append_astring(str, entry); + if (value.value != NULL) { + str_printfa(str, " {%"PRIuUOFF_T"}\r\n%s", value_len, value.value); + o_stream_send(ctx->cmd->client->output, str_data(str), str_len(str)); + } else { + str_printfa(str, " ~{%"PRIuUOFF_T"}\r\n", value_len); + o_stream_send(ctx->cmd->client->output, str_data(str), str_len(str)); + + ctx->cur_stream_offset = 0; + ctx->cur_stream_size = value_len; + ctx->cur_stream = value.value_stream; + } +} + +static bool +cmd_getmetadata_stream_continue(struct imap_getmetadata_context *ctx) +{ + off_t ret; + + o_stream_set_max_buffer_size(ctx->cmd->client->output, 0); + ret = o_stream_send_istream(ctx->cmd->client->output, ctx->cur_stream); + o_stream_set_max_buffer_size(ctx->cmd->client->output, (size_t)-1); + + if (ret > 0) + ctx->cur_stream_offset += ret; + + if (ctx->cur_stream_offset == ctx->cur_stream_size) { + /* finished */ + return TRUE; + } + if (ctx->cur_stream->stream_errno != 0) { + i_error("read(%s) failed: %s", + i_stream_get_name(ctx->cur_stream), + i_stream_get_error(ctx->cur_stream)); + client_disconnect(ctx->cmd->client, + "Internal GETMETADATA failure"); + return -1; + } + if (!i_stream_have_bytes_left(ctx->cur_stream)) { + /* Input stream gave less data than expected */ + i_error("read(%s): GETMETADATA stream had less data than expected", + i_stream_get_name(ctx->cur_stream)); + client_disconnect(ctx->cmd->client, + "Internal GETMETADATA failure"); + return -1; + } + o_stream_set_flush_pending(ctx->cmd->client->output, TRUE); + return FALSE; +} + +static int cmd_getmetadata_send_entry_tree(struct imap_getmetadata_context *ctx, + const char *entry) +{ + const char *key; + enum mail_attribute_type type; + + if (o_stream_get_buffer_used_size(ctx->cmd->client->output) >= + CLIENT_OUTPUT_OPTIMAL_SIZE) { + if (o_stream_flush(ctx->cmd->client->output) <= 0) { + o_stream_set_flush_pending(ctx->cmd->client->output, TRUE); + return 0; + } + } + + if (ctx->iter != NULL) { + /* DEPTH iteration */ + do { + key = mailbox_attribute_iter_next(ctx->iter); + if (key == NULL) { + /* iteration finished, get to the next entry */ + if (mailbox_attribute_iter_deinit(&ctx->iter) < 0) { + client_send_untagged_storage_error(ctx->cmd->client, + mailbox_get_storage(ctx->box)); + ctx->failed = TRUE; + } + return -1; + } + } while (ctx->depth == 1 && strchr(key, '/') != NULL); + entry = t_strconcat(str_c(ctx->iter_entry_prefix), key, NULL); + } + cmd_getmetadata_send_entry(ctx, entry); + + if (ctx->cur_stream != NULL) { + if (!cmd_getmetadata_stream_continue(ctx)) + return 0; + i_stream_unref(&ctx->cur_stream); + } + + if (ctx->iter != NULL) { + /* already iterating the entry */ + return 1; + } else if (ctx->depth == 0) { + /* no iteration for the entry */ + return -1; + } else { + /* we just sent the entry root. iterate its children. */ + str_truncate(ctx->iter_entry_prefix, 0); + str_append(ctx->iter_entry_prefix, entry); + str_append_c(ctx->iter_entry_prefix, '/'); From dovecot at dovecot.org Sat Nov 2 22:32:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 22:32:36 +0200 Subject: dovecot-2.2: lib-imap: NIL as astring is the same as "NIL" Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b631a76e8992 changeset: 16917:b631a76e8992 user: Timo Sirainen date: Sat Nov 02 22:30:42 2013 +0200 description: lib-imap: NIL as astring is the same as "NIL" This is a bit klugdy but works as long as all callers use imap_arg_get_astring() instead of trying to handle it themselves. diffstat: src/lib-imap/imap-arg.c | 6 ++++++ src/lib-imap/imap-arg.h | 4 +++- 2 files changed, 9 insertions(+), 1 deletions(-) diffs (30 lines): diff -r 0a08efeb3f40 -r b631a76e8992 src/lib-imap/imap-arg.c --- a/src/lib-imap/imap-arg.c Sat Nov 02 21:29:39 2013 +0200 +++ b/src/lib-imap/imap-arg.c Sat Nov 02 22:30:42 2013 +0200 @@ -32,6 +32,12 @@ bool imap_arg_get_astring(const struct imap_arg *arg, const char **str_r) { + if (arg->type == IMAP_ARG_NIL) { + /* RFC 3501 4.5. specifies that NIL is the same as "NIL" when + reading astring. */ + *str_r = "NIL"; + return TRUE; + } if (!IMAP_ARG_IS_ASTRING(arg)) return FALSE; diff -r 0a08efeb3f40 -r b631a76e8992 src/lib-imap/imap-arg.h --- a/src/lib-imap/imap-arg.h Sat Nov 02 21:29:39 2013 +0200 +++ b/src/lib-imap/imap-arg.h Sat Nov 02 22:30:42 2013 +0200 @@ -60,7 +60,9 @@ unsigned int literal8:1; /* BINARY literal8 used */ }; -/* RFC 3501's astring type */ +/* RFC 3501's astring type. Note that this doesn't return TRUE for + IMAP_ARG_NIL, although it should be treated the same as "NIL" string when + reading an astring. */ #define IMAP_ARG_TYPE_IS_ASTRING(type) \ ((type) == IMAP_ARG_ATOM || \ (type) == IMAP_ARG_STRING || \ From dovecot at dovecot.org Sat Nov 2 22:32:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 22:32:36 +0200 Subject: dovecot-2.2: imap: Fixed GETMETADATA NIL to treat it as "NIL" Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/081b1ba65fd4 changeset: 16918:081b1ba65fd4 user: Timo Sirainen date: Sat Nov 02 22:31:14 2013 +0200 description: imap: Fixed GETMETADATA NIL to treat it as "NIL" diffstat: src/imap/cmd-getmetadata.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r b631a76e8992 -r 081b1ba65fd4 src/imap/cmd-getmetadata.c --- a/src/imap/cmd-getmetadata.c Sat Nov 02 22:30:42 2013 +0200 +++ b/src/imap/cmd-getmetadata.c Sat Nov 02 22:31:14 2013 +0200 @@ -327,7 +327,7 @@ struct imap_getmetadata_context *ctx; struct mail_namespace *ns; const struct imap_arg *args, *options, *entries; - const char *mailbox; + const char *mailbox, *entry_name; if (!client_read_args(cmd, 0, 0, &args)) return FALSE; @@ -351,7 +351,7 @@ return TRUE; } if (!imap_arg_get_list(&args[2], &entries)) { - if (!IMAP_ARG_IS_ASTRING(&args[2]) || + if (!imap_arg_get_astring(&args[2], &entry_name) || !IMAP_ARG_IS_EOL(&args[3])) { client_send_command_error(cmd, "Invalid arguments."); return TRUE; From dovecot at dovecot.org Sat Nov 2 22:32:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 22:32:36 +0200 Subject: dovecot-2.2: lib-imap: When writing "NIL" as astring, write it a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/720c7d01ad8a changeset: 16919:720c7d01ad8a user: Timo Sirainen date: Sat Nov 02 22:32:23 2013 +0200 description: lib-imap: When writing "NIL" as astring, write it always as a "quoted". diffstat: src/lib-imap/imap-quote.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 081b1ba65fd4 -r 720c7d01ad8a src/lib-imap/imap-quote.c --- a/src/lib-imap/imap-quote.c Sat Nov 02 22:31:14 2013 +0200 +++ b/src/lib-imap/imap-quote.c Sat Nov 02 22:32:23 2013 +0200 @@ -34,7 +34,8 @@ return; } } - if (i == 0) + /* don't mix up NIL and "NIL"! */ + if (i == 0 || strcasecmp(src, "NIL") == 0) imap_append_string(dest, src); else str_append(dest, src); From dovecot at dovecot.org Sat Nov 2 22:42:43 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 22:42:43 +0200 Subject: dovecot-2.2: lib-imap: Fixed NIL astring to not lose its case-se... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1d7e2cee2c4b changeset: 16920:1d7e2cee2c4b user: Timo Sirainen date: Sat Nov 02 22:42:32 2013 +0200 description: lib-imap: Fixed NIL astring to not lose its case-sensitivity. diffstat: src/lib-imap/imap-arg.c | 10 +++------- src/lib-imap/imap-parser.c | 8 +++++--- 2 files changed, 8 insertions(+), 10 deletions(-) diffs (43 lines): diff -r 720c7d01ad8a -r 1d7e2cee2c4b src/lib-imap/imap-arg.c --- a/src/lib-imap/imap-arg.c Sat Nov 02 22:32:23 2013 +0200 +++ b/src/lib-imap/imap-arg.c Sat Nov 02 22:42:32 2013 +0200 @@ -32,13 +32,9 @@ bool imap_arg_get_astring(const struct imap_arg *arg, const char **str_r) { - if (arg->type == IMAP_ARG_NIL) { - /* RFC 3501 4.5. specifies that NIL is the same as "NIL" when - reading astring. */ - *str_r = "NIL"; - return TRUE; - } - if (!IMAP_ARG_IS_ASTRING(arg)) + /* RFC 3501 4.5. specifies that NIL is the same as "NIL" when + reading astring. */ + if (!IMAP_ARG_IS_ASTRING(arg) && arg->type != IMAP_ARG_NIL) return FALSE; *str_r = arg->_data.str; diff -r 720c7d01ad8a -r 1d7e2cee2c4b src/lib-imap/imap-parser.c --- a/src/lib-imap/imap-parser.c Sat Nov 02 22:32:23 2013 +0200 +++ b/src/lib-imap/imap-parser.c Sat Nov 02 22:42:32 2013 +0200 @@ -234,14 +234,16 @@ case ARG_PARSE_ATOM: case ARG_PARSE_TEXT: if (size == 3 && i_memcasecmp(data, "NIL", 3) == 0) { - /* NIL argument */ + /* NIL argument. it might be an actual NIL, but if + we're reading astring, it's an atom and we can't + lose its case. */ arg->type = IMAP_ARG_NIL; } else { /* simply save the string */ arg->type = IMAP_ARG_ATOM; - arg->_data.str = imap_parser_strdup(parser, data, size); - arg->str_len = size; } + arg->_data.str = imap_parser_strdup(parser, data, size); + arg->str_len = size; break; case ARG_PARSE_STRING: /* data is quoted and may contain escapes. */ From dovecot at dovecot.org Sat Nov 2 22:57:40 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 02 Nov 2013 22:57:40 +0200 Subject: dovecot-2.2: lib-storage: Fixed parsing NILs in search parameters. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1e75d6c90c4a changeset: 16921:1e75d6c90c4a user: Timo Sirainen date: Sat Nov 02 22:57:27 2013 +0200 description: lib-storage: Fixed parsing NILs in search parameters. diffstat: src/lib-storage/mail-search-parser-imap.c | 4 ---- 1 files changed, 0 insertions(+), 4 deletions(-) diffs (21 lines): diff -r 1d7e2cee2c4b -r 1e75d6c90c4a src/lib-storage/mail-search-parser-imap.c --- a/src/lib-storage/mail-search-parser-imap.c Sat Nov 02 22:42:32 2013 +0200 +++ b/src/lib-storage/mail-search-parser-imap.c Sat Nov 02 22:57:27 2013 +0200 @@ -26,8 +26,6 @@ switch (arg->type) { case IMAP_ARG_NIL: - _parser->error = "Unexpected NIL"; - return -1; case IMAP_ARG_ATOM: *key_r = imap_arg_as_astring(arg); break; @@ -67,8 +65,6 @@ switch (arg->type) { case IMAP_ARG_NIL: - _parser->error = "Unexpected NIL"; - return -1; case IMAP_ARG_ATOM: case IMAP_ARG_STRING: case IMAP_ARG_LITERAL: From dovecot at dovecot.org Sun Nov 3 01:12:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 Nov 2013 01:12:35 +0200 Subject: dovecot-2.2: lib-imap: imap_arg_get_atom() should also treat NIL... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/55e03b34ae73 changeset: 16922:55e03b34ae73 user: Timo Sirainen date: Sun Nov 03 01:12:24 2013 +0200 description: lib-imap: imap_arg_get_atom() should also treat NIL as valid atom. diffstat: src/lib-imap/imap-arg.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 1e75d6c90c4a -r 55e03b34ae73 src/lib-imap/imap-arg.c --- a/src/lib-imap/imap-arg.c Sat Nov 02 22:57:27 2013 +0200 +++ b/src/lib-imap/imap-arg.c Sun Nov 03 01:12:24 2013 +0200 @@ -5,7 +5,7 @@ bool imap_arg_get_atom(const struct imap_arg *arg, const char **str_r) { - if (arg->type != IMAP_ARG_ATOM) + if (arg->type != IMAP_ARG_ATOM && arg->type != IMAP_ARG_NIL) return FALSE; *str_r = arg->_data.str; From pigeonhole at rename-it.nl Sun Nov 3 03:03:22 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 03 Nov 2013 02:03:22 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Added skeletons for mboxmetad... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/58e6a73e5bc6 changeset: 1813:58e6a73e5bc6 user: Stephan Bosch date: Sun Nov 03 02:02:56 2013 +0100 description: lib-sieve: Added skeletons for mboxmetadata and servermetadata extensions. This is still an unfinished feature. diffstat: configure.ac | 1 + src/lib-sieve/Makefile.am | 3 +- src/lib-sieve/plugins/Makefile.am | 2 +- src/lib-sieve/plugins/metadata/Makefile.am | 24 + src/lib-sieve/plugins/metadata/ext-metadata-common.h | 39 ++ src/lib-sieve/plugins/metadata/ext-metadata.c | 83 +++++ src/lib-sieve/plugins/metadata/tst-metadata.c | 286 +++++++++++++++++++ src/lib-sieve/plugins/metadata/tst-metadataexists.c | 246 ++++++++++++++++ src/lib-sieve/sieve-extensions.c | 4 +- 9 files changed, 685 insertions(+), 3 deletions(-) diffs (truncated from 753 to 300 lines): diff -r ab3638d7aa4a -r 58e6a73e5bc6 configure.ac --- a/configure.ac Fri Oct 11 07:57:21 2013 +0200 +++ b/configure.ac Sun Nov 03 02:02:56 2013 +0100 @@ -120,6 +120,7 @@ src/lib-sieve/plugins/spamvirustest/Makefile src/lib-sieve/plugins/ihave/Makefile src/lib-sieve/plugins/editheader/Makefile +src/lib-sieve/plugins/metadata/Makefile src/lib-sieve/plugins/vnd.dovecot/Makefile src/lib-sieve/plugins/vnd.dovecot/debug/Makefile src/lib-sieve/plugins/vnd.dovecot/duplicate/Makefile diff -r ab3638d7aa4a -r 58e6a73e5bc6 src/lib-sieve/Makefile.am --- a/src/lib-sieve/Makefile.am Fri Oct 11 07:57:21 2013 +0200 +++ b/src/lib-sieve/Makefile.am Sun Nov 03 02:02:56 2013 +0100 @@ -42,7 +42,8 @@ cmp-i-ascii-casemap.c if BUILD_UNFINISHED -unfinished_plugins = +unfinished_plugins = \ + $(extdir)/metadata/libsieve_ext_metadata.la endif # These are not actual plugins just yet... diff -r ab3638d7aa4a -r 58e6a73e5bc6 src/lib-sieve/plugins/Makefile.am --- a/src/lib-sieve/plugins/Makefile.am Fri Oct 11 07:57:21 2013 +0200 +++ b/src/lib-sieve/plugins/Makefile.am Sun Nov 03 02:02:56 2013 +0100 @@ -1,5 +1,5 @@ if BUILD_UNFINISHED -UNFINISHED = +UNFINISHED = metadata endif SUBDIRS = \ diff -r ab3638d7aa4a -r 58e6a73e5bc6 src/lib-sieve/plugins/metadata/Makefile.am --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/metadata/Makefile.am Sun Nov 03 02:02:56 2013 +0100 @@ -0,0 +1,24 @@ + +noinst_LTLIBRARIES = libsieve_ext_metadata.la + +libsieve_ext_metadata_la_LDFLAGS = -module -avoid-version + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib-sieve \ + -I$(top_srcdir)/src/lib-sieve/plugins/variables \ + $(LIBDOVECOT_INCLUDE) + +tests = \ + tst-metadata.c \ + tst-metadataexists.c + +extensions = \ + ext-metadata.c + +libsieve_ext_metadata_la_SOURCES = \ + $(tests) \ + $(extensions) + +noinst_HEADERS = \ + ext-metadata-common.h + diff -r ab3638d7aa4a -r 58e6a73e5bc6 src/lib-sieve/plugins/metadata/ext-metadata-common.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/metadata/ext-metadata-common.h Sun Nov 03 02:02:56 2013 +0100 @@ -0,0 +1,39 @@ +/* Copyright (c) 2002-2013 Pigeonhole authors, see the included COPYING file + */ + +#ifndef __EXT_METADATA_COMMON_H +#define __EXT_METADATA_COMMON_H + +#include "sieve-common.h" + +/* + * Extension + */ + +extern const struct sieve_extension_def mboxmetadata_extension; +extern const struct sieve_extension_def servermetadata_extension; + +/* + * Commands + */ + +extern const struct sieve_command_def metadata_test; +extern const struct sieve_command_def servermetadata_test; +extern const struct sieve_command_def metadataexists_test; +extern const struct sieve_command_def servermetadataexists_test; + +/* + * Operations + */ + +enum ext_metadata_opcode { + EXT_METADATA_OPERATION_METADATA, + EXT_METADATA_OPERATION_METADATAEXISTS +}; + +extern const struct sieve_operation_def metadata_operation; +extern const struct sieve_operation_def servermetadata_operation; +extern const struct sieve_operation_def metadataexists_operation; +extern const struct sieve_operation_def servermetadataexists_operation; + +#endif /* __EXT_METADATA_COMMON_H */ diff -r ab3638d7aa4a -r 58e6a73e5bc6 src/lib-sieve/plugins/metadata/ext-metadata.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/metadata/ext-metadata.c Sun Nov 03 02:02:56 2013 +0100 @@ -0,0 +1,83 @@ +/* Copyright (c) 2002-2013 Pigeonhole authors, see the included COPYING file + */ + +#include "lib.h" + +#include "sieve-extensions.h" +#include "sieve-commands.h" +#include "sieve-binary.h" + +#include "sieve-validator.h" +#include "sieve-interpreter.h" + +#include "ext-metadata-common.h" + +/* + * Extension mboxmetadata + * ----------------------------- + * + * Authors: Stephan Bosch + * Specification: RFC 5490; Section 3 + * Implementation: skeleton + * Status: development + * + */ + +const struct sieve_operation_def *mboxmetadata_operations[] = { + &metadata_operation, + &metadataexists_operation, +}; + +static bool ext_mboxmetadata_validator_load + (const struct sieve_extension *ext, struct sieve_validator *valdtr); + +const struct sieve_extension_def mboxmetadata_extension = { + .name = "mboxmetadata", + .validator_load = ext_mboxmetadata_validator_load, + SIEVE_EXT_DEFINE_OPERATIONS(mboxmetadata_operations) +}; + +static bool ext_mboxmetadata_validator_load +(const struct sieve_extension *ext, struct sieve_validator *valdtr) +{ + sieve_validator_register_command(valdtr, ext, &metadata_test); + sieve_validator_register_command(valdtr, ext, &metadataexists_test); + + return TRUE; +} + +/* + * Extension servermetadata + * ----------------------------- + * + * Authors: Stephan Bosch + * Specification: RFC 5490; Section 4 + * Implementation: skeleton + * Status: development + * + */ + +const struct sieve_operation_def *servermetadata_operations[] = { + &servermetadata_operation, + &servermetadataexists_operation, +}; + +static bool ext_servermetadata_validator_load + (const struct sieve_extension *ext, struct sieve_validator *valdtr); + +const struct sieve_extension_def servermetadata_extension = { + .name = "servermetadata", + .validator_load = ext_servermetadata_validator_load, + SIEVE_EXT_DEFINE_OPERATIONS(servermetadata_operations) +}; + +static bool ext_servermetadata_validator_load +(const struct sieve_extension *ext, struct sieve_validator *valdtr) +{ + sieve_validator_register_command(valdtr, ext, &servermetadata_test); + sieve_validator_register_command(valdtr, ext, &servermetadataexists_test); + + return TRUE; +} + + diff -r ab3638d7aa4a -r 58e6a73e5bc6 src/lib-sieve/plugins/metadata/tst-metadata.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/metadata/tst-metadata.c Sun Nov 03 02:02:56 2013 +0100 @@ -0,0 +1,286 @@ +/* Copyright (c) 2002-2013 Pigeonhole authors, see the included COPYING file + */ + +#include "sieve-common.h" +#include "sieve-commands.h" +#include "sieve-stringlist.h" +#include "sieve-code.h" +#include "sieve-comparators.h" +#include "sieve-match-types.h" +#include "sieve-validator.h" +#include "sieve-generator.h" +#include "sieve-interpreter.h" +#include "sieve-dump.h" +#include "sieve-match.h" + +#include "ext-metadata-common.h" + +/* + * Test definitions + */ + +/* Forward declarations */ + +static bool tst_metadata_registered + (struct sieve_validator *valdtr, const struct sieve_extension *ext, + struct sieve_command_registration *cmd_reg); +static bool tst_metadata_validate + (struct sieve_validator *valdtr, struct sieve_command *tst); +static bool tst_metadata_generate + (const struct sieve_codegen_env *cgenv, struct sieve_command *cmd); + +/* Metadata test + * + * Syntax: + * metadata [MATCH-TYPE] [COMPARATOR] + * + * + */ + +const struct sieve_command_def metadata_test = { + "metadata", + SCT_TEST, + 3, 0, FALSE, FALSE, + tst_metadata_registered, + NULL, + tst_metadata_validate, + NULL, + tst_metadata_generate, + NULL +}; + +/* Servermetadata test + * + * Syntax: + * servermetadata [MATCH-TYPE] [COMPARATOR] + * + */ + +const struct sieve_command_def servermetadata_test = { + "servermetadata", + SCT_TEST, + 2, 0, FALSE, FALSE, + tst_metadata_registered, + NULL, + tst_metadata_validate, + NULL, + tst_metadata_generate, + NULL +}; + +/* + * Opcode definitions + */ + +static bool tst_metadata_operation_dump + (const struct sieve_dumptime_env *denv, sieve_size_t *address); +static int tst_metadata_operation_execute + (const struct sieve_runtime_env *renv, sieve_size_t *address); + +/* Metadata operation */ + +const struct sieve_operation_def metadata_operation = { + "METADATA", + &mboxmetadata_extension, + EXT_METADATA_OPERATION_METADATA, + tst_metadata_operation_dump, + tst_metadata_operation_execute +}; + +/* Servermetadata operation */ + +const struct sieve_operation_def servermetadata_operation = { + "SERVERMETADATA", + &servermetadata_extension, + EXT_METADATA_OPERATION_METADATA, + tst_metadata_operation_dump, + tst_metadata_operation_execute +}; + +/* + * Test registration + */ + +static bool tst_metadata_registered From dovecot at dovecot.org Sun Nov 3 20:56:37 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 Nov 2013 20:56:37 +0200 Subject: dovecot-2.2: Call lib_atexit() callbacks before destroying the l... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5eae6c491d44 changeset: 16923:5eae6c491d44 user: Timo Sirainen date: Sun Nov 03 20:56:25 2013 +0200 description: Call lib_atexit() callbacks before destroying the lib-master ioloop. diffstat: src/lib-master/master-service.c | 2 ++ src/lib/lib.c | 20 ++++++++++++-------- src/lib/lib.h | 3 +++ 3 files changed, 17 insertions(+), 8 deletions(-) diffs (62 lines): diff -r 55e03b34ae73 -r 5eae6c491d44 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Sun Nov 03 01:12:24 2013 +0200 +++ b/src/lib-master/master-service.c Sun Nov 03 20:56:25 2013 +0200 @@ -760,6 +760,8 @@ pool_unref(&service->set_pool); } lib_signals_deinit(); + /* run atexit callbacks before destroying ioloop */ + lib_atexit_run(); io_loop_destroy(&service->ioloop); if (service->listener_names != NULL) diff -r 55e03b34ae73 -r 5eae6c491d44 src/lib/lib.c --- a/src/lib/lib.c Sun Nov 03 01:12:24 2013 +0200 +++ b/src/lib/lib.c Sun Nov 03 20:56:25 2013 +0200 @@ -53,6 +53,17 @@ array_append(&atexit_callbacks, &callback, 1); } +void lib_atexit_run(void) +{ + lib_atexit_callback_t *const *cbp; + + if (array_is_created(&atexit_callbacks)) { + array_foreach(&atexit_callbacks, cbp) + (**cbp)(); + array_free(&atexit_callbacks); + } +} + void lib_init(void) { struct timeval tv; @@ -68,14 +79,7 @@ void lib_deinit(void) { - lib_atexit_callback_t *const *cbp; - - if (array_is_created(&atexit_callbacks)) { - array_foreach(&atexit_callbacks, cbp) - (**cbp)(); - array_free(&atexit_callbacks); - } - + lib_atexit_run(); ipwd_deinit(); hostpid_deinit(); data_stack_deinit(); diff -r 55e03b34ae73 -r 5eae6c491d44 src/lib/lib.h --- a/src/lib/lib.h Sun Nov 03 01:12:24 2013 +0200 +++ b/src/lib/lib.h Sun Nov 03 20:56:25 2013 +0200 @@ -50,6 +50,9 @@ functions are still available. Also if lib_atexit() is called multiple times to the same callback, it's added only once. */ void lib_atexit(lib_atexit_callback_t *callback); +/* Manually run the atexit callbacks. lib_deinit() also does this if not + explicitly called. */ +void lib_atexit_run(void); void lib_init(void); void lib_deinit(void); From dovecot at dovecot.org Sun Nov 3 21:42:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 Nov 2013 21:42:58 +0200 Subject: dovecot-2.2: auth: Potential crashfix Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f66b0fe68439 changeset: 16924:f66b0fe68439 user: Timo Sirainen date: Sun Nov 03 21:42:47 2013 +0200 description: auth: Potential crashfix diffstat: src/auth/auth-request-handler.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 5eae6c491d44 -r f66b0fe68439 src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Sun Nov 03 20:56:25 2013 +0200 +++ b/src/auth/auth-request-handler.c Sun Nov 03 21:42:47 2013 +0200 @@ -173,7 +173,8 @@ auth_fields_append(request->extra_fields, dest, AUTH_FIELD_FLAG_HIDDEN, 0); - if (strcmp(request->original_username, request->user) != 0) { + if (request->original_username != NULL && + strcmp(request->original_username, request->user) != 0) { auth_str_add_keyvalue(dest, "original_user", request->original_username); } From dovecot at dovecot.org Sun Nov 3 21:44:13 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 Nov 2013 21:44:13 +0200 Subject: dovecot-2.2: auth: Crashfix to "doveadm auth cache flush" when a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a87295904845 changeset: 16925:a87295904845 user: Timo Sirainen date: Sun Nov 03 21:44:06 2013 +0200 description: auth: Crashfix to "doveadm auth cache flush" when auth cache was disabled diffstat: src/auth/auth-master-connection.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r f66b0fe68439 -r a87295904845 src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Sun Nov 03 21:42:47 2013 +0200 +++ b/src/auth/auth-master-connection.c Sun Nov 03 21:44:06 2013 +0200 @@ -150,7 +150,7 @@ if (passdb_cache == NULL) { /* cache disabled */ count = 0; - } if (list[1] == NULL) { + } else if (list[1] == NULL) { /* flush the whole cache */ count = auth_cache_clear(passdb_cache); } else { From dovecot at dovecot.org Sun Nov 3 21:45:46 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 Nov 2013 21:45:46 +0200 Subject: dovecot-2.2: auth-worker: Potential crashfix at deinit Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c4fb4637a732 changeset: 16926:c4fb4637a732 user: Timo Sirainen date: Sun Nov 03 21:45:37 2013 +0200 description: auth-worker: Potential crashfix at deinit diffstat: src/auth/auth-worker-client.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (15 lines): diff -r a87295904845 -r c4fb4637a732 src/auth/auth-worker-client.c --- a/src/auth/auth-worker-client.c Sun Nov 03 21:44:06 2013 +0200 +++ b/src/auth/auth-worker-client.c Sun Nov 03 21:45:37 2013 +0200 @@ -782,8 +782,9 @@ void auth_worker_client_send_success(void) { auth_worker_client_error = FALSE; - if (auth_worker_client != NULL && - auth_worker_client->error_sent) { + if (auth_worker_client == NULL) + return; + if (auth_worker_client->error_sent) { o_stream_nsend_str(auth_worker_client->output, "SUCCESS\n"); auth_worker_client->error_sent = FALSE; } From dovecot at dovecot.org Sun Nov 3 21:47:52 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 Nov 2013 21:47:52 +0200 Subject: dovecot-2.2: Make static analyzer happier. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/27c7f25b1f27 changeset: 16927:27c7f25b1f27 user: Timo Sirainen date: Sun Nov 03 21:47:46 2013 +0200 description: Make static analyzer happier. diffstat: src/doveadm/dsync/dsync-mailbox-tree-sync.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r c4fb4637a732 -r 27c7f25b1f27 src/doveadm/dsync/dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c Sun Nov 03 21:45:37 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Sun Nov 03 21:47:46 2013 +0200 @@ -424,6 +424,9 @@ struct dsync_mailbox_node *parent; const char *name, *other_name; + i_assert(node != NULL); + i_assert(other_node != NULL); + /* move/rename node in the tree, so that its position/name is identical to other_node (in other_tree). temp_node's name is changed to temporary name (i.e. it assumes that node's name becomes temp_node's From dovecot at dovecot.org Sun Nov 3 21:48:37 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 Nov 2013 21:48:37 +0200 Subject: dovecot-2.2: Make static analyzer happier Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fb3c54258b5b changeset: 16928:fb3c54258b5b user: Timo Sirainen date: Sun Nov 03 21:48:24 2013 +0200 description: Make static analyzer happier diffstat: src/lib-http/test-http-response-parser.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 27c7f25b1f27 -r fb3c54258b5b src/lib-http/test-http-response-parser.c --- a/src/lib-http/test-http-response-parser.c Sun Nov 03 21:47:46 2013 +0200 +++ b/src/lib-http/test-http-response-parser.c Sun Nov 03 21:48:24 2013 +0200 @@ -104,6 +104,7 @@ unsigned int pos, response_text_len; int ret = 0; + memset(&response, 0, sizeof(response)); test = &valid_response_parse_tests[i]; response_text = test->response; response_text_len = strlen(response_text); From dovecot at dovecot.org Sun Nov 3 22:08:44 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 Nov 2013 22:08:44 +0200 Subject: dovecot-2.2: Released v2.2.7. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b5b118701aa8 changeset: 16929:b5b118701aa8 user: Timo Sirainen date: Sun Nov 03 22:04:50 2013 +0200 description: Released v2.2.7. diffstat: NEWS | 18 ++++++++++++++++++ configure.ac | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diffs (39 lines): diff -r fb3c54258b5b -r b5b118701aa8 NEWS --- a/NEWS Sun Nov 03 21:48:24 2013 +0200 +++ b/NEWS Sun Nov 03 22:04:50 2013 +0200 @@ -1,3 +1,21 @@ +v2.2.7 2013-11-03 Timo Sirainen + + * Some usage of passdb checkpassword could have been exploitable by + local users. You may need to modify your setup to keep it working. + See http://wiki2.dovecot.org/AuthDatabase/CheckPassword#Security + + + auth: Added ability to truncate values logged by + auth_verbose_passwords (see 10-logging.conf comment) + + mdbox: Added "mdbox_deleted" storage, which can be used to access + messages with refcount=0. For example: doveadm import + mdbox_deleted:~/mdbox "" mailbox inbox subject oops + + ssl-params: Added ssl_dh_parameters_length setting. + - master process was doing a hostname.domain lookup for each created + process, which may have caused a lot of unnecessary DNS lookups. + - dsync: Syncing over 100 messages at once caused problems in some + situations, causing messages to get new UIDs. + - fts-solr: Different Solr hosts for different users didn't work. + v2.2.6 2013-09-25 Timo Sirainen * acl: If public/shared namespace has a shared subscriptions file for diff -r fb3c54258b5b -r b5b118701aa8 configure.ac --- a/configure.ac Sun Nov 03 21:48:24 2013 +0200 +++ b/configure.ac Sun Nov 03 22:04:50 2013 +0200 @@ -2,8 +2,8 @@ # Be sure to update ABI version also if anything changes that might require # recompiling plugins. Most importantly that means if any structs are changed. -AC_INIT([Dovecot],[2.2.6],[dovecot at dovecot.org]) -AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv6($PACKAGE_VERSION)", [Dovecot ABI version]) +AC_INIT([Dovecot],[2.2.7],[dovecot at dovecot.org]) +AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv7($PACKAGE_VERSION)", [Dovecot ABI version]) AC_CONFIG_SRCDIR([src]) From dovecot at dovecot.org Sun Nov 3 22:08:44 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 Nov 2013 22:08:44 +0200 Subject: dovecot-2.2: Added tag 2.2.7 for changeset b5b118701aa8 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f78413ac22c6 changeset: 16930:f78413ac22c6 user: Timo Sirainen date: Sun Nov 03 22:04:50 2013 +0200 description: Added tag 2.2.7 for changeset b5b118701aa8 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r b5b118701aa8 -r f78413ac22c6 .hgtags --- a/.hgtags Sun Nov 03 22:04:50 2013 +0200 +++ b/.hgtags Sun Nov 03 22:04:50 2013 +0200 @@ -110,3 +110,4 @@ 4b3c9c3e4fb8499ca7133a4d229e5eef349995e0 2.2.4 71f0696749ab347119278d7f68a01f81ed584c53 2.2.5 1de9a494cb2520986da97fa73cf87a3781ea2d17 2.2.6 +b5b118701aa83e7b047e53f51aa5a4622ee812df 2.2.7 From dovecot at dovecot.org Sun Nov 3 22:08:44 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 Nov 2013 22:08:44 +0200 Subject: dovecot-2.2: Added signature for changeset b5b118701aa8 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/653d5a81a22e changeset: 16931:653d5a81a22e user: Timo Sirainen date: Sun Nov 03 22:04:53 2013 +0200 description: Added signature for changeset b5b118701aa8 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r f78413ac22c6 -r 653d5a81a22e .hgsigs --- a/.hgsigs Sun Nov 03 22:04:50 2013 +0200 +++ b/.hgsigs Sun Nov 03 22:04:53 2013 +0200 @@ -73,3 +73,4 @@ 4b3c9c3e4fb8499ca7133a4d229e5eef349995e0 0 iEYEABECAAYFAlHI1NMACgkQyUhSUUBVismCOQCbB/qdjOfUBvm0AUGBudvg2TaKbJ0AnRmr8JEV1QnowwA0g0vGF8Hkm6ib 71f0696749ab347119278d7f68a01f81ed584c53 0 iEYEABECAAYFAlH/+4cACgkQyUhSUUBVislEUwCfc7GeL9I5US5yJltYpWe8ZU7MAIkAnRlUGvhpFMBnehwSlUDRsssRCJ+a 1de9a494cb2520986da97fa73cf87a3781ea2d17 0 iEYEABECAAYFAlJCi2cACgkQyUhSUUBVisnoWQCfecKzvsHw/MC/5DDuZXIF/CmkaBUAn2DfADK7V0fIWZ49GPyL+BPbYCXy +b5b118701aa83e7b047e53f51aa5a4622ee812df 0 iEYEABECAAYFAlJ2rGIACgkQyUhSUUBVislFRwCfTgg6BYRGpKAOAvE3hhhIskLwxLUAoJjRlth0ob0ahrqIhg7du4+Z0C3j From dovecot at dovecot.org Sun Nov 3 23:27:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 03 Nov 2013 23:27:25 +0200 Subject: dovecot-2.2: ostream: Mark stream closed before handling its cal... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/10c0aae82d0d changeset: 16932:10c0aae82d0d user: Timo Sirainen date: Sun Nov 03 23:27:15 2013 +0200 description: ostream: Mark stream closed before handling its callback to avoid infinite loops. The callback could call o_stream_copy_error_from_parent(), which in turn would try to close the same ostream again. diffstat: src/lib/ostream.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 653d5a81a22e -r 10c0aae82d0d src/lib/ostream.c --- a/src/lib/ostream.c Sun Nov 03 22:04:53 2013 +0200 +++ b/src/lib/ostream.c Sun Nov 03 23:27:15 2013 +0200 @@ -46,8 +46,8 @@ static void o_stream_close_full(struct ostream *stream, bool close_parents) { if (!stream->closed) { + stream->closed = TRUE; io_stream_close(&stream->real_stream->iostream, close_parents); - stream->closed = TRUE; } if (stream->stream_errno == 0) From dovecot at dovecot.org Mon Nov 4 09:47:29 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 04 Nov 2013 09:47:29 +0200 Subject: dovecot-2.2: ssl-params: Compiler warning fix when building with... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/775b1e025939 changeset: 16933:775b1e025939 user: Timo Sirainen date: Mon Nov 04 09:47:17 2013 +0200 description: ssl-params: Compiler warning fix when building without OpenSSL diffstat: src/ssl-params/ssl-params.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (15 lines): diff -r 10c0aae82d0d -r 775b1e025939 src/ssl-params/ssl-params.c --- a/src/ssl-params/ssl-params.c Sun Nov 03 23:27:15 2013 +0200 +++ b/src/ssl-params/ssl-params.c Mon Nov 04 09:47:17 2013 +0200 @@ -34,8 +34,9 @@ ssl_params_callback_t *callback; }; -static void ssl_params_if_unchanged(const char *path, time_t mtime, - unsigned int ssl_dh_parameters_length) +static void +ssl_params_if_unchanged(const char *path, time_t mtime, + unsigned int ssl_dh_parameters_length ATTR_UNUSED) { const char *temp_path; struct file_lock *lock; From dovecot at dovecot.org Tue Nov 5 13:12:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 05 Nov 2013 13:12:36 +0200 Subject: dovecot-2.2: lib-storage: Added FULLDIRNAME=name to mail_location. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5c876ed0c3e0 changeset: 16934:5c876ed0c3e0 user: Timo Sirainen date: Tue Nov 05 13:12:25 2013 +0200 description: lib-storage: Added FULLDIRNAME=name to mail_location. The difference to DIRNAME is that the name is used also for index and control directories. diffstat: src/lib-storage/list/mailbox-list-fs.c | 25 +++++++++++++++++++------ src/lib-storage/mailbox-list.c | 7 ++++++- src/lib-storage/mailbox-list.h | 4 ++++ 3 files changed, 29 insertions(+), 7 deletions(-) diffs (100 lines): diff -r 775b1e025939 -r 5c876ed0c3e0 src/lib-storage/list/mailbox-list-fs.c --- a/src/lib-storage/list/mailbox-list-fs.c Mon Nov 04 09:47:17 2013 +0200 +++ b/src/lib-storage/list/mailbox-list-fs.c Tue Nov 05 13:12:25 2013 +0200 @@ -46,6 +46,20 @@ return '/'; } +static const char * +fs_list_get_path_to(const struct mailbox_list_settings *set, + const char *root_dir, const char *name) +{ + if (*set->maildir_name != '\0' && set->index_control_use_maildir_name) { + return t_strdup_printf("%s/%s%s/%s", root_dir, + set->mailbox_dir_name, name, + set->maildir_name); + } else { + return t_strdup_printf("%s/%s%s", root_dir, + set->mailbox_dir_name, name); + } +} + static int fs_list_get_path(struct mailbox_list *_list, const char *name, enum mailbox_list_path_type type, const char **path_r) @@ -78,6 +92,8 @@ if (set->alt_dir == NULL) return 0; if (*set->maildir_name != '\0') { + /* maildir_name is for the mailbox, caller is asking + for the directory name */ *path_r = t_strdup_printf("%s/%s%s", set->alt_dir, set->mailbox_dir_name, name); return 1; @@ -93,8 +109,7 @@ break; case MAILBOX_LIST_PATH_TYPE_CONTROL: if (set->control_dir != NULL) { - *path_r = t_strdup_printf("%s/%s%s", set->control_dir, - set->mailbox_dir_name, name); + *path_r = fs_list_get_path_to(set, set->control_dir, name); return 1; } break; @@ -102,16 +117,14 @@ if (set->index_dir != NULL) { if (*set->index_dir == '\0') return 0; - *path_r = t_strdup_printf("%s/%s%s", set->index_dir, - set->mailbox_dir_name, name); + *path_r = fs_list_get_path_to(set, set->index_dir, name); return 1; } break; case MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE: if (set->index_pvt_dir == NULL) return 0; - *path_r = t_strdup_printf("%s/%s%s", set->index_pvt_dir, - set->mailbox_dir_name, name); + *path_r = fs_list_get_path_to(set, set->index_pvt_dir, name); return 1; } diff -r 775b1e025939 -r 5c876ed0c3e0 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Mon Nov 04 09:47:17 2013 +0200 +++ b/src/lib-storage/mailbox-list.c Tue Nov 05 13:12:25 2013 +0200 @@ -167,6 +167,8 @@ p_strdup(list->pool, set->mailbox_dir_name); list->set.alt_dir = p_strdup(list->pool, set->alt_dir); list->set.alt_dir_nocheck = set->alt_dir_nocheck; + list->set.index_control_use_maildir_name = + set->index_control_use_maildir_name; if (*set->mailbox_dir_name == '\0') list->set.mailbox_dir_name = ""; @@ -323,7 +325,10 @@ dest = &set_r->maildir_name; else if (strcmp(key, "MAILBOXDIR") == 0) dest = &set_r->mailbox_dir_name; - else { + else if (strcmp(key, "FULLDIRNAME") == 0) { + set_r->index_control_use_maildir_name = TRUE; + dest = &set_r->maildir_name; + } else { *error_r = t_strdup_printf("Unknown setting: %s", key); return -1; } diff -r 775b1e025939 -r 5c876ed0c3e0 src/lib-storage/mailbox-list.h --- a/src/lib-storage/mailbox-list.h Mon Nov 04 09:47:17 2013 +0200 +++ b/src/lib-storage/mailbox-list.h Tue Nov 05 13:12:25 2013 +0200 @@ -125,6 +125,10 @@ bool utf8; /* Don't check/create the alt-dir symlink. */ bool alt_dir_nocheck; + /* Use maildir_name also for index/control directories. This should + have been the default since the beginning, but for backwards + compatibility it had to be made an option. */ + bool index_control_use_maildir_name; }; struct mailbox_permissions { From dovecot at dovecot.org Tue Nov 5 20:11:22 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 05 Nov 2013 20:11:22 +0200 Subject: dovecot-2.2: lib-mail: Fixed infinite loop in message-parser if ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aa1aede0f7f2 changeset: 16935:aa1aede0f7f2 user: Timo Sirainen date: Tue Nov 05 20:11:11 2013 +0200 description: lib-mail: Fixed infinite loop in message-parser if message ends with --boundary+CR Thanks to Tomasz Potega for finding this. diffstat: src/lib-mail/message-parser.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 5c876ed0c3e0 -r aa1aede0f7f2 src/lib-mail/message-parser.c --- a/src/lib-mail/message-parser.c Tue Nov 05 13:12:25 2013 +0200 +++ b/src/lib-mail/message-parser.c Tue Nov 05 20:11:11 2013 +0200 @@ -403,7 +403,7 @@ } else if (boundary_start == 0) { /* no linefeeds in this block. we can just skip it. */ ret = 0; - if (block_r->data[block_r->size-1] == '\r') { + if (block_r->data[block_r->size-1] == '\r' && !ctx->eof) { /* this may be the beginning of the \r\n--boundary */ block_r->size--; } From dovecot at dovecot.org Wed Nov 6 12:54:16 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 Nov 2013 12:54:16 +0200 Subject: dovecot-2.2: imap: Fixed potential crash if client disconnected ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/672350cdf509 changeset: 16936:672350cdf509 user: Timo Sirainen date: Wed Nov 06 12:53:58 2013 +0200 description: imap: Fixed potential crash if client disconnected during APPEND. diffstat: src/imap/cmd-append.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (31 lines): diff -r aa1aede0f7f2 -r 672350cdf509 src/imap/cmd-append.c --- a/src/imap/cmd-append.c Tue Nov 05 20:11:11 2013 +0200 +++ b/src/imap/cmd-append.c Wed Nov 06 12:53:58 2013 +0200 @@ -60,7 +60,7 @@ str_printfa(str, "Disconnected in APPEND (%u msgs, %u secs", ctx->count, secs); - if (ctx->litinput != NULL) { + if (ctx->literal_size > 0) { str_printfa(str, ", %"PRIuUOFF_T"/%"PRIuUOFF_T" bytes", lit_offset, ctx->literal_size); } @@ -74,6 +74,7 @@ struct client *client = cmd->client; const char *reason; bool finished; + uoff_t lit_offset; i_assert(!client->destroyed); @@ -83,7 +84,9 @@ switch (i_stream_read(client->input)) { case -1: /* disconnected */ - reason = get_disconnect_reason(ctx, ctx->litinput->v_offset); + lit_offset = ctx->litinput == NULL ? 0 : + ctx->litinput->v_offset; + reason = get_disconnect_reason(ctx, lit_offset); cmd_append_finish(cmd->context); /* Reset command so that client_destroy() doesn't try to call cmd_append_continue_message() anymore. */ From dovecot at dovecot.org Wed Nov 6 13:00:33 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 Nov 2013 13:00:33 +0200 Subject: dovecot-2.2: lib-imap-storage: Fixed leaking istream on error co... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0505adfe2d8e changeset: 16937:0505adfe2d8e user: Timo Sirainen date: Wed Nov 06 13:00:13 2013 +0200 description: lib-imap-storage: Fixed leaking istream on error conditions. Caught by Tomasz Pot?ga diffstat: src/lib-imap-storage/imap-msgpart.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 672350cdf509 -r 0505adfe2d8e src/lib-imap-storage/imap-msgpart.c --- a/src/lib-imap-storage/imap-msgpart.c Wed Nov 06 12:53:58 2013 +0200 +++ b/src/lib-imap-storage/imap-msgpart.c Wed Nov 06 13:00:13 2013 +0200 @@ -398,6 +398,7 @@ if (message_skip_virtual(input, virtual_skip, &cr_skipped) < 0) { errinput = i_stream_create_error(errno); i_stream_set_name(errinput, i_stream_get_name(input)); + i_stream_unref(&input); return errinput; } From dovecot at dovecot.org Wed Nov 6 15:30:26 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 Nov 2013 15:30:26 +0200 Subject: dovecot-2.2: mdbox: Fixed race condition when mailbox GUID was l... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fa6466dcc37a changeset: 16938:fa6466dcc37a user: Timo Sirainen date: Wed Nov 06 15:30:15 2013 +0200 description: mdbox: Fixed race condition when mailbox GUID was looked up before mailbox create was finished. diffstat: src/lib-storage/index/dbox-multi/mdbox-storage.c | 18 +++++++++++++++--- 1 files changed, 15 insertions(+), 3 deletions(-) diffs (39 lines): diff -r 0505adfe2d8e -r fa6466dcc37a src/lib-storage/index/dbox-multi/mdbox-storage.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage.c Wed Nov 06 13:00:13 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c Wed Nov 06 15:30:15 2013 +0200 @@ -370,9 +370,19 @@ static int mdbox_mailbox_get_guid(struct mdbox_mailbox *mbox, guid_128_t guid_r) { + const struct mail_index_header *idx_hdr; struct mdbox_index_header hdr; bool need_resize; + int ret = 0; + i_assert(!mbox->creating); + + /* there's a race condition between mkdir and getting the mailbox GUID. + normally this is handled by mdbox syncing, but GUID can be looked up + without syncing. when mbox->creating=TRUE, the errors are hidden + and we'll simply finish the mailbox creation */ + idx_hdr = mail_index_get_header(mbox->box.view); + mbox->creating = idx_hdr->uid_validity == 0 && idx_hdr->next_uid == 1; if (mdbox_read_header(mbox, &hdr, &need_resize) < 0) memset(&hdr, 0, sizeof(hdr)); @@ -380,10 +390,12 @@ /* regenerate it */ if (mdbox_write_index_header(&mbox->box, NULL, NULL) < 0 || mdbox_read_header(mbox, &hdr, &need_resize) < 0) - return -1; + ret = -1; } - memcpy(guid_r, hdr.mailbox_guid, GUID_128_SIZE); - return 0; + if (ret == 0) + memcpy(guid_r, hdr.mailbox_guid, GUID_128_SIZE); + mbox->creating = FALSE; + return ret; } static int From dovecot at dovecot.org Wed Nov 6 16:33:50 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 Nov 2013 16:33:50 +0200 Subject: dovecot-2.2: replicator: Database should be exported (not import... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b013764afe7c changeset: 16939:b013764afe7c user: Timo Sirainen date: Wed Nov 06 16:33:37 2013 +0200 description: replicator: Database should be exported (not imported!) every 15 mins. diffstat: src/replication/replicator/replicator.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r fa6466dcc37a -r b013764afe7c src/replication/replicator/replicator.c --- a/src/replication/replicator/replicator.c Wed Nov 06 15:30:15 2013 +0200 +++ b/src/replication/replicator/replicator.c Wed Nov 06 16:33:37 2013 +0200 @@ -70,7 +70,7 @@ const char *path; path = t_strconcat(service_set->state_dir, "/"REPLICATOR_DB_FNAME, NULL); - (void)replicator_queue_import(queue, path); + (void)replicator_queue_export(queue, path); } static void main_init(void) From dovecot at dovecot.org Wed Nov 6 17:14:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 Nov 2013 17:14:08 +0200 Subject: dovecot-2.2: dsync: Use i_stream_get_error() instead of just err... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/38f404297728 changeset: 16940:38f404297728 user: Timo Sirainen date: Wed Nov 06 17:13:53 2013 +0200 description: dsync: Use i_stream_get_error() instead of just errno in stream error messages. diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 13 +++++++------ src/doveadm/dsync/dsync-mailbox-import.c | 12 +++++++----- 2 files changed, 14 insertions(+), 11 deletions(-) diffs (70 lines): diff -r b013764afe7c -r 38f404297728 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Wed Nov 06 16:33:37 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Wed Nov 06 17:13:53 2013 +0200 @@ -174,8 +174,8 @@ } while (i_stream_read(ibc->value_input) > 0); if (ibc->value_input->eof) { if (ibc->value_input->stream_errno != 0) { - errno = ibc->value_input->stream_errno; - i_error("dsync(%s): read() failed: %m", ibc->name); + i_error("dsync(%s): read() failed: %s", ibc->name, + i_stream_get_error(ibc->value_input)); dsync_ibc_stream_stop(ibc); return -1; } @@ -246,8 +246,9 @@ i_assert(ret == -1); if (ibc->value_output->stream_errno != 0) { - i_error("dsync(%s): read(%s) failed: %m", - ibc->name, i_stream_get_name(ibc->value_output)); + i_error("dsync(%s): read(%s) failed: %s", + ibc->name, i_stream_get_name(ibc->value_output), + i_stream_get_error(ibc->value_output)); dsync_ibc_stream_stop(ibc); return -1; } @@ -371,8 +372,8 @@ return -1; error = t_str_new(128); if (ibc->input->stream_errno != 0) { - errno = ibc->input->stream_errno; - str_printfa(error, "read(%s) failed: %m", ibc->name); + str_printfa(error, "read(%s) failed: %s", ibc->name, + i_stream_get_error(ibc->input)); } else { i_assert(ibc->input->eof); str_printfa(error, "read(%s) failed: EOF", ibc->name); diff -r b013764afe7c -r 38f404297728 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Wed Nov 06 16:33:37 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Wed Nov 06 17:13:53 2013 +0200 @@ -299,12 +299,14 @@ } if (input1->stream_errno != 0) { errno = input1->stream_errno; - i_error("read(%s) failed: %m", i_stream_get_name(input1)); + i_error("read(%s) failed: %s", i_stream_get_name(input1), + i_stream_get_error(input1)); return -1; } if (input2->stream_errno != 0) { errno = input2->stream_errno; - i_error("read(%s) failed: %m", i_stream_get_name(input2)); + i_error("read(%s) failed: %s", i_stream_get_name(input2), + i_stream_get_error(input2)); return -1; } if (size1 == 0 && size2 == 0) @@ -1960,9 +1962,9 @@ i_assert(ret == -1); if (mail->input->stream_errno != 0) { - errno = mail->input->stream_errno; - i_error("Mailbox %s: read(msg input) failed: %m", - mailbox_get_vname(importer->box)); + i_error("Mailbox %s: read(msg input) failed: %s", + mailbox_get_vname(importer->box), + i_stream_get_error(mail->input)); mailbox_save_cancel(&save_ctx); importer->failed = TRUE; } else if (save_failed) { From dovecot at dovecot.org Wed Nov 6 19:03:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 Nov 2013 19:03:25 +0200 Subject: dovecot-2.2: lib-http: Use [io]_stream_get_error() instead of ju... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e67095b8486e changeset: 16941:e67095b8486e user: Timo Sirainen date: Wed Nov 06 19:03:10 2013 +0200 description: lib-http: Use [io]_stream_get_error() instead of just errno strings. diffstat: src/lib-http/http-client-connection.c | 10 ++++++---- src/lib-http/http-client-request.c | 20 ++++++++++++-------- src/lib-http/test-http-client.c | 7 ++++--- 3 files changed, 22 insertions(+), 15 deletions(-) diffs (99 lines): diff -r 38f404297728 -r e67095b8486e src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Wed Nov 06 17:13:53 2013 +0200 +++ b/src/lib-http/http-client-connection.c Wed Nov 06 19:03:10 2013 +0200 @@ -361,7 +361,7 @@ /* retry pending requests if possible */ error = _conn->input == NULL ? "Connection lost" : t_strdup_printf("Connection lost: %s", - strerror(_conn->input->stream_errno)); + i_stream_get_error(_conn->input)); http_client_connection_debug(conn, "%s", error); http_client_connection_retry_requests(conn, HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, error); @@ -646,7 +646,8 @@ t_strdup_printf("Connection lost: read(%s) failed: %s", i_stream_get_name(conn->conn.input), stream_errno != 0 ? - strerror(stream_errno) : "EOF")); + i_stream_get_error(conn->conn.input) : + "EOF")); return; } @@ -681,8 +682,9 @@ if (ret < 0) { http_client_connection_abort_temp_error(&conn, HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, - t_strdup_printf("Connection lost: write(%s) failed: %m", - o_stream_get_name(output))); + t_strdup_printf("Connection lost: write(%s) failed: %s", + o_stream_get_name(output), + o_stream_get_error(output))); } return ret; } diff -r 38f404297728 -r e67095b8486e src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Wed Nov 06 17:13:53 2013 +0200 +++ b/src/lib-http/http-client-request.c Wed Nov 06 19:03:10 2013 +0200 @@ -256,8 +256,9 @@ req->payload_input = input; if ((ret = i_stream_get_size(input, TRUE, &req->payload_size)) <= 0) { if (ret < 0) { - i_error("i_stream_get_size(%s) failed: %m", - i_stream_get_name(input)); + i_error("i_stream_get_size(%s) failed: %s", + i_stream_get_name(input), + i_stream_get_error(input)); } req->payload_size = 0; req->payload_chunked = TRUE; @@ -505,14 +506,16 @@ /* we're in the middle of sending a request, so the connection will also have to be aborted */ errno = req->payload_input->stream_errno; - *error_r = t_strdup_printf("read(%s) failed: %m", - i_stream_get_name(req->payload_input)); + *error_r = t_strdup_printf("read(%s) failed: %s", + i_stream_get_name(req->payload_input), + i_stream_get_error(req->payload_input)); ret = -1; } else if (output->stream_errno != 0) { /* failed to send request */ errno = output->stream_errno; - *error_r = t_strdup_printf("write(%s) failed: %m", - o_stream_get_name(output)); + *error_r = t_strdup_printf("write(%s) failed: %s", + o_stream_get_name(output), + o_stream_get_error(output)); ret = -1; } else { i_assert(ret >= 0); @@ -631,8 +634,9 @@ req->state = HTTP_REQUEST_STATE_PAYLOAD_OUT; o_stream_cork(output); if (o_stream_sendv(output, iov, N_ELEMENTS(iov)) < 0) { - *error_r = t_strdup_printf("write(%s) failed: %m", - o_stream_get_name(output)); + *error_r = t_strdup_printf("write(%s) failed: %s", + o_stream_get_name(output), + o_stream_get_error(output)); ret = -1; } diff -r 38f404297728 -r e67095b8486e src/lib-http/test-http-client.c --- a/src/lib-http/test-http-client.c Wed Nov 06 17:13:53 2013 +0200 +++ b/src/lib-http/test-http-client.c Wed Nov 06 19:03:10 2013 +0200 @@ -31,9 +31,10 @@ i_info("DEBUG: REQUEST: NEED MORE DATA"); /* we will be called again for this request */ } else { - if (req->payload->stream_errno != 0) - i_error("REQUEST PAYLOAD READ ERROR: %m"); - else + if (req->payload->stream_errno != 0) { + i_error("REQUEST PAYLOAD READ ERROR: %s", + i_stream_get_error(req->payload)); + } else i_info("DEBUG: REQUEST: Finished"); io_remove(&req->io); i_stream_unref(&req->payload); From dovecot at dovecot.org Wed Nov 6 23:01:13 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 Nov 2013 23:01:13 +0200 Subject: dovecot-2.2: lib-fs: Added istream-fs-file to lazily read from f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5c72e7d65e6f changeset: 16943:5c72e7d65e6f user: Timo Sirainen date: Wed Nov 06 21:06:15 2013 +0200 description: lib-fs: Added istream-fs-file to lazily read from fs_file. diffstat: src/lib-fs/Makefile.am | 2 + src/lib-fs/istream-fs-file.c | 61 ++++++++++++++++++++++++++++++++++++++++++++ src/lib-fs/istream-fs-file.h | 13 +++++++++ 3 files changed, 76 insertions(+), 0 deletions(-) diffs (101 lines): diff -r 1c9ecb6bcccc -r 5c72e7d65e6f src/lib-fs/Makefile.am --- a/src/lib-fs/Makefile.am Wed Nov 06 21:05:51 2013 +0200 +++ b/src/lib-fs/Makefile.am Wed Nov 06 21:06:15 2013 +0200 @@ -12,6 +12,7 @@ fs-sis.c \ fs-sis-common.c \ fs-sis-queue.c \ + istream-fs-file.c \ istream-metawrap.c \ ostream-metawrap.c \ ostream-cmp.c @@ -20,6 +21,7 @@ fs-api.h \ fs-api-private.h \ fs-sis-common.h \ + istream-fs-file.h \ istream-metawrap.h \ ostream-metawrap.h \ ostream-cmp.h diff -r 1c9ecb6bcccc -r 5c72e7d65e6f src/lib-fs/istream-fs-file.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fs/istream-fs-file.c Wed Nov 06 21:06:15 2013 +0200 @@ -0,0 +1,61 @@ +/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream-private.h" +#include "fs-api-private.h" +#include "istream-fs-file.h" + +struct fs_file_istream { + struct istream_private istream; + struct fs_file *file; +}; + +static void i_stream_fs_file_close(struct iostream_private *stream, + bool close_parent ATTR_UNUSED) +{ + struct fs_file_istream *fstream = (struct fs_file_istream *)stream; + + i_stream_destroy(&fstream->istream.parent); + fs_file_deinit(&fstream->file); +} + +static ssize_t i_stream_fs_file_read(struct istream_private *stream) +{ + struct fs_file_istream *fstream = (struct fs_file_istream *)stream; + struct istream *input; + + if (fstream->istream.parent == NULL) { + input = fs_read_stream(fstream->file, + fstream->istream.max_buffer_size); + i_stream_init_parent(stream, input); + i_stream_unref(&input); + } + + i_stream_seek(stream->parent, stream->parent_start_offset + + stream->istream.v_offset); + return i_stream_read_copy_from_parent(&stream->istream); +} + +struct istream * +i_stream_create_fs_file(struct fs_file **file, size_t max_buffer_size) +{ + struct fs_file_istream *fstream; + struct istream *input; + + fstream = i_new(struct fs_file_istream, 1); + fstream->file = *file; + fstream->istream.iostream.close = i_stream_fs_file_close; + fstream->istream.max_buffer_size = max_buffer_size; + fstream->istream.read = i_stream_fs_file_read; + fstream->istream.stream_size_passthrough = TRUE; + + fstream->istream.istream.blocking = + ((*file)->flags & FS_OPEN_FLAG_ASYNC) == 0; + fstream->istream.istream.seekable = + ((*file)->flags & FS_OPEN_FLAG_SEEKABLE) != 0; + + input = i_stream_create(&fstream->istream, NULL, -1); + i_stream_set_name(input, fs_file_path(*file)); + *file = NULL; + return input; +} diff -r 1c9ecb6bcccc -r 5c72e7d65e6f src/lib-fs/istream-fs-file.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fs/istream-fs-file.h Wed Nov 06 21:06:15 2013 +0200 @@ -0,0 +1,13 @@ +#ifndef ISTREAM_FS_FILE_H +#define ISTREAM_FS_FILE_H + +struct fs_file; + +/* Open the given file only when something is actually tried to be read from + the stream. The file is automatically deinitialized when the stream is + destroyed (which is why it's also set to NULL so it's not accidentally + double-freed). */ +struct istream * +i_stream_create_fs_file(struct fs_file **file, size_t max_buffer_size); + +#endif From dovecot at dovecot.org Wed Nov 6 23:01:13 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 Nov 2013 23:01:13 +0200 Subject: dovecot-2.2: istream: Added i_stream_init_parent() to lazily ini... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1c9ecb6bcccc changeset: 16942:1c9ecb6bcccc user: Timo Sirainen date: Wed Nov 06 21:05:51 2013 +0200 description: istream: Added i_stream_init_parent() to lazily initialize the parent stream. diffstat: src/lib/istream-private.h | 3 +++ src/lib/istream.c | 29 +++++++++++++++++------------ 2 files changed, 20 insertions(+), 12 deletions(-) diffs (56 lines): diff -r e67095b8486e -r 1c9ecb6bcccc src/lib/istream-private.h --- a/src/lib/istream-private.h Wed Nov 06 19:03:10 2013 +0200 +++ b/src/lib/istream-private.h Wed Nov 06 21:05:51 2013 +0200 @@ -52,6 +52,9 @@ struct istream * ATTR_NOWARN_UNUSED_RESULT i_stream_create(struct istream_private *stream, struct istream *parent, int fd) ATTR_NULL(2); +/* Initialize parent lazily after i_stream_create() has already been called. */ +void i_stream_init_parent(struct istream_private *_stream, + struct istream *parent); void i_stream_compress(struct istream_private *stream); void i_stream_grow_buffer(struct istream_private *stream, size_t bytes); diff -r e67095b8486e -r 1c9ecb6bcccc src/lib/istream.c --- a/src/lib/istream.c Wed Nov 06 19:03:10 2013 +0200 +++ b/src/lib/istream.c Wed Nov 06 21:05:51 2013 +0200 @@ -703,22 +703,27 @@ return 1; } +void i_stream_init_parent(struct istream_private *_stream, + struct istream *parent) +{ + _stream->access_counter = parent->real_stream->access_counter; + _stream->parent = parent; + _stream->parent_start_offset = parent->v_offset; + _stream->parent_expected_offset = parent->v_offset; + _stream->abs_start_offset = parent->v_offset + + parent->real_stream->abs_start_offset; + /* if parent stream is an istream-error, copy the error */ + _stream->istream.stream_errno = parent->stream_errno; + _stream->istream.eof = parent->eof; + i_stream_ref(parent); +} + struct istream * i_stream_create(struct istream_private *_stream, struct istream *parent, int fd) { _stream->fd = fd; - if (parent != NULL) { - _stream->access_counter = parent->real_stream->access_counter; - _stream->parent = parent; - _stream->parent_start_offset = parent->v_offset; - _stream->parent_expected_offset = parent->v_offset; - _stream->abs_start_offset = parent->v_offset + - parent->real_stream->abs_start_offset; - /* if parent stream is an istream-error, copy the error */ - _stream->istream.stream_errno = parent->stream_errno; - _stream->istream.eof = parent->eof; - i_stream_ref(parent); - } + if (parent != NULL) + i_stream_init_parent(_stream, parent); _stream->istream.real_stream = _stream; if (_stream->iostream.close == NULL) From dovecot at dovecot.org Wed Nov 6 23:01:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 Nov 2013 23:01:14 +0200 Subject: dovecot-2.2: lib-storage: Added mailbox_list_init_fs() to easily... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9d5bfd8e50c9 changeset: 16944:9d5bfd8e50c9 user: Timo Sirainen date: Wed Nov 06 21:08:42 2013 +0200 description: lib-storage: Added mailbox_list_init_fs() to easily initialize fs. Use it for dbox's attachment_fs. diffstat: src/lib-storage/Makefile.am | 2 ++ src/lib-storage/index/dbox-common/dbox-storage.c | 11 ++++------- src/lib-storage/mailbox-list-private.h | 5 +++++ src/lib-storage/mailbox-list.c | 24 ++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 7 deletions(-) diffs (100 lines): diff -r 5c72e7d65e6f -r 9d5bfd8e50c9 src/lib-storage/Makefile.am --- a/src/lib-storage/Makefile.am Wed Nov 06 21:06:15 2013 +0200 +++ b/src/lib-storage/Makefile.am Wed Nov 06 21:08:42 2013 +0200 @@ -7,6 +7,8 @@ -I$(top_srcdir)/src/lib-test \ -I$(top_srcdir)/src/lib-auth \ -I$(top_srcdir)/src/lib-dict \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -I$(top_srcdir)/src/lib-fs \ -I$(top_srcdir)/src/lib-master \ -I$(top_srcdir)/src/lib-settings \ -I$(top_srcdir)/src/lib-charset \ diff -r 5c72e7d65e6f -r 9d5bfd8e50c9 src/lib-storage/index/dbox-common/dbox-storage.c --- a/src/lib-storage/index/dbox-common/dbox-storage.c Wed Nov 06 21:06:15 2013 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-storage.c Wed Nov 06 21:08:42 2013 +0200 @@ -94,12 +94,8 @@ { struct dbox_storage *storage = (struct dbox_storage *)_storage; const struct mail_storage_settings *set = _storage->set; - struct fs_settings fs_set; const char *error; - memset(&fs_set, 0, sizeof(fs_set)); - fs_set.temp_file_prefix = mailbox_list_get_global_temp_prefix(ns->list); - if (*set->mail_attachment_fs != '\0') { const char *name, *args, *dir; @@ -121,9 +117,10 @@ dir = mail_user_home_expand(_storage->user, set->mail_attachment_dir); storage->attachment_dir = p_strdup(_storage->pool, dir); - fs_set.root_path = storage->attachment_dir; - if (fs_init(name, args, &fs_set, &storage->attachment_fs, - &error) < 0) { + + if (mailbox_list_init_fs(ns->list, name, args, + storage->attachment_dir, + &storage->attachment_fs, &error) < 0) { *error_r = t_strdup_printf("mail_attachment_fs: %s", error); return -1; diff -r 5c72e7d65e6f -r 9d5bfd8e50c9 src/lib-storage/mailbox-list-private.h --- a/src/lib-storage/mailbox-list-private.h Wed Nov 06 21:06:15 2013 +0200 +++ b/src/lib-storage/mailbox-list-private.h Wed Nov 06 21:08:42 2013 +0200 @@ -22,6 +22,7 @@ struct stat; struct dirent; +struct fs; struct imap_match_glob; struct mailbox_tree_context; struct mailbox_list_notify; @@ -210,4 +211,8 @@ void mailbox_list_set_internal_error(struct mailbox_list *list); bool mailbox_list_set_error_from_errno(struct mailbox_list *list); +int mailbox_list_init_fs(struct mailbox_list *list, const char *driver, + const char *args, const char *root_dir, + struct fs **fs_r, const char **error_r); + #endif diff -r 5c72e7d65e6f -r 9d5bfd8e50c9 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Wed Nov 06 21:06:15 2013 +0200 +++ b/src/lib-storage/mailbox-list.c Wed Nov 06 21:08:42 2013 +0200 @@ -12,6 +12,8 @@ #include "time-util.h" #include "unichar.h" #include "settings-parser.h" +#include "iostream-ssl.h" +#include "fs-api.h" #include "imap-utf7.h" #include "mailbox-log.h" #include "mailbox-tree.h" @@ -1803,3 +1805,25 @@ mailbox_list_set_error(list, error, error_string); return TRUE; } + +int mailbox_list_init_fs(struct mailbox_list *list, const char *driver, + const char *args, const char *root_dir, + struct fs **fs_r, const char **error_r) +{ + struct fs_settings fs_set; + struct ssl_iostream_settings ssl_set; + + memset(&ssl_set, 0, sizeof(ssl_set)); + ssl_set.ca_dir = list->mail_set->ssl_client_ca_dir; + ssl_set.ca_file = list->mail_set->ssl_client_ca_file; + + memset(&fs_set, 0, sizeof(fs_set)); + fs_set.temp_file_prefix = mailbox_list_get_global_temp_prefix(list); + fs_set.base_dir = list->ns->user->set->base_dir; + fs_set.temp_dir = list->ns->user->set->mail_temp_dir; + fs_set.ssl_client_set = &ssl_set; + fs_set.root_path = root_dir; + fs_set.debug = list->ns->user->mail_debug; + + return fs_init(driver, args, &fs_set, fs_r, error_r); +} From dovecot at dovecot.org Wed Nov 6 23:01:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 Nov 2013 23:01:14 +0200 Subject: dovecot-2.2: dbox: Use lib-fs API also for reading the attachments. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/883ba4a47829 changeset: 16945:883ba4a47829 user: Timo Sirainen date: Wed Nov 06 21:10:22 2013 +0200 description: dbox: Use lib-fs API also for reading the attachments. diffstat: src/lib-storage/index/dbox-common/dbox-attachment.c | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-) diffs (31 lines): diff -r 9d5bfd8e50c9 -r 883ba4a47829 src/lib-storage/index/dbox-common/dbox-attachment.c --- a/src/lib-storage/index/dbox-common/dbox-attachment.c Wed Nov 06 21:08:42 2013 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-attachment.c Wed Nov 06 21:10:22 2013 +0200 @@ -3,6 +3,8 @@ #include "lib.h" #include "istream.h" #include "str.h" +#include "fs-api.h" +#include "istream-fs-file.h" #include "istream-attachment-connector.h" #include "dbox-file.h" #include "dbox-save.h" @@ -143,6 +145,7 @@ ARRAY_TYPE(mail_attachment_extref) extrefs_arr; const struct mail_attachment_extref *extref; struct istream_attachment_connector *conn; + struct fs_file *fsfile; struct istream *input; const char *path, *path_suffix; uoff_t msg_size; @@ -170,7 +173,9 @@ array_foreach(&extrefs_arr, extref) { path = t_strdup_printf("%s/%s%s", file->storage->attachment_dir, extref->path, path_suffix); - input = i_stream_create_file(path, IO_BLOCK_SIZE); + fsfile = fs_file_init(file->storage->attachment_fs, path, + FS_OPEN_MODE_READONLY); + input = i_stream_create_fs_file(&fsfile, IO_BLOCK_SIZE); ret = istream_attachment_connector_add(conn, input, extref->start_offset, extref->size, From dovecot at dovecot.org Wed Nov 6 23:01:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 06 Nov 2013 23:01:14 +0200 Subject: dovecot-2.2: lib-storage: Moved more of dbox attachments code to... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0ed18f77ff08 changeset: 16946:0ed18f77ff08 user: Timo Sirainen date: Wed Nov 06 23:00:58 2013 +0200 description: lib-storage: Moved more of dbox attachments code to generic code. diffstat: src/lib-storage/index/dbox-common/dbox-attachment.c | 163 +------------------- src/lib-storage/index/dbox-common/dbox-attachment.h | 4 - src/lib-storage/index/dbox-multi/mdbox-purge.c | 8 +- src/lib-storage/index/dbox-single/sdbox-copy.c | 2 +- src/lib-storage/index/dbox-single/sdbox-file.c | 2 +- src/lib-storage/index/dbox-single/sdbox-sync.c | 3 +- src/lib-storage/index/index-attachment.c | 158 +++++++++++++++++++- src/lib-storage/index/index-attachment.h | 12 + 8 files changed, 185 insertions(+), 167 deletions(-) diffs (truncated from 482 to 300 lines): diff -r 883ba4a47829 -r 0ed18f77ff08 src/lib-storage/index/dbox-common/dbox-attachment.c --- a/src/lib-storage/index/dbox-common/dbox-attachment.c Wed Nov 06 21:10:22 2013 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-attachment.c Wed Nov 06 23:00:58 2013 +0200 @@ -3,155 +3,32 @@ #include "lib.h" #include "istream.h" #include "str.h" -#include "fs-api.h" -#include "istream-fs-file.h" -#include "istream-attachment-connector.h" #include "dbox-file.h" #include "dbox-save.h" #include "dbox-attachment.h" -enum dbox_attachment_decode_option { - DBOX_ATTACHMENT_DECODE_OPTION_NONE = '-', - DBOX_ATTACHMENT_DECODE_OPTION_BASE64 = 'B', - DBOX_ATTACHMENT_DECODE_OPTION_CRLF = 'C' -}; - void dbox_attachment_save_write_metadata(struct mail_save_context *ctx, string_t *str) { const ARRAY_TYPE(mail_attachment_extref) *extrefs; - const struct mail_attachment_extref *extref; - bool add_space = FALSE; - unsigned int startpos; extrefs = index_attachment_save_get_extrefs(ctx); if (extrefs == NULL || array_count(extrefs) == 0) return; str_append_c(str, DBOX_METADATA_EXT_REF); - array_foreach(extrefs, extref) { - if (!add_space) - add_space = TRUE; - else - str_append_c(str, ' '); - str_printfa(str, "%"PRIuUOFF_T" %"PRIuUOFF_T" ", - extref->start_offset, extref->size); - - startpos = str_len(str); - if (extref->base64_have_crlf) - str_append_c(str, DBOX_ATTACHMENT_DECODE_OPTION_CRLF); - if (extref->base64_blocks_per_line > 0) { - str_printfa(str, "%c%u", - DBOX_ATTACHMENT_DECODE_OPTION_BASE64, - extref->base64_blocks_per_line * 4); - } - if (startpos == str_len(str)) { - /* make it clear there are no options */ - str_append_c(str, DBOX_ATTACHMENT_DECODE_OPTION_NONE); - } - str_append_c(str, ' '); - str_append(str, extref->path); - } + index_attachment_append_extrefs(str, extrefs); str_append_c(str, '\n'); } -static bool -parse_extref_decode_options(const char *str, - struct mail_attachment_extref *extref) -{ - unsigned int num; - - if (*str == DBOX_ATTACHMENT_DECODE_OPTION_NONE) - return str[1] == '\0'; - - while (*str != '\0') { - switch (*str) { - case DBOX_ATTACHMENT_DECODE_OPTION_BASE64: - str++; num = 0; - while (*str >= '0' && *str <= '9') { - num = num*10 + (*str-'0'); - str++; - } - if (num == 0 || num % 4 != 0) - return FALSE; - - extref->base64_blocks_per_line = num/4; - break; - case DBOX_ATTACHMENT_DECODE_OPTION_CRLF: - extref->base64_have_crlf = TRUE; - str++; - break; - default: - return FALSE; - } - } - return TRUE; -} - -static bool -dbox_attachment_parse_extref_real(const char *line, pool_t pool, - ARRAY_TYPE(mail_attachment_extref) *extrefs) -{ - struct mail_attachment_extref extref; - const char *const *args; - unsigned int i, len; - uoff_t last_voffset; - - args = t_strsplit(line, " "); - len = str_array_length(args); - if ((len % 4) != 0) - return FALSE; - - last_voffset = 0; - for (i = 0; args[i] != NULL; i += 4) { - const char *start_offset_str = args[i+0]; - const char *size_str = args[i+1]; - const char *decode_options = args[i+2]; - const char *path = args[i+3]; - - memset(&extref, 0, sizeof(extref)); - if (str_to_uoff(start_offset_str, &extref.start_offset) < 0 || - str_to_uoff(size_str, &extref.size) < 0 || - extref.start_offset < last_voffset || - !parse_extref_decode_options(decode_options, &extref)) - return FALSE; - - last_voffset += extref.size + - (extref.start_offset - last_voffset); - - extref.path = p_strdup(pool, path); - array_append(extrefs, &extref, 1); - } - return TRUE; -} - -bool dbox_attachment_parse_extref(const char *line, pool_t pool, - ARRAY_TYPE(mail_attachment_extref) *extrefs) -{ - bool ret; - - T_BEGIN { - ret = dbox_attachment_parse_extref_real(line, pool, extrefs); - } T_END; - return ret; -} - static int dbox_attachment_file_get_stream_from(struct dbox_file *file, const char *ext_refs, struct istream **stream, const char **error_r) { - ARRAY_TYPE(mail_attachment_extref) extrefs_arr; - const struct mail_attachment_extref *extref; - struct istream_attachment_connector *conn; - struct fs_file *fsfile; - struct istream *input; - const char *path, *path_suffix; + const char *path_suffix; uoff_t msg_size; - int ret; - - *error_r = NULL; if (*file->storage->attachment_dir == '\0') { mail_storage_set_critical(&file->storage->storage, @@ -160,37 +37,13 @@ return -1; } - t_array_init(&extrefs_arr, 16); - if (!dbox_attachment_parse_extref_real(ext_refs, pool_datastack_create(), - &extrefs_arr)) { - *error_r = "Broken ext-refs string"; + msg_size = dbox_file_get_plaintext_size(file); + path_suffix = file->storage->v.get_attachment_path_suffix(file); + if (index_attachment_stream_get(file->storage->attachment_fs, + file->storage->attachment_dir, + path_suffix, stream, msg_size, + ext_refs, error_r) < 0) return 0; - } - msg_size = dbox_file_get_plaintext_size(file); - conn = istream_attachment_connector_begin(*stream, msg_size); - - path_suffix = file->storage->v.get_attachment_path_suffix(file); - array_foreach(&extrefs_arr, extref) { - path = t_strdup_printf("%s/%s%s", file->storage->attachment_dir, - extref->path, path_suffix); - fsfile = fs_file_init(file->storage->attachment_fs, path, - FS_OPEN_MODE_READONLY); - input = i_stream_create_fs_file(&fsfile, IO_BLOCK_SIZE); - - ret = istream_attachment_connector_add(conn, input, - extref->start_offset, extref->size, - extref->base64_blocks_per_line, - extref->base64_have_crlf, error_r); - i_stream_unref(&input); - if (ret < 0) { - istream_attachment_connector_abort(&conn); - return 0; - } - } - - input = istream_attachment_connector_finish(&conn); - i_stream_unref(stream); - *stream = input; return 1; } diff -r 883ba4a47829 -r 0ed18f77ff08 src/lib-storage/index/dbox-common/dbox-attachment.h --- a/src/lib-storage/index/dbox-common/dbox-attachment.h Wed Nov 06 21:10:22 2013 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-attachment.h Wed Nov 06 23:00:58 2013 +0200 @@ -8,10 +8,6 @@ void dbox_attachment_save_write_metadata(struct mail_save_context *ctx, string_t *str); -/* Parse DBOX_METADATA_EXT_REF line to given array. Names are allocated - from the given pool. */ -bool dbox_attachment_parse_extref(const char *line, pool_t pool, - ARRAY_TYPE(mail_attachment_extref) *extrefs); /* Build a single message body stream out of the current message and all of its attachments. */ int dbox_attachment_file_get_stream(struct dbox_file *file, diff -r 883ba4a47829 -r 0ed18f77ff08 src/lib-storage/index/dbox-multi/mdbox-purge.c --- a/src/lib-storage/index/dbox-multi/mdbox-purge.c Wed Nov 06 21:10:22 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-purge.c Wed Nov 06 23:00:58 2013 +0200 @@ -147,13 +147,13 @@ /* end of metadata */ break; } - if (*line == DBOX_METADATA_EXT_REF) { - if (!dbox_attachment_parse_extref(line+1, ext_refs_pool, - extrefs)) { + if (*line == DBOX_METADATA_EXT_REF) T_BEGIN { + if (!index_attachment_parse_extrefs(line+1, ext_refs_pool, + extrefs)) { i_warning("%s: Ignoring corrupted extref: %s", file->cur_path, line); } - } + } T_END; } i_stream_set_max_buffer_size(file->input, buf_size); diff -r 883ba4a47829 -r 0ed18f77ff08 src/lib-storage/index/dbox-single/sdbox-copy.c --- a/src/lib-storage/index/dbox-single/sdbox-copy.c Wed Nov 06 21:10:22 2013 +0200 +++ b/src/lib-storage/index/dbox-single/sdbox-copy.c Wed Nov 06 23:00:58 2013 +0200 @@ -40,7 +40,7 @@ pool = pool_alloconly_create("sdbox attachments copy", 1024); p_array_init(&extrefs, pool, 16); - if (!dbox_attachment_parse_extref(extrefs_line, pool, &extrefs)) { + if (!index_attachment_parse_extrefs(extrefs_line, pool, &extrefs)) { mail_storage_set_critical(&dest_storage->storage, "Can't copy %s with corrupted extref metadata: %s", src_file->file.cur_path, extrefs_line); diff -r 883ba4a47829 -r 0ed18f77ff08 src/lib-storage/index/dbox-single/sdbox-file.c --- a/src/lib-storage/index/dbox-single/sdbox-file.c Wed Nov 06 21:10:22 2013 +0200 +++ b/src/lib-storage/index/dbox-single/sdbox-file.c Wed Nov 06 23:00:58 2013 +0200 @@ -436,7 +436,7 @@ pool = pool_alloconly_create("sdbox attachments unlink", 1024); p_array_init(&extrefs, pool, 16); - if (!dbox_attachment_parse_extref(extrefs_line, pool, &extrefs)) { + if (!index_attachment_parse_extrefs(extrefs_line, pool, &extrefs)) { i_warning("%s: Ignoring corrupted extref: %s", sfile->file.cur_path, extrefs_line); array_clear(&extrefs); diff -r 883ba4a47829 -r 0ed18f77ff08 src/lib-storage/index/dbox-single/sdbox-sync.c --- a/src/lib-storage/index/dbox-single/sdbox-sync.c Wed Nov 06 21:10:22 2013 +0200 +++ b/src/lib-storage/index/dbox-single/sdbox-sync.c Wed Nov 06 23:00:58 2013 +0200 @@ -160,8 +160,9 @@ /* NOTE: Index is no longer locked. Multiple processes may be unlinking the files at the same time. */ ctx->mbox->box.tmp_sync_view = ctx->sync_view; - array_foreach(&ctx->expunged_uids, uidp) + array_foreach(&ctx->expunged_uids, uidp) T_BEGIN { dbox_sync_file_expunge(ctx, *uidp); + } T_END; if (ctx->mbox->box.v.sync_notify != NULL) ctx->mbox->box.v.sync_notify(&ctx->mbox->box, 0, 0); ctx->mbox->box.tmp_sync_view = NULL; diff -r 883ba4a47829 -r 0ed18f77ff08 src/lib-storage/index/index-attachment.c --- a/src/lib-storage/index/index-attachment.c Wed Nov 06 21:10:22 2013 +0200 +++ b/src/lib-storage/index/index-attachment.c Wed Nov 06 23:00:58 2013 +0200 @@ -10,11 +10,20 @@ #include "str.h" #include "message-parser.h" #include "rfc822-parser.h" +#include "fs-api.h" +#include "istream-fs-file.h" +#include "istream-attachment-connector.h" #include "istream-attachment-extractor.h" #include "mail-user.h" #include "index-mail.h" #include "index-attachment.h" +enum mail_attachment_decode_option { + MAIL_ATTACHMENT_DECODE_OPTION_NONE = '-', + MAIL_ATTACHMENT_DECODE_OPTION_BASE64 = 'B', + MAIL_ATTACHMENT_DECODE_OPTION_CRLF = 'C' +}; + struct mail_save_attachment { From dovecot at dovecot.org Fri Nov 8 15:45:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 08 Nov 2013 15:45:08 +0200 Subject: dovecot-2.2: doveadm mailbox create: Added back -s parameter tha... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f1c2d889966c changeset: 16947:f1c2d889966c user: Timo Sirainen date: Fri Nov 08 14:44:57 2013 +0100 description: doveadm mailbox create: Added back -s parameter that was recently accidentally removed. diffstat: src/doveadm/doveadm-mail-mailbox.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (22 lines): diff -r 0ed18f77ff08 -r f1c2d889966c src/doveadm/doveadm-mail-mailbox.c --- a/src/doveadm/doveadm-mail-mailbox.c Wed Nov 06 23:00:58 2013 +0200 +++ b/src/doveadm/doveadm-mail-mailbox.c Fri Nov 08 14:44:57 2013 +0100 @@ -268,6 +268,9 @@ if (guid_128_from_string(optarg, ctx->update.mailbox_guid) < 0) doveadm_mail_help_name("mailbox create"); break; + case 's': + ctx->ctx.subscriptions = TRUE; + break; default: return FALSE; } @@ -282,7 +285,7 @@ ctx->ctx.ctx.v.init = cmd_mailbox_create_init; ctx->ctx.ctx.v.run = cmd_mailbox_create_run; ctx->ctx.ctx.v.parse_arg = cmd_mailbox_create_parse_arg; - ctx->ctx.ctx.getopt_args = "g:"; + ctx->ctx.ctx.getopt_args = "g:s"; p_array_init(&ctx->mailboxes, ctx->ctx.ctx.pool, 16); return &ctx->ctx.ctx; } From dovecot at dovecot.org Fri Nov 8 17:12:43 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 08 Nov 2013 17:12:43 +0200 Subject: dovecot-2.2: zlib: Give a name to zlib istreams Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b62eaae51894 changeset: 16948:b62eaae51894 user: Timo Sirainen date: Fri Nov 08 16:12:32 2013 +0100 description: zlib: Give a name to zlib istreams diffstat: src/plugins/zlib/zlib-plugin.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r f1c2d889966c -r b62eaae51894 src/plugins/zlib/zlib-plugin.c --- a/src/plugins/zlib/zlib-plugin.c Fri Nov 08 14:44:57 2013 +0100 +++ b/src/plugins/zlib/zlib-plugin.c Fri Nov 08 16:12:32 2013 +0100 @@ -93,6 +93,8 @@ input = i_stream_create_seekable_path(inputs, i_stream_get_max_buffer_size(inputs[0]), str_c(temp_prefix)); + i_stream_set_name(input, t_strdup_printf("zlib(%s)", + i_stream_get_name(inputs[0]))); i_stream_unref(&inputs[0]); cache->to = timeout_add(ZLIB_MAIL_CACHE_EXPIRE_MSECS, From dovecot at dovecot.org Fri Nov 8 17:40:19 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 08 Nov 2013 17:40:19 +0200 Subject: dovecot-2.2: lib-storage: Give name to attachments-connector ist... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/23b55e448784 changeset: 16949:23b55e448784 user: Timo Sirainen date: Fri Nov 08 16:39:52 2013 +0100 description: lib-storage: Give name to attachments-connector istream. diffstat: src/lib-storage/index/index-attachment.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r b62eaae51894 -r 23b55e448784 src/lib-storage/index/index-attachment.c --- a/src/lib-storage/index/index-attachment.c Fri Nov 08 16:12:32 2013 +0100 +++ b/src/lib-storage/index/index-attachment.c Fri Nov 08 16:39:52 2013 +0100 @@ -438,6 +438,8 @@ } input = istream_attachment_connector_finish(&conn); + i_stream_set_name(input, t_strdup_printf( + "attachments-connector(%s)", i_stream_get_name(*stream))); i_stream_unref(stream); *stream = input; return 0; From dovecot at dovecot.org Fri Nov 8 17:40:19 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 08 Nov 2013 17:40:19 +0200 Subject: dovecot-2.2: mdbox_deleted: Index was synced with wrong flags. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/02a746fd9830 changeset: 16950:02a746fd9830 user: Timo Sirainen date: Fri Nov 08 16:40:13 2013 +0100 description: mdbox_deleted: Index was synced with wrong flags. diffstat: src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (30 lines): diff -r 23b55e448784 -r 02a746fd9830 src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c --- a/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c Fri Nov 08 16:39:52 2013 +0100 +++ b/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c Fri Nov 08 16:40:13 2013 +0100 @@ -153,13 +153,14 @@ } static int mdbox_deleted_sync(struct mdbox_mailbox *mbox, - enum mdbox_sync_flags flags) + enum mdbox_sync_flags flags ATTR_UNUSED) { struct mail_index_sync_ctx *index_sync_ctx; struct mail_index_view *sync_view; struct mail_index_transaction *trans; struct mdbox_mail_index_record rec; struct mdbox_map_mail_index_record map_rec; + enum mail_index_sync_flags sync_flags; uint16_t refcount; uint32_t map_seq, map_count, seq, uid = 0; int ret = 0; @@ -185,8 +186,9 @@ memset(&rec, 0, sizeof(rec)); rec.save_date = ioloop_time; + sync_flags = index_storage_get_sync_flags(&mbox->box); if (mail_index_sync_begin(mbox->box.index, &index_sync_ctx, - &sync_view, &trans, flags) < 0) { + &sync_view, &trans, sync_flags) < 0) { mailbox_set_index_error(&mbox->box); return -1; } From dovecot at dovecot.org Fri Nov 8 17:44:31 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 08 Nov 2013 17:44:31 +0200 Subject: dovecot-2.2: lib-fs: Don't crash if istream-fs-file is closed wi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/37054c79e04b changeset: 16951:37054c79e04b user: Timo Sirainen date: Fri Nov 08 17:44:25 2013 +0200 description: lib-fs: Don't crash if istream-fs-file is closed without never being read. diffstat: src/lib-fs/istream-fs-file.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 02a746fd9830 -r 37054c79e04b src/lib-fs/istream-fs-file.c --- a/src/lib-fs/istream-fs-file.c Fri Nov 08 16:40:13 2013 +0100 +++ b/src/lib-fs/istream-fs-file.c Fri Nov 08 17:44:25 2013 +0200 @@ -15,7 +15,8 @@ { struct fs_file_istream *fstream = (struct fs_file_istream *)stream; - i_stream_destroy(&fstream->istream.parent); + if (fstream->istream.parent != NULL) + i_stream_destroy(&fstream->istream.parent); fs_file_deinit(&fstream->file); } From dovecot at dovecot.org Fri Nov 8 17:49:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 08 Nov 2013 17:49:08 +0200 Subject: dovecot-2.2: mail_attachment_*: Fixed "stream doesn't support se... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/87ff0023fe79 changeset: 16952:87ff0023fe79 user: Timo Sirainen date: Fri Nov 08 17:48:54 2013 +0200 description: mail_attachment_*: Fixed "stream doesn't support seeking backwards" panic Caused by recent changes. diffstat: src/lib-storage/index/index-attachment.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 37054c79e04b -r 87ff0023fe79 src/lib-storage/index/index-attachment.c --- a/src/lib-storage/index/index-attachment.c Fri Nov 08 17:44:25 2013 +0200 +++ b/src/lib-storage/index/index-attachment.c Fri Nov 08 17:48:54 2013 +0200 @@ -423,7 +423,8 @@ array_foreach(&extrefs_arr, extref) { path = t_strdup_printf("%s/%s%s", attachment_dir, extref->path, path_suffix); - file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY); + file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY | + FS_OPEN_FLAG_SEEKABLE); input = i_stream_create_fs_file(&file, IO_BLOCK_SIZE); ret = istream_attachment_connector_add(conn, input, From pigeonhole at rename-it.nl Sat Nov 9 12:42:14 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 09 Nov 2013 11:42:14 +0100 Subject: dovecot-2.1-pigeonhole: lib-sievestorage: Fixed active link veri... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/406f6cc2f1e1 changeset: 1703:406f6cc2f1e1 user: Stephan Bosch date: Sat Nov 09 11:42:07 2013 +0100 description: lib-sievestorage: Fixed active link verification to handle redundant path slashes correctly. diffstat: src/lib-sievestorage/sieve-storage-script.c | 51 +++++++++++++++++++++++++++- 1 files changed, 49 insertions(+), 2 deletions(-) diffs (68 lines): diff -r 7b09c4608d42 -r 406f6cc2f1e1 src/lib-sievestorage/sieve-storage-script.c --- a/src/lib-sievestorage/sieve-storage-script.c Thu Sep 26 21:44:37 2013 +0200 +++ b/src/lib-sievestorage/sieve-storage-script.c Sat Nov 09 11:42:07 2013 +0100 @@ -30,6 +30,53 @@ struct sieve_storage *storage; }; +static int _file_path_cmp(const char *path1, const char *path2) +{ + const char *p1, *p2; + int ret; + + p1 = path1; p2 = path2; + if (*p2 == '\0' && p1 != '\0') + return 1; + if (*p1 == '\0' && p2 != '\0') + return -1; + if (*p1 == '/' && *p2 != '/') + return 1; + if (*p2 == '/' && *p1 != '/') + return -1; + for (;;) { + const char *s1, *s2; + size_t size1, size2; + + /* skip repeated slashes */ + for (; *p1 == '/'; p1++); + for (; *p2 == '/'; p2++); + /* check for end of comparison */ + if (*p1 == '\0' || *p2 == '\0') + break; + /* mark start of path element */ + s1 = p1; + s2 = p2; + /* scan to end of path elements */ + for (; *p1 != '\0' && *p1 != '/'; p1++); + for (; *p2 != '\0' && *p2 != '/'; p2++); + /* compare sizes */ + size1 = p1 - s1; + size2 = p2 - s2; + if (size1 != size2) + return size1 - size2; + /* compare */ + if (size1 > 0 && (ret=memcmp(s1, s2, size1)) != 0) + return ret; + } + if (*p1 == '\0') { + if (*p2 == '\0') + return 0; + return -1; + } + return 1; +} + struct sieve_script *sieve_storage_script_init_from_path (struct sieve_storage *storage, const char *path, const char *scriptname) @@ -182,8 +229,8 @@ } /* Check whether the path is any good */ - if ( strcmp(scriptpath, storage->link_path) != 0 && - strcmp(scriptpath, storage->dir) != 0 ) { + if ( _file_path_cmp(scriptpath, storage->link_path) != 0 && + _file_path_cmp(scriptpath, storage->dir) != 0 ) { i_warning ("sieve-storage: Active sieve script symlink %s is broken: " "invalid/unknown path to storage (points to %s).", From pigeonhole at rename-it.nl Mon Nov 11 02:31:24 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 Nov 2013 01:31:24 +0100 Subject: dovecot-2.1-pigeonhole: lib-sieve: deprecated notify extension: ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/44f659981723 changeset: 1704:44f659981723 user: Stephan Bosch date: Mon Nov 11 01:31:08 2013 +0100 description: lib-sieve: deprecated notify extension: Fixed segfault problems in message string substitution. diffstat: src/lib-sieve/plugins/notify/ext-notify-common.c | 12 +++--- tests/deprecated/notify/basic.svtest | 48 ++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) diffs (95 lines): diff -r 406f6cc2f1e1 -r 44f659981723 src/lib-sieve/plugins/notify/ext-notify-common.c --- a/src/lib-sieve/plugins/notify/ext-notify-common.c Sat Nov 09 11:42:07 2013 +0100 +++ b/src/lib-sieve/plugins/notify/ext-notify-common.c Mon Nov 11 01:31:08 2013 +0100 @@ -243,14 +243,14 @@ /* Scan message for substitutions */ p = msg_format; while ( *p != '\0' ) { - const char *const *header; + const char *header; if ( strncasecmp(p, "$from$", 6) == 0 ) { p += 6; - /* Fetch sender from oriinal message */ - if ( mail_get_headers_utf8(msgdata->mail, "from", &header) >= 0 ) - str_append(out_msg, header[0]); + /* Fetch sender from original message */ + if ( mail_get_first_header_utf8(msgdata->mail, "from", &header) > 0 ) + str_append(out_msg, header); } else if ( strncasecmp(p, "$env-from$", 10) == 0 ) { p += 10; @@ -262,8 +262,8 @@ p += 9; /* Fetch sender from oriinal message */ - if ( mail_get_headers_utf8(msgdata->mail, "subject", &header) >= 0 ) - str_append(out_msg, header[0]); + if ( mail_get_first_header_utf8(msgdata->mail, "subject", &header) > 0 ) + str_append(out_msg, header); } else if ( strncasecmp(p, "$text", 5) == 0 && (p[5] == '[' || p[5] == '$') ) { diff -r 406f6cc2f1e1 -r 44f659981723 tests/deprecated/notify/basic.svtest --- a/tests/deprecated/notify/basic.svtest Sat Nov 09 11:42:07 2013 +0100 +++ b/tests/deprecated/notify/basic.svtest Mon Nov 11 01:31:08 2013 +0100 @@ -1,5 +1,6 @@ require "vnd.dovecot.testsuite"; require "notify"; +require "body"; test "Execute" { /* Test to catch runtime segfaults */ @@ -8,4 +9,51 @@ :low :method "mailto" :options ["stephan at example.com", "stephan at example.org"]; + + if not test_result_execute { + test_fail "Execute failed"; + } } + +test_result_reset; + +test_set "message" text: +To: user at example.com +From: stephan at example.org +Subject: Mail + +Test! +. +; + +test "Substitutions" { + notify + :message "$from$: $subject$" + :options "stephan at example.com"; + if not test_result_execute { + test_fail "Execute failed"; + } + test_message :smtp 0; + if not body :contains "stephan at example.org: Mail" { + test_fail "Substitution failed"; + } +} + +test_result_reset; + +test_set "message" text: +To: user at example.com + +Test! +. +; + +test "Empty substitutions" { + notify + :message "$from$: $subject$" + :options "stephan at example.com"; + if not test_result_execute { + test_fail "Execute failed"; + } +} + From pigeonhole at rename-it.nl Mon Nov 11 02:44:28 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 Nov 2013 01:44:28 +0100 Subject: dovecot-2.2-pigeonhole: Released v0.3.6 for Dovecot v2.1.17. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/87d8f0aae957 changeset: 1814:87d8f0aae957 user: Stephan Bosch date: Thu Sep 26 21:41:24 2013 +0200 description: Released v0.3.6 for Dovecot v2.1.17. diffstat: NEWS | 12 ++++++++++++ configure.in | 2 +- 2 files changed, 13 insertions(+), 1 deletions(-) diffs (28 lines): diff -r aadc77880254 -r 87d8f0aae957 NEWS --- a/NEWS Mon Sep 16 00:55:29 2013 +0200 +++ b/NEWS Thu Sep 26 21:41:24 2013 +0200 @@ -1,3 +1,15 @@ +v0.3.6 26-09-2013 Stephan Bosch + + - Fixed a binary code read problem in the `set' command of the Sieve variables + extension. Using the set command with a modifier and an empty string value + would cause code corruption problems while running the script. + - Various fixes for compiler and static analyzer warnings, as reported + by CLang. + - ManageSieve: Fixed '[' ']' stupidity for response codes (only happened + before login). + - Fixed setting name in example-config/conf.d/20-managesieve.conf. + - Fixed messed up hex output for sieve-dump tool. + v0.3.5 09-05-2013 Stephan Bosch - Sieve editheader extension: fixed interaction with the Sieve body extension. diff -r aadc77880254 -r 87d8f0aae957 configure.in --- a/configure.in Mon Sep 16 00:55:29 2013 +0200 +++ b/configure.in Thu Sep 26 21:41:24 2013 +0200 @@ -1,4 +1,4 @@ -AC_INIT([Pigeonhole], [0.3.5], [dovecot at dovecot.org], [dovecot-2.1-pigeonhole]) +AC_INIT([Pigeonhole], [0.3.6], [dovecot at dovecot.org], [dovecot-2.1-pigeonhole]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_SRCDIR([src]) AC_CONFIG_MACRO_DIR([m4]) From pigeonhole at rename-it.nl Mon Nov 11 02:44:29 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 Nov 2013 01:44:29 +0100 Subject: dovecot-2.2-pigeonhole: Added signature for changeset 87d8f0aae957 Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/7b09c4608d42 changeset: 1816:7b09c4608d42 user: Stephan Bosch date: Thu Sep 26 21:44:37 2013 +0200 description: Added signature for changeset 87d8f0aae957 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r b9f545cd2ec8 -r 7b09c4608d42 .hgsigs --- a/.hgsigs Thu Sep 26 21:41:35 2013 +0200 +++ b/.hgsigs Thu Sep 26 21:44:37 2013 +0200 @@ -9,3 +9,4 @@ 64474c35967852bc452f71bc099ec3f8ded0369a 0 iQEcBAABAgAGBQJQWMvhAAoJEATWKx49+7T0NrYH/2PQuuFqzlku+NG8Iw0UN2yeDEML+2n1xG31ud7m3sNWw8lX+03gEd+LU8+LygHJJ0IAde/jBYRBbC8zj9UXDl3v5FIRwhcvGnllBCMMH7motfg+aLrCR/xs+0jV/AqpRin1VILHYFaB9UFP5PUgvJJiCUniQWoe+r41gra1hRA7OK3923YOOi9t4zJxoat7e0OMhc0IcdB7n3iQmyicbb8izKw/UvR2tR3T7fVcEl6u1LlbGaojtJA03V1L+a8QkmltiurD9VNmiHz++bGGJlA7LSmVYBq7BeC1lDnXUGO9ryZgln6aXRwUS0VaTI51F9gSMw+0UDJCwA5yBKqYyR8= 4932026768454443d87d2e6445552b331589dbb1 0 iQEcBAABAgAGBQJRYIePAAoJEATWKx49+7T0p2oH/ROhjt/m+wZmT9+2NxEDwnaOoQ8m9TxkZiZ50mmi/k4L7OQe/xffxM2T3NTTgRkaLCK2+MEz0pSLJXL+n/AjTBtiynwSxYY+W0wtYKBIs0tcQHaSafN2u5LjtQZ2RHg+Fi1szhJQu/jy31w12KGTqdyVw05JuEPgyfM7fKpqKh8CQZJucEyn9Vf8boGPQMFJ7N6o4wpOeW8RuVcBAEToHMpkDI1OQmB22cEQJfZrdOMPaucUOG6Abfw0FDwwV4bHa1nmxiPZXF8DpW31SVDoZgyvi08TSWFHS9t8Pij+XyWIdXhbCzqdpkvLGEsvMJjhiro9agf2tSj9jP+D4xiFRe8= 374ec93999588b1acc554f81f61c93bf9fad7037 0 iQEcBAABAgAGBQJRi6LgAAoJEATWKx49+7T0OPgH+gPOOYqdODGONhZTwxjKmvbVNzb428t/fS/pi2bSIeOXVzLone/P7NYstdM67eQsfxY4Kn8qN/eFkjdkozbc+3xx0VFvptFTd9NXGM57O84umgrvKXQDQM333TVMxMG5u6TPPkCmFcWmggcc1IdQxZ66dpgfETdh+IgJXdLQg4oc0IqHtqx19dVyYXRbtgY+GfA9ovFqOkzcWoXgSHRMmUf2gCGWPc3jSZPVHd3ObRvrwn3NcWULK4WCt5QPc6fU53eDrEvkSu6C5U0IV5G3jakrUK0PYtQ3jvxXGm8J3XQ+ea79Bp+mG/21Woz30Ey6AlFbuAHobGesGafjPLDLSNs= +87d8f0aae957ad290ea2e10033ce9ef70b291bb4 0 iQEcBAABAgAGBQJSRI6hAAoJEATWKx49+7T0j/YH/1ugfus0Bj/9MhG788eaaYXuF1UjusxvckT5ufkruQLkY0xt5aeDVPOCwEEhgjR5mr4yHYYQUg5xl6XvgErPymmx2SvTzjDvATSNK+smVU375Ho57RGNV9NVO1sKKXN5fqjeURfpu+iMNbaEpP/j9oQ+nukL1fFh3NLEsVXBCJTmJPqvV14jIpEFAwgUXSpUDVaTN8iAo/NuhmpcjnxAUR01tkJEVHH0hxExDYwXjGb76wUzgG9cLLqyFTn5qB8E2b0qaqaLY1h/K0exOJ01yWOYMiefm30k7viwQCEM+ZyyZ7ff355Gqcy/gJJSD7+jdTY7tEHbtBZcoE/zI9CYuqw= From pigeonhole at rename-it.nl Mon Nov 11 02:44:29 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 Nov 2013 01:44:29 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: deprecated notify extension: ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/44f659981723 changeset: 1818:44f659981723 user: Stephan Bosch date: Mon Nov 11 01:31:08 2013 +0100 description: lib-sieve: deprecated notify extension: Fixed segfault problems in message string substitution. diffstat: src/lib-sieve/plugins/notify/ext-notify-common.c | 12 +++--- tests/deprecated/notify/basic.svtest | 48 ++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) diffs (95 lines): diff -r 406f6cc2f1e1 -r 44f659981723 src/lib-sieve/plugins/notify/ext-notify-common.c --- a/src/lib-sieve/plugins/notify/ext-notify-common.c Sat Nov 09 11:42:07 2013 +0100 +++ b/src/lib-sieve/plugins/notify/ext-notify-common.c Mon Nov 11 01:31:08 2013 +0100 @@ -243,14 +243,14 @@ /* Scan message for substitutions */ p = msg_format; while ( *p != '\0' ) { - const char *const *header; + const char *header; if ( strncasecmp(p, "$from$", 6) == 0 ) { p += 6; - /* Fetch sender from oriinal message */ - if ( mail_get_headers_utf8(msgdata->mail, "from", &header) >= 0 ) - str_append(out_msg, header[0]); + /* Fetch sender from original message */ + if ( mail_get_first_header_utf8(msgdata->mail, "from", &header) > 0 ) + str_append(out_msg, header); } else if ( strncasecmp(p, "$env-from$", 10) == 0 ) { p += 10; @@ -262,8 +262,8 @@ p += 9; /* Fetch sender from oriinal message */ - if ( mail_get_headers_utf8(msgdata->mail, "subject", &header) >= 0 ) - str_append(out_msg, header[0]); + if ( mail_get_first_header_utf8(msgdata->mail, "subject", &header) > 0 ) + str_append(out_msg, header); } else if ( strncasecmp(p, "$text", 5) == 0 && (p[5] == '[' || p[5] == '$') ) { diff -r 406f6cc2f1e1 -r 44f659981723 tests/deprecated/notify/basic.svtest --- a/tests/deprecated/notify/basic.svtest Sat Nov 09 11:42:07 2013 +0100 +++ b/tests/deprecated/notify/basic.svtest Mon Nov 11 01:31:08 2013 +0100 @@ -1,5 +1,6 @@ require "vnd.dovecot.testsuite"; require "notify"; +require "body"; test "Execute" { /* Test to catch runtime segfaults */ @@ -8,4 +9,51 @@ :low :method "mailto" :options ["stephan at example.com", "stephan at example.org"]; + + if not test_result_execute { + test_fail "Execute failed"; + } } + +test_result_reset; + +test_set "message" text: +To: user at example.com +From: stephan at example.org +Subject: Mail + +Test! +. +; + +test "Substitutions" { + notify + :message "$from$: $subject$" + :options "stephan at example.com"; + if not test_result_execute { + test_fail "Execute failed"; + } + test_message :smtp 0; + if not body :contains "stephan at example.org: Mail" { + test_fail "Substitution failed"; + } +} + +test_result_reset; + +test_set "message" text: +To: user at example.com + +Test! +. +; + +test "Empty substitutions" { + notify + :message "$from$: $subject$" + :options "stephan at example.com"; + if not test_result_execute { + test_fail "Execute failed"; + } +} + From pigeonhole at rename-it.nl Mon Nov 11 02:44:29 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 Nov 2013 01:44:29 +0100 Subject: dovecot-2.2-pigeonhole: Merged changes from Pigeonhole v0.3 tree. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/16671ce66f75 changeset: 1819:16671ce66f75 user: Stephan Bosch date: Mon Nov 11 01:44:19 2013 +0100 description: Merged changes from Pigeonhole v0.3 tree. diffstat: .hgsigs | 1 + .hgtags | 1 + NEWS | 12 +++++ src/lib-sieve/plugins/notify/ext-notify-common.c | 12 ++-- src/lib-sievestorage/sieve-storage-script.c | 51 +++++++++++++++++++++++- tests/deprecated/notify/basic.svtest | 48 ++++++++++++++++++++++ 6 files changed, 117 insertions(+), 8 deletions(-) diffs (207 lines): diff -r 58e6a73e5bc6 -r 16671ce66f75 .hgsigs --- a/.hgsigs Sun Nov 03 02:02:56 2013 +0100 +++ b/.hgsigs Mon Nov 11 01:44:19 2013 +0100 @@ -9,6 +9,7 @@ 64474c35967852bc452f71bc099ec3f8ded0369a 0 iQEcBAABAgAGBQJQWMvhAAoJEATWKx49+7T0NrYH/2PQuuFqzlku+NG8Iw0UN2yeDEML+2n1xG31ud7m3sNWw8lX+03gEd+LU8+LygHJJ0IAde/jBYRBbC8zj9UXDl3v5FIRwhcvGnllBCMMH7motfg+aLrCR/xs+0jV/AqpRin1VILHYFaB9UFP5PUgvJJiCUniQWoe+r41gra1hRA7OK3923YOOi9t4zJxoat7e0OMhc0IcdB7n3iQmyicbb8izKw/UvR2tR3T7fVcEl6u1LlbGaojtJA03V1L+a8QkmltiurD9VNmiHz++bGGJlA7LSmVYBq7BeC1lDnXUGO9ryZgln6aXRwUS0VaTI51F9gSMw+0UDJCwA5yBKqYyR8= 4932026768454443d87d2e6445552b331589dbb1 0 iQEcBAABAgAGBQJRYIePAAoJEATWKx49+7T0p2oH/ROhjt/m+wZmT9+2NxEDwnaOoQ8m9TxkZiZ50mmi/k4L7OQe/xffxM2T3NTTgRkaLCK2+MEz0pSLJXL+n/AjTBtiynwSxYY+W0wtYKBIs0tcQHaSafN2u5LjtQZ2RHg+Fi1szhJQu/jy31w12KGTqdyVw05JuEPgyfM7fKpqKh8CQZJucEyn9Vf8boGPQMFJ7N6o4wpOeW8RuVcBAEToHMpkDI1OQmB22cEQJfZrdOMPaucUOG6Abfw0FDwwV4bHa1nmxiPZXF8DpW31SVDoZgyvi08TSWFHS9t8Pij+XyWIdXhbCzqdpkvLGEsvMJjhiro9agf2tSj9jP+D4xiFRe8= 374ec93999588b1acc554f81f61c93bf9fad7037 0 iQEcBAABAgAGBQJRi6LgAAoJEATWKx49+7T0OPgH+gPOOYqdODGONhZTwxjKmvbVNzb428t/fS/pi2bSIeOXVzLone/P7NYstdM67eQsfxY4Kn8qN/eFkjdkozbc+3xx0VFvptFTd9NXGM57O84umgrvKXQDQM333TVMxMG5u6TPPkCmFcWmggcc1IdQxZ66dpgfETdh+IgJXdLQg4oc0IqHtqx19dVyYXRbtgY+GfA9ovFqOkzcWoXgSHRMmUf2gCGWPc3jSZPVHd3ObRvrwn3NcWULK4WCt5QPc6fU53eDrEvkSu6C5U0IV5G3jakrUK0PYtQ3jvxXGm8J3XQ+ea79Bp+mG/21Woz30Ey6AlFbuAHobGesGafjPLDLSNs= +87d8f0aae957ad290ea2e10033ce9ef70b291bb4 0 iQEcBAABAgAGBQJSRI6hAAoJEATWKx49+7T0j/YH/1ugfus0Bj/9MhG788eaaYXuF1UjusxvckT5ufkruQLkY0xt5aeDVPOCwEEhgjR5mr4yHYYQUg5xl6XvgErPymmx2SvTzjDvATSNK+smVU375Ho57RGNV9NVO1sKKXN5fqjeURfpu+iMNbaEpP/j9oQ+nukL1fFh3NLEsVXBCJTmJPqvV14jIpEFAwgUXSpUDVaTN8iAo/NuhmpcjnxAUR01tkJEVHH0hxExDYwXjGb76wUzgG9cLLqyFTn5qB8E2b0qaqaLY1h/K0exOJ01yWOYMiefm30k7viwQCEM+ZyyZ7ff355Gqcy/gJJSD7+jdTY7tEHbtBZcoE/zI9CYuqw= 3a8dc1250e9b850044acbcd9d63d63597e67c7cb 0 iQEcBAABAgAGBQJRi6eKAAoJEATWKx49+7T0+3sIAKj87rGC9kakJPL4RyFDc762HueCez8mUTRhe62K9+2LzVJL+2inGPY33PL5F7xZd0H/pxIfNfk/WGA78q6BkOaJSlGtz+wb5H/iZzSc+6GRfzPavlXlENgOKtnmzf3AoEKc0iNnieB5oYJTkJ2GtbPv2GD4u/Y6w7eTISfh/HlJeGvybR3L4XT6scNCr90/iQWjoZMNZSbYvKInSmrN9SGgtg0exqFfdkFmMahhSO/PGBq4hi5ZyFKqsnYEZpkLOduKW86nsn1GZiPD5/bEUD7lxJeeoRk3aobZoMQUlz4VRma6ag0MPRLpmzqUQFhpvDzpdg3dZSw/8NUQBa78w1M= f1535e2255cd84849d8b4fad6d82bdd6e01f810f 0 iQEcBAABAgAGBQJR1IRdAAoJEATWKx49+7T0uKMIANauBvbSO72bhrg8EI3Cvr2qy5nZrgqdgtZI0ZtIHbRBa9aBmxbX5lGqjWNLZaXwLc9dk7aeDoZds73kjiCYB+ac3UaXBaRscKNArkigTKFE0L2UZ5X5T/76iJ6diPFs8Eo/4ajYERDa3UQgUqHJ/8Fe2884bFDXMRmjbtVu2ceCw8Dlkee4kiVfjzuPRWud2jjtXcCoJXkFIQSbGcKsjlhd2yAGToJ9iw06tPNAFjupht9/wozyaZiZqJj7AnNvflHGMO1FDhOQJYxjz9JYA61h9F1vytGwfm1n3BVANpWSBJtkkuM/Tjp9GxeDB2jNrUbMikicSUceNtRLg9i0b+A= 2176d400eca4fa5d89597793a09c0b194eaed05f 0 iQEcBAABAgAGBQJSRI5JAAoJEATWKx49+7T0kGcIAKNHvRh/4idzHekHk5L81e0nu1IZRLyXg6YLkviF52ZYsB4WSbsiVhVYGoVS0XdSCvouQFq//In5S8nkUaSc1r4/2VK/vEchisYxwnF0YyLz5o7CaixLBCARLGGbjZTCyHvqOCHj6sRTJtc7GP1cdFXzYo7yIGg+W1hZn7yBB32p6stE+9SurxinK312LGFvBRrJ9t7y9gIeP1aV9yd0AhTorAP0XsOMflNQnnxsNXR/8LhwsqpOt9TR8hTVdGZHi2ZqjqK37/XAGYAJpBvpQRFq2UPWj6fEGNRLsh9TzGodQh/RoESvTeoHpEVy5o2yHUcnY71bYfMpb2IJH7NHKAA= diff -r 58e6a73e5bc6 -r 16671ce66f75 .hgtags --- a/.hgtags Sun Nov 03 02:02:56 2013 +0100 +++ b/.hgtags Mon Nov 11 01:44:19 2013 +0100 @@ -15,6 +15,7 @@ 64474c35967852bc452f71bc099ec3f8ded0369a 0.3.3 4932026768454443d87d2e6445552b331589dbb1 0.3.4 374ec93999588b1acc554f81f61c93bf9fad7037 0.3.5 +87d8f0aae957ad290ea2e10033ce9ef70b291bb4 0.3.6 3a8dc1250e9b850044acbcd9d63d63597e67c7cb 0.4.0 f1535e2255cd84849d8b4fad6d82bdd6e01f810f 0.4.1 2176d400eca4fa5d89597793a09c0b194eaed05f 0.4.2 diff -r 58e6a73e5bc6 -r 16671ce66f75 NEWS --- a/NEWS Sun Nov 03 02:02:56 2013 +0100 +++ b/NEWS Mon Nov 11 01:44:19 2013 +0100 @@ -56,6 +56,18 @@ possible. Environment items "location", "phase" and "domain" now also return a usable value. +v0.3.6 26-09-2013 Stephan Bosch + + - Fixed a binary code read problem in the `set' command of the Sieve variables + extension. Using the set command with a modifier and an empty string value + would cause code corruption problems while running the script. + - Various fixes for compiler and static analyzer warnings, as reported + by CLang. + - ManageSieve: Fixed '[' ']' stupidity for response codes (only happened + before login). + - Fixed setting name in example-config/conf.d/20-managesieve.conf. + - Fixed messed up hex output for sieve-dump tool. + v0.3.5 09-05-2013 Stephan Bosch - Sieve editheader extension: fixed interaction with the Sieve body extension. diff -r 58e6a73e5bc6 -r 16671ce66f75 src/lib-sieve/plugins/notify/ext-notify-common.c --- a/src/lib-sieve/plugins/notify/ext-notify-common.c Sun Nov 03 02:02:56 2013 +0100 +++ b/src/lib-sieve/plugins/notify/ext-notify-common.c Mon Nov 11 01:44:19 2013 +0100 @@ -243,14 +243,14 @@ /* Scan message for substitutions */ p = msg_format; while ( *p != '\0' ) { - const char *const *header; + const char *header; if ( strncasecmp(p, "$from$", 6) == 0 ) { p += 6; - /* Fetch sender from oriinal message */ - if ( mail_get_headers_utf8(msgdata->mail, "from", &header) >= 0 ) - str_append(out_msg, header[0]); + /* Fetch sender from original message */ + if ( mail_get_first_header_utf8(msgdata->mail, "from", &header) > 0 ) + str_append(out_msg, header); } else if ( strncasecmp(p, "$env-from$", 10) == 0 ) { p += 10; @@ -262,8 +262,8 @@ p += 9; /* Fetch sender from oriinal message */ - if ( mail_get_headers_utf8(msgdata->mail, "subject", &header) >= 0 ) - str_append(out_msg, header[0]); + if ( mail_get_first_header_utf8(msgdata->mail, "subject", &header) > 0 ) + str_append(out_msg, header); } else if ( strncasecmp(p, "$text", 5) == 0 && (p[5] == '[' || p[5] == '$') ) { diff -r 58e6a73e5bc6 -r 16671ce66f75 src/lib-sievestorage/sieve-storage-script.c --- a/src/lib-sievestorage/sieve-storage-script.c Sun Nov 03 02:02:56 2013 +0100 +++ b/src/lib-sievestorage/sieve-storage-script.c Mon Nov 11 01:44:19 2013 +0100 @@ -31,6 +31,53 @@ struct sieve_storage *storage; }; +static int _file_path_cmp(const char *path1, const char *path2) +{ + const char *p1, *p2; + int ret; + + p1 = path1; p2 = path2; + if (*p2 == '\0' && p1 != '\0') + return 1; + if (*p1 == '\0' && p2 != '\0') + return -1; + if (*p1 == '/' && *p2 != '/') + return 1; + if (*p2 == '/' && *p1 != '/') + return -1; + for (;;) { + const char *s1, *s2; + size_t size1, size2; + + /* skip repeated slashes */ + for (; *p1 == '/'; p1++); + for (; *p2 == '/'; p2++); + /* check for end of comparison */ + if (*p1 == '\0' || *p2 == '\0') + break; + /* mark start of path element */ + s1 = p1; + s2 = p2; + /* scan to end of path elements */ + for (; *p1 != '\0' && *p1 != '/'; p1++); + for (; *p2 != '\0' && *p2 != '/'; p2++); + /* compare sizes */ + size1 = p1 - s1; + size2 = p2 - s2; + if (size1 != size2) + return size1 - size2; + /* compare */ + if (size1 > 0 && (ret=memcmp(s1, s2, size1)) != 0) + return ret; + } + if (*p1 == '\0') { + if (*p2 == '\0') + return 0; + return -1; + } + return 1; +} + struct sieve_script *sieve_storage_script_init_from_path (struct sieve_storage *storage, const char *path, const char *scriptname) @@ -183,8 +230,8 @@ } /* Check whether the path is any good */ - if ( strcmp(scriptpath, storage->link_path) != 0 && - strcmp(scriptpath, storage->dir) != 0 ) { + if ( _file_path_cmp(scriptpath, storage->link_path) != 0 && + _file_path_cmp(scriptpath, storage->dir) != 0 ) { i_warning ("sieve-storage: Active sieve script symlink %s is broken: " "invalid/unknown path to storage (points to %s).", diff -r 58e6a73e5bc6 -r 16671ce66f75 tests/deprecated/notify/basic.svtest --- a/tests/deprecated/notify/basic.svtest Sun Nov 03 02:02:56 2013 +0100 +++ b/tests/deprecated/notify/basic.svtest Mon Nov 11 01:44:19 2013 +0100 @@ -1,5 +1,6 @@ require "vnd.dovecot.testsuite"; require "notify"; +require "body"; test "Execute" { /* Test to catch runtime segfaults */ @@ -8,4 +9,51 @@ :low :method "mailto" :options ["stephan at example.com", "stephan at example.org"]; + + if not test_result_execute { + test_fail "Execute failed"; + } } + +test_result_reset; + +test_set "message" text: +To: user at example.com +From: stephan at example.org +Subject: Mail + +Test! +. +; + +test "Substitutions" { + notify + :message "$from$: $subject$" + :options "stephan at example.com"; + if not test_result_execute { + test_fail "Execute failed"; + } + test_message :smtp 0; + if not body :contains "stephan at example.org: Mail" { + test_fail "Substitution failed"; + } +} + +test_result_reset; + +test_set "message" text: +To: user at example.com + +Test! +. +; + +test "Empty substitutions" { + notify + :message "$from$: $subject$" + :options "stephan at example.com"; + if not test_result_execute { + test_fail "Execute failed"; + } +} + From pigeonhole at rename-it.nl Mon Nov 11 02:44:29 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 Nov 2013 01:44:29 +0100 Subject: dovecot-2.2-pigeonhole: lib-sievestorage: Fixed active link veri... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/406f6cc2f1e1 changeset: 1817:406f6cc2f1e1 user: Stephan Bosch date: Sat Nov 09 11:42:07 2013 +0100 description: lib-sievestorage: Fixed active link verification to handle redundant path slashes correctly. diffstat: src/lib-sievestorage/sieve-storage-script.c | 51 +++++++++++++++++++++++++++- 1 files changed, 49 insertions(+), 2 deletions(-) diffs (68 lines): diff -r 7b09c4608d42 -r 406f6cc2f1e1 src/lib-sievestorage/sieve-storage-script.c --- a/src/lib-sievestorage/sieve-storage-script.c Thu Sep 26 21:44:37 2013 +0200 +++ b/src/lib-sievestorage/sieve-storage-script.c Sat Nov 09 11:42:07 2013 +0100 @@ -30,6 +30,53 @@ struct sieve_storage *storage; }; +static int _file_path_cmp(const char *path1, const char *path2) +{ + const char *p1, *p2; + int ret; + + p1 = path1; p2 = path2; + if (*p2 == '\0' && p1 != '\0') + return 1; + if (*p1 == '\0' && p2 != '\0') + return -1; + if (*p1 == '/' && *p2 != '/') + return 1; + if (*p2 == '/' && *p1 != '/') + return -1; + for (;;) { + const char *s1, *s2; + size_t size1, size2; + + /* skip repeated slashes */ + for (; *p1 == '/'; p1++); + for (; *p2 == '/'; p2++); + /* check for end of comparison */ + if (*p1 == '\0' || *p2 == '\0') + break; + /* mark start of path element */ + s1 = p1; + s2 = p2; + /* scan to end of path elements */ + for (; *p1 != '\0' && *p1 != '/'; p1++); + for (; *p2 != '\0' && *p2 != '/'; p2++); + /* compare sizes */ + size1 = p1 - s1; + size2 = p2 - s2; + if (size1 != size2) + return size1 - size2; + /* compare */ + if (size1 > 0 && (ret=memcmp(s1, s2, size1)) != 0) + return ret; + } + if (*p1 == '\0') { + if (*p2 == '\0') + return 0; + return -1; + } + return 1; +} + struct sieve_script *sieve_storage_script_init_from_path (struct sieve_storage *storage, const char *path, const char *scriptname) @@ -182,8 +229,8 @@ } /* Check whether the path is any good */ - if ( strcmp(scriptpath, storage->link_path) != 0 && - strcmp(scriptpath, storage->dir) != 0 ) { + if ( _file_path_cmp(scriptpath, storage->link_path) != 0 && + _file_path_cmp(scriptpath, storage->dir) != 0 ) { i_warning ("sieve-storage: Active sieve script symlink %s is broken: " "invalid/unknown path to storage (points to %s).", From pigeonhole at rename-it.nl Mon Nov 11 02:44:29 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 11 Nov 2013 01:44:29 +0100 Subject: dovecot-2.2-pigeonhole: Added tag 0.3.6 for changeset 87d8f0aae957 Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/b9f545cd2ec8 changeset: 1815:b9f545cd2ec8 user: Stephan Bosch date: Thu Sep 26 21:41:35 2013 +0200 description: Added tag 0.3.6 for changeset 87d8f0aae957 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 87d8f0aae957 -r b9f545cd2ec8 .hgtags --- a/.hgtags Thu Sep 26 21:41:24 2013 +0200 +++ b/.hgtags Thu Sep 26 21:41:35 2013 +0200 @@ -15,3 +15,4 @@ 64474c35967852bc452f71bc099ec3f8ded0369a 0.3.3 4932026768454443d87d2e6445552b331589dbb1 0.3.4 374ec93999588b1acc554f81f61c93bf9fad7037 0.3.5 +87d8f0aae957ad290ea2e10033ce9ef70b291bb4 0.3.6 From pigeonhole at rename-it.nl Sat Nov 16 00:13:38 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 15 Nov 2013 23:13:38 +0100 Subject: dovecot-2.2-pigeonhole: Sieve extprograms plugin: Improved handi... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/b7434a02fc51 changeset: 1820:b7434a02fc51 user: Stephan Bosch date: Fri Nov 15 23:13:32 2013 +0100 description: Sieve extprograms plugin: Improved handing of extension configuration. diffstat: src/plugins/sieve-extprograms/sieve-extprograms-common.c | 18 ++++++++------- 1 files changed, 10 insertions(+), 8 deletions(-) diffs (50 lines): diff -r 16671ce66f75 -r b7434a02fc51 src/plugins/sieve-extprograms/sieve-extprograms-common.c --- a/src/plugins/sieve-extprograms/sieve-extprograms-common.c Mon Nov 11 01:44:19 2013 +0100 +++ b/src/plugins/sieve-extprograms/sieve-extprograms-common.c Fri Nov 15 23:13:32 2013 +0100 @@ -62,8 +62,7 @@ struct sieve_extprograms_config *ext_config; const char *extname = sieve_extension_name(ext); const char *bin_dir, *socket_dir; - sieve_number_t execute_timeout = - SIEVE_EXTPROGRAMS_DEFAULT_EXEC_TIMEOUT_SECS; + sieve_number_t execute_timeout; extname = strrchr(extname, '.'); i_assert(extname != NULL); @@ -75,6 +74,8 @@ (svinst, t_strdup_printf("sieve_%s_socket_dir", extname)); ext_config = i_new(struct sieve_extprograms_config, 1); + ext_config->execute_timeout = + SIEVE_EXTPROGRAMS_DEFAULT_EXEC_TIMEOUT_SECS; if ( bin_dir == NULL && socket_dir == NULL ) { if ( svinst->debug ) { @@ -86,12 +87,12 @@ } else { ext_config->bin_dir = i_strdup(bin_dir); ext_config->socket_dir = i_strdup(socket_dir); - } - if (sieve_setting_get_duration_value - (svinst, t_strdup_printf("sieve_%s_exec_timeout", extname), - &execute_timeout)) { - ext_config->execute_timeout = execute_timeout; + if (sieve_setting_get_duration_value + (svinst, t_strdup_printf("sieve_%s_exec_timeout", extname), + &execute_timeout)) { + ext_config->execute_timeout = execute_timeout; + } } if ( sieve_extension_is(ext, pipe_extension) ) @@ -415,7 +416,8 @@ "running program: %s", action, program_name); } - if ( ext_config == NULL ) { + if ( ext_config == NULL || + (ext_config->bin_dir == NULL && ext_config->socket_dir == NULL) ) { sieve_sys_error(svinst, "action %s: " "failed to execute program `%s': " "vnd.dovecot.%s extension is unconfigured", action, program_name, action); From dovecot at dovecot.org Sat Nov 16 16:12:01 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 Nov 2013 16:12:01 +0200 Subject: dovecot-2.2: man: Added description for command `doveadm copy`. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a913f29068a7 changeset: 16953:a913f29068a7 user: Pascal Volk date: Sat Nov 16 14:06:32 2013 +0000 description: man: Added description for command `doveadm copy`. diffstat: doc/man/Makefile.am | 1 + doc/man/doveadm-copy.1 | 1 + doc/man/doveadm-move.1.in | 37 ++++++++++++++++++++++++++++++------- doc/man/doveadm.1.in | 7 ++++++- 4 files changed, 38 insertions(+), 8 deletions(-) diffs (126 lines): diff -r 87ff0023fe79 -r a913f29068a7 doc/man/Makefile.am --- a/doc/man/Makefile.am Fri Nov 08 17:48:54 2013 +0200 +++ b/doc/man/Makefile.am Sat Nov 16 14:06:32 2013 +0000 @@ -5,6 +5,7 @@ dist_man1_MANS = \ deliver.1 \ doveadm-config.1 \ + doveadm-copy.1 \ doveadm-reload.1 \ doveadm-stop.1 diff -r 87ff0023fe79 -r a913f29068a7 doc/man/doveadm-copy.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/man/doveadm-copy.1 Sat Nov 16 14:06:32 2013 +0000 @@ -0,0 +1,1 @@ +.so man1/doveadm-move.1 diff -r 87ff0023fe79 -r a913f29068a7 doc/man/doveadm-move.1.in --- a/doc/man/doveadm-move.1.in Fri Nov 08 17:48:54 2013 +0200 +++ b/doc/man/doveadm-move.1.in Sat Nov 16 14:06:32 2013 +0000 @@ -1,8 +1,11 @@ .\" Copyright (c) 2011 Dovecot authors, see the included COPYING file -.TH DOVEADM\-MOVE 1 "2011-09-25" "Dovecot v2.2" "Dovecot" +.TH DOVEADM\-MOVE 1 "2013-11-16" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-move \- Move messages matching the given search query into another mailbox +.br +doveadm\-copy \- Copy messages matching the given search query into another +mailbox .\"------------------------------------------------------------------------ .SH SYNOPSIS .BR doveadm " [" \-Dv "] " move " [" \-S @@ -17,6 +20,20 @@ .BR doveadm " [" \-Dv "] " move " [" \-S .IR socket_path "] " .BI \-u " user destination search_query" +.\"------------------------------------- +.PP +.BR doveadm " [" \-Dv "] " copy " [" \-S +.IR socket_path "] " "destination search_query" +.br +.\"------------------------------------- +.BR doveadm " [" \-Dv "] " copy " [" \-S +.IR socket_path "] " +.BI \-A " destination search_query" +.br +.\"------------------------------------- +.BR doveadm " [" \-Dv "] " copy " [" \-S +.IR socket_path "] " +.BI \-u " user destination search_query" .\"------------------------------------------------------------------------ .SH DESCRIPTION .B doveadm move @@ -26,23 +43,28 @@ is used to restrict which messages are moved into the .I destination mailbox. +.br +.B doveadm copy +behaves the same as +.BR "doveadm move" , +except that copied messages will not be expunged after copying. .PP In the first form, .BR doveadm (1) -will executed the -.B move +will execute the +.BR move / copy action with the environment of the logged in system user. .PP In the second form, .BR doveadm (1) will iterate over all users, found in the configured .IR user_db (s), -and move each user\(aqs messages, matching the given +and move or copy each user\(aqs messages, matching the given .IR search_query , into the user\(aqs .IR destination " mailbox." .PP -In the third form, matching mails will be moved only for given +In the third form, matching mails will be moved or copied only for given .IR user (s). .\"------------------------------------------------------------------------ @INCLUDE:global-options@ @@ -60,14 +82,15 @@ .SH ARGUMENTS .TP .I destination -The name of the destination mailbox, into which the mails should be moved. +The name of the destination mailbox, into which the mails should be moved +or copied. The .I destination mailbox must exist, otherwise this command will fail. .\"------------------------------------- .TP .I search_query -Move messages matching the given search query. +Move/copy messages matching the given search query. See .BR doveadm\-search\-query (7) for details. diff -r 87ff0023fe79 -r a913f29068a7 doc/man/doveadm.1.in --- a/doc/man/doveadm.1.in Fri Nov 08 17:48:54 2013 +0200 +++ b/doc/man/doveadm.1.in Sat Nov 16 14:06:32 2013 +0000 @@ -1,5 +1,5 @@ .\" Copyright (c) 2010-2013 Dovecot authors, see the included COPYING file -.TH DOVEADM 1 "2013-08-05" "Dovecot v2.2" "Dovecot" +.TH DOVEADM 1 "2013-11-16" "Dovecot v2.2" "Dovecot" .SH NAME doveadm \- Dovecot\(aqs administration utility .\"------------------------------------------------------------------------ @@ -102,6 +102,11 @@ Move matching mails to the alternative storage. .\"------------------------------------- .TP +.B doveadm copy +.BR doveadm\-copy (1), +Copy messages matching the given search query into another mailbox. +.\"------------------------------------- +.TP .B doveadm deduplicate .BR doveadm\-deduplicate (1), Expunge duplicate messages. From dovecot at dovecot.org Sat Nov 16 17:46:22 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 Nov 2013 17:46:22 +0200 Subject: dovecot-2.2: auth: Don't crash if passwd-file is used for authen... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7e813ff117a3 changeset: 16954:7e813ff117a3 user: Timo Sirainen date: Sat Nov 16 17:46:10 2013 +0200 description: auth: Don't crash if passwd-file is used for authentication but it has no passwords. diffstat: src/auth/passdb-passwd-file.c | 8 +++----- 1 files changed, 3 insertions(+), 5 deletions(-) diffs (23 lines): diff -r a913f29068a7 -r 7e813ff117a3 src/auth/passdb-passwd-file.c --- a/src/auth/passdb-passwd-file.c Sat Nov 16 14:06:32 2013 +0000 +++ b/src/auth/passdb-passwd-file.c Sat Nov 16 17:46:10 2013 +0200 @@ -28,16 +28,14 @@ string_t *str; char **p; - *crypted_pass_r = pu->password; + *crypted_pass_r = pu->password != NULL ? pu->password : ""; *scheme_r = password_get_scheme(crypted_pass_r); if (*scheme_r == NULL) *scheme_r = request->passdb->passdb->default_pass_scheme; /* save the password so cache can use it */ - if (*crypted_pass_r != NULL) { - auth_request_set_field(request, "password", - *crypted_pass_r, *scheme_r); - } + auth_request_set_field(request, "password", + *crypted_pass_r, *scheme_r); if (pu->extra_fields != NULL) { str = t_str_new(512); From dovecot at dovecot.org Sat Nov 16 19:20:21 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 Nov 2013 19:20:21 +0200 Subject: dovecot-2.2: lib-index: mail_cache_lookup*() can now finds also ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c7e55077ea4e changeset: 16955:c7e55077ea4e user: Timo Sirainen date: Sat Nov 16 19:20:10 2013 +0200 description: lib-index: mail_cache_lookup*() can now finds also the latest mail_cache_add()ed data. diffstat: src/lib-index/mail-cache-transaction.c | 26 +++++++++++++++++++++----- 1 files changed, 21 insertions(+), 5 deletions(-) diffs (59 lines): diff -r 7e813ff117a3 -r c7e55077ea4e src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Sat Nov 16 17:46:10 2013 +0200 +++ b/src/lib-index/mail-cache-transaction.c Sat Nov 16 19:20:10 2013 +0200 @@ -52,6 +52,7 @@ &mail_index_module_register); static int mail_cache_transaction_lock(struct mail_cache_transaction_ctx *ctx); +static size_t mail_cache_transaction_update_last_rec_size(struct mail_cache_transaction_ctx *ctx); static void mail_index_transaction_cache_reset(struct mail_index_transaction *t) { @@ -293,7 +294,14 @@ recs[i].cache_data_pos); } } - *trans_next_idx = i; + *trans_next_idx = i + 1; + if (seq == ctx->prev_seq && i == count) { + /* update the unfinished record's (temporary) size and + return it */ + mail_cache_transaction_update_last_rec_size(ctx); + return CONST_PTR_OFFSET(ctx->cache_data->data, + ctx->last_rec_pos); + } return NULL; } @@ -446,10 +454,9 @@ return ret; } -static void -mail_cache_transaction_update_last_rec(struct mail_cache_transaction_ctx *ctx) +static size_t +mail_cache_transaction_update_last_rec_size(struct mail_cache_transaction_ctx *ctx) { - struct mail_cache_transaction_rec *trans_rec; struct mail_cache_record *rec; void *data; size_t size; @@ -458,8 +465,17 @@ rec = PTR_OFFSET(data, ctx->last_rec_pos); rec->size = size - ctx->last_rec_pos; i_assert(rec->size > sizeof(*rec)); + return rec->size; +} - if (rec->size > MAIL_CACHE_RECORD_MAX_SIZE) { +static void +mail_cache_transaction_update_last_rec(struct mail_cache_transaction_ctx *ctx) +{ + struct mail_cache_transaction_rec *trans_rec; + size_t size; + + size = mail_cache_transaction_update_last_rec_size(ctx); + if (size > MAIL_CACHE_RECORD_MAX_SIZE) { buffer_set_used_size(ctx->cache_data, ctx->last_rec_pos); return; } From dovecot at dovecot.org Sat Nov 16 19:36:52 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 Nov 2013 19:36:52 +0200 Subject: dovecot-2.2: lib-index: Support a small in-memory cache even wit... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/03bf99704fe5 changeset: 16956:03bf99704fe5 user: Timo Sirainen date: Sat Nov 16 19:34:57 2013 +0200 description: lib-index: Support a small in-memory cache even with INDEX=MEMORY diffstat: src/lib-index/mail-cache-lookup.c | 8 +++++--- src/lib-index/mail-cache-transaction.c | 22 ++++++++++++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diffs (63 lines): diff -r c7e55077ea4e -r 03bf99704fe5 src/lib-index/mail-cache-lookup.c --- a/src/lib-index/mail-cache-lookup.c Sat Nov 16 19:20:10 2013 +0200 +++ b/src/lib-index/mail-cache-lookup.c Sat Nov 16 19:34:57 2013 +0200 @@ -191,15 +191,17 @@ if (ctx->offset == 0) { /* end of this record list. check newly appended data. */ if (view->trans_seq1 > ctx->seq || - view->trans_seq2 < ctx->seq || - MAIL_CACHE_IS_UNUSABLE(view->cache)) + view->trans_seq2 < ctx->seq) return 0; - /* check data still in memory */ + /* check data still in memory. this works for recent mails + even with INDEX=MEMORY */ if (!ctx->memory_appends_checked) { if (mail_cache_lookup_iter_transaction(ctx)) return 1; ctx->memory_appends_checked = TRUE; } + if (MAIL_CACHE_IS_UNUSABLE(view->cache)) + return 0; /* check data already written to cache file */ if (ctx->disk_appends_checked || diff -r c7e55077ea4e -r 03bf99704fe5 src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Sat Nov 16 19:20:10 2013 +0200 +++ b/src/lib-index/mail-cache-transaction.c Sat Nov 16 19:34:57 2013 +0200 @@ -592,6 +592,20 @@ struct mail_cache *cache = ctx->cache; int ret; + if (MAIL_INDEX_IS_IN_MEMORY(cache->index)) { + if (cache->file_fields_count <= field_idx) { + cache->file_field_map = + i_realloc(cache->file_field_map, + cache->file_fields_count * + sizeof(unsigned int), + (field_idx+1) * sizeof(unsigned int)); + cache->file_fields_count = field_idx+1; + } + cache->file_field_map[field_idx] = field_idx; + cache->field_file_map[field_idx] = field_idx; + return 0; + } + if (mail_cache_transaction_lock(ctx) <= 0) { if (MAIL_CACHE_IS_UNUSABLE(cache)) return -1; @@ -679,8 +693,12 @@ if (ret < 0) return; - if (ctx->cache_file_seq == 0) - ctx->cache_file_seq = ctx->cache->hdr->file_seq; + if (ctx->cache_file_seq == 0) { + if (MAIL_INDEX_IS_IN_MEMORY(ctx->cache->index)) + ctx->cache_file_seq = 1; + else + ctx->cache_file_seq = ctx->cache->hdr->file_seq; + } file_field = ctx->cache->field_file_map[field_idx]; i_assert(file_field != (uint32_t)-1); From dovecot at dovecot.org Sat Nov 16 19:36:52 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 Nov 2013 19:36:52 +0200 Subject: dovecot-2.2: lib-storage: API comment update Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/63a9b750027b changeset: 16957:63a9b750027b user: Timo Sirainen date: Sat Nov 16 19:35:10 2013 +0200 description: lib-storage: API comment update diffstat: src/lib-storage/mail-storage.h | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (15 lines): diff -r 03bf99704fe5 -r 63a9b750027b src/lib-storage/mail-storage.h --- a/src/lib-storage/mail-storage.h Sat Nov 16 19:34:57 2013 +0200 +++ b/src/lib-storage/mail-storage.h Sat Nov 16 19:35:10 2013 +0200 @@ -870,7 +870,10 @@ Do not use for structured fields (see mail_get_first_header_utf8()). */ int mail_get_headers_utf8(struct mail *mail, const char *field, const char *const **value_r); -/* Returns stream containing specified headers. */ +/* Returns stream containing specified headers. The returned stream will be + automatically freed when the mail is closed, or when another + mail_get_header_stream() call is made (so you can't have multiple header + streams open at the same time). */ int mail_get_header_stream(struct mail *mail, struct mailbox_header_lookup_ctx *headers, struct istream **stream_r); From dovecot at dovecot.org Sat Nov 16 19:36:52 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 Nov 2013 19:36:52 +0200 Subject: dovecot-2.2: imapc: Added imapc_feature fetch-header. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/00fc1e040603 changeset: 16958:00fc1e040603 user: Timo Sirainen date: Sat Nov 16 19:36:40 2013 +0200 description: imapc: Added imapc_feature fetch-header. It uses FETCH BODY.PEEK[HEADER.FIELDS (...)] whenever possible instead of fetching the entire header. diffstat: TODO | 4 - src/lib-storage/index/imapc/imapc-mail-fetch.c | 146 +++++++++++++++++++++++- src/lib-storage/index/imapc/imapc-mail.c | 78 ++++++++++++- src/lib-storage/index/imapc/imapc-mail.h | 5 +- src/lib-storage/index/imapc/imapc-settings.c | 1 + src/lib-storage/index/imapc/imapc-settings.h | 3 +- 6 files changed, 218 insertions(+), 19 deletions(-) diffs (truncated from 416 to 300 lines): diff -r 63a9b750027b -r 00fc1e040603 TODO --- a/TODO Sat Nov 16 19:35:10 2013 +0200 +++ b/TODO Sat Nov 16 19:36:40 2013 +0200 @@ -111,10 +111,6 @@ - doveadm search savedbefore 7d could be optimized in large mailboxes.. - mdbox: storage rebuilding could log about changes it does - mdbox: broken extrefs header keeps causing index rebuilds - - cache: mail_cache_lookup() should be able to return mail_cache_add()ed - fields even before they've been flushed to disk. this is useful when copying - messages and some plugin (e.g. mail_log) fetches some fields that are - already added to cache (to avoid opening and parsing the message) - doveadm -A <storage->client->client); } +static bool +headers_have_subset(const char *const *superset, const char *const *subset) +{ + unsigned int i; + + if (superset == NULL) + return FALSE; + if (subset != NULL) { + for (i = 0; subset[i] != NULL; i++) { + if (!str_array_icase_find(superset, subset[i])) + return FALSE; + } + } + return TRUE; +} + +static const char *const * +headers_merge(pool_t pool, const char *const *h1, const char *const *h2) +{ + ARRAY_TYPE(const_string) headers; + const char *value; + unsigned int i; + + p_array_init(&headers, pool, 16); + if (h1 != NULL) { + for (i = 0; h1[i] != NULL; i++) { + value = p_strdup(pool, h1[i]); + array_append(&headers, &value, 1); + } + } + if (h2 != NULL) { + for (i = 0; h2[i] != NULL; i++) { + if (h1 == NULL || !str_array_icase_find(h1, h2[i])) { + value = p_strdup(pool, h2[i]); + array_append(&headers, &value, 1); + } + } + } + array_append_zero(&headers); + return array_idx(&headers, 0); +} + static int -imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields) +imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields, + const char *const *headers) { struct imapc_mail *mail = (struct imapc_mail *)_mail; struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; @@ -60,13 +105,16 @@ struct mail_index_view *view; string_t *str; uint32_t seq; + unsigned int i; if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) return -1; /* drop any fields that we may already be fetching currently */ fields &= ~mail->fetching_fields; - if (fields == 0) + if (headers_have_subset(mail->fetching_headers, headers)) + headers = NULL; + if (fields == 0 && headers == NULL) return 0; if (!_mail->saving) { @@ -106,6 +154,19 @@ str_append(str, "BODY.PEEK[] "); else if ((fields & MAIL_FETCH_STREAM_HEADER) != 0) str_append(str, "BODY.PEEK[HEADER] "); + else if (headers != NULL) { + mail->fetching_headers = + headers_merge(mail->imail.mail.pool, headers, + mail->fetching_headers); + str_append(str, "BODY.PEEK[HEADER.FIELDS ("); + for (i = 0; mail->fetching_headers[i] != NULL; i++) { + if (i > 0) + str_append_c(str, ' '); + imap_append_astring(str, mail->fetching_headers[i]); + } + str_append(str, ")] "); + mail->header_list_fetched = FALSE; + } str_truncate(str, str_len(str)-1); str_append_c(str, ')'); @@ -174,7 +235,8 @@ fields |= MAIL_FETCH_STREAM_HEADER; } if (fields != 0) T_BEGIN { - (void)imapc_mail_send_fetch(_mail, fields); + (void)imapc_mail_send_fetch(_mail, fields, + data->wanted_headers->name); } T_END; return !mail->imail.data.prefetch_sent; } @@ -207,7 +269,8 @@ return TRUE; } -int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields) +int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields, + const char *const *headers) { struct imapc_mail *imail = (struct imapc_mail *)_mail; struct imapc_mailbox *mbox = @@ -223,7 +286,7 @@ } T_BEGIN { - ret = imapc_mail_send_fetch(_mail, fields); + ret = imapc_mail_send_fetch(_mail, fields, headers); } T_END; if (ret < 0) return -1; @@ -231,7 +294,9 @@ /* we'll continue waiting until we've got all the fields we wanted, or until all FETCH replies have been received (i.e. some FETCHes failed) */ - while (!imapc_mail_have_fields(imail, fields) && imail->fetch_count > 0) + while (imail->fetch_count > 0 && + (!imapc_mail_have_fields(imail, fields) || + !imail->header_list_fetched)) imapc_storage_run(mbox->storage); return 0; } @@ -360,6 +425,72 @@ imapc_mail_init_stream(mail, body); } +static void +imapc_fetch_header_stream(struct imapc_mail *mail, + const struct imapc_untagged_reply *reply, + const struct imap_arg *args) +{ + const enum message_header_parser_flags hdr_parser_flags = + MESSAGE_HEADER_PARSER_FLAG_SKIP_INITIAL_LWSP | + MESSAGE_HEADER_PARSER_FLAG_DROP_CR; + const struct imap_arg *hdr_list; + struct mailbox_header_lookup_ctx *headers_ctx; + struct message_header_parser_ctx *parser; + struct message_header_line *hdr; + struct istream *input; + ARRAY_TYPE(const_string) hdr_arr; + const char *value; + int ret, fd; + + if (!imap_arg_get_list(args, &hdr_list)) + return; + if (!imap_arg_atom_equals(args+1, "]")) + return; + args += 2; + + /* see if this is reply to the latest headers list request + (parse it even if it's not) */ + t_array_init(&hdr_arr, 16); + while (imap_arg_get_astring(hdr_list, &value)) { + array_append(&hdr_arr, &value, 1); + hdr_list++; + } + if (hdr_list->type != IMAP_ARG_EOL) + return; + array_append_zero(&hdr_arr); + + if (headers_have_subset(array_idx(&hdr_arr, 0), mail->fetching_headers)) + mail->header_list_fetched = TRUE; + + if (args->type == IMAP_ARG_LITERAL_SIZE) { + if (!imapc_find_lfile_arg(reply, args, &fd)) + return; + input = i_stream_create_fd(fd, 0, FALSE); + } else { + if (!imap_arg_get_nstring(args, &value)) + return; + if (value == NULL) { + mail_set_expunged(&mail->imail.mail.mail); + return; + } + input = i_stream_create_from_data(value, args->str_len); + } + + headers_ctx = mailbox_header_lookup_init(mail->imail.mail.mail.box, + array_idx(&hdr_arr, 0)); + index_mail_parse_header_init(&mail->imail, headers_ctx); + + parser = message_parse_header_init(input, NULL, hdr_parser_flags); + while ((ret = message_parse_header_next(parser, &hdr)) > 0) + index_mail_parse_header(NULL, hdr, &mail->imail); + i_assert(ret != 0); + index_mail_parse_header(NULL, NULL, &mail->imail); + message_parse_header_deinit(&parser); + + mailbox_header_lookup_unref(&headers_ctx); + i_stream_destroy(&input); +} + void imapc_mail_fetch_update(struct imapc_mail *mail, const struct imapc_untagged_reply *reply, const struct imap_arg *args) @@ -384,6 +515,9 @@ } else if (strcasecmp(key, "BODY[HEADER]") == 0) { imapc_fetch_stream(mail, reply, &args[i+1], FALSE); match = TRUE; + } else if (strcasecmp(key, "BODY[HEADER.FIELDS") == 0) { + imapc_fetch_header_stream(mail, reply, &args[i+1]); + match = TRUE; } else if (strcasecmp(key, "INTERNALDATE") == 0) { if (imap_arg_get_astring(&args[i+1], &value) && imap_parse_datetime(value, &t, &tz)) diff -r 63a9b750027b -r 00fc1e040603 src/lib-storage/index/imapc/imapc-mail.c --- a/src/lib-storage/index/imapc/imapc-mail.c Sat Nov 16 19:35:10 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail.c Sat Nov 16 19:36:40 2013 +0200 @@ -80,7 +80,7 @@ return 0; if (data->received_date == (time_t)-1) { - if (imapc_mail_fetch(_mail, MAIL_FETCH_RECEIVED_DATE) < 0) + if (imapc_mail_fetch(_mail, MAIL_FETCH_RECEIVED_DATE, NULL) < 0) return -1; if (data->received_date == (time_t)-1) { if (imapc_mail_failed(_mail, "INTERNALDATE") < 0) @@ -127,7 +127,7 @@ if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE) && data->stream == NULL) { /* trust RFC822.SIZE to be correct */ - if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE) < 0) + if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE, NULL) < 0) return -1; if (data->physical_size == (uoff_t)-1) { if (imapc_mail_failed(_mail, "RFC822.SIZE") < 0) @@ -159,6 +159,70 @@ } static int +imapc_mail_get_header_stream(struct mail *_mail, + struct mailbox_header_lookup_ctx *headers, + struct istream **stream_r) +{ + struct imapc_mail *mail = (struct imapc_mail *)_mail; + struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; + enum mail_lookup_abort old_abort = _mail->lookup_abort; + int ret; + + if (mail->imail.data.access_part != 0 || + !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS)) { + /* we're going to be reading the header/body anyway */ + return index_mail_get_header_stream(_mail, headers, stream_r); + } + + /* see if the wanted headers are already in cache */ + _mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL; + ret = index_mail_get_header_stream(_mail, headers, stream_r); + _mail->lookup_abort = old_abort; + if (ret == 0) + return 0; + + /* fetch only the wanted headers */ + if (imapc_mail_fetch(_mail, 0, headers->name) < 0) + return -1; + /* the headers should cached now. */ + return index_mail_get_header_stream(_mail, headers, stream_r); +} + +static int +imapc_mail_get_headers(struct mail *_mail, const char *field, + bool decode_to_utf8, const char *const **value_r) +{ From dovecot at dovecot.org Sat Nov 16 19:41:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 Nov 2013 19:41:14 +0200 Subject: dovecot-2.2: imapc: Crashfix for previous change Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/50026d936001 changeset: 16959:50026d936001 user: Timo Sirainen date: Sat Nov 16 19:41:07 2013 +0200 description: imapc: Crashfix for previous change diffstat: src/lib-storage/index/imapc/imapc-mail-fetch.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 00fc1e040603 -r 50026d936001 src/lib-storage/index/imapc/imapc-mail-fetch.c --- a/src/lib-storage/index/imapc/imapc-mail-fetch.c Sat Nov 16 19:36:40 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c Sat Nov 16 19:41:07 2013 +0200 @@ -236,6 +236,7 @@ } if (fields != 0) T_BEGIN { (void)imapc_mail_send_fetch(_mail, fields, + data->wanted_headers == NULL ? NULL : data->wanted_headers->name); } T_END; return !mail->imail.data.prefetch_sent; From dovecot at dovecot.org Sat Nov 16 20:10:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 Nov 2013 20:10:36 +0200 Subject: dovecot-2.2: imapc: Don't send FETCHes for data that is already ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/52fccca876b9 changeset: 16960:52fccca876b9 user: Timo Sirainen date: Sat Nov 16 20:06:34 2013 +0200 description: imapc: Don't send FETCHes for data that is already in cache. diffstat: src/lib-storage/index/imapc/imapc-mail-fetch.c | 3 +++ src/lib-storage/index/imapc/imapc-mail.c | 4 ++-- src/lib-storage/index/imapc/imapc-mail.h | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diffs (44 lines): diff -r 50026d936001 -r 52fccca876b9 src/lib-storage/index/imapc/imapc-mail-fetch.c --- a/src/lib-storage/index/imapc/imapc-mail-fetch.c Sat Nov 16 19:41:07 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c Sat Nov 16 20:06:34 2013 +0200 @@ -217,6 +217,9 @@ if (mbox->prev_mail_cache.uid == _mail->uid) imapc_mail_cache_get(mail, &mbox->prev_mail_cache); + /* try to get as much from cache as possible */ + imapc_mail_update_access_parts(&mail->imail); + if ((data->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0 && data->received_date == (time_t)-1) fields |= MAIL_FETCH_RECEIVED_DATE; diff -r 50026d936001 -r 52fccca876b9 src/lib-storage/index/imapc/imapc-mail.c --- a/src/lib-storage/index/imapc/imapc-mail.c Sat Nov 16 19:41:07 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail.c Sat Nov 16 20:06:34 2013 +0200 @@ -285,7 +285,7 @@ return TRUE; } -static void index_mail_update_access_parts(struct index_mail *mail) +void imapc_mail_update_access_parts(struct index_mail *mail) { struct mail *_mail = &mail->mail.mail; struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; @@ -340,7 +340,7 @@ struct index_mail *mail = (struct index_mail *)_mail; index_mail_add_temp_wanted_fields(_mail, fields, headers); - index_mail_update_access_parts(mail); + imapc_mail_update_access_parts(mail); } static void imapc_mail_close(struct mail *_mail) diff -r 50026d936001 -r 52fccca876b9 src/lib-storage/index/imapc/imapc-mail.h --- a/src/lib-storage/index/imapc/imapc-mail.h Sat Nov 16 19:41:07 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail.h Sat Nov 16 20:06:34 2013 +0200 @@ -33,5 +33,6 @@ void imapc_mail_fetch_update(struct imapc_mail *mail, const struct imapc_untagged_reply *reply, const struct imap_arg *args); +void imapc_mail_update_access_parts(struct index_mail *mail); #endif From dovecot at dovecot.org Sat Nov 16 20:10:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 Nov 2013 20:10:36 +0200 Subject: dovecot-2.2: imapc: Avoid FETCH INTERNALDATE for getting save_da... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4597e6572ba1 changeset: 16961:4597e6572ba1 user: Timo Sirainen date: Sat Nov 16 20:09:19 2013 +0200 description: imapc: Avoid FETCH INTERNALDATE for getting save_date when we already know received_date. diffstat: src/lib-storage/index/imapc/imapc-mail-fetch.c | 3 +++ src/lib-storage/index/imapc/imapc-mail.c | 6 ++++++ 2 files changed, 9 insertions(+), 0 deletions(-) diffs (29 lines): diff -r 52fccca876b9 -r 4597e6572ba1 src/lib-storage/index/imapc/imapc-mail-fetch.c --- a/src/lib-storage/index/imapc/imapc-mail-fetch.c Sat Nov 16 20:06:34 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c Sat Nov 16 20:09:19 2013 +0200 @@ -223,6 +223,9 @@ if ((data->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0 && data->received_date == (time_t)-1) fields |= MAIL_FETCH_RECEIVED_DATE; + if ((data->wanted_fields & MAIL_FETCH_SAVE_DATE) != 0 && + data->save_date == (time_t)-1 && data->received_date == (time_t)-1) + fields |= MAIL_FETCH_RECEIVED_DATE; if ((data->wanted_fields & MAIL_FETCH_PHYSICAL_SIZE) != 0 && data->physical_size == (uoff_t)-1 && IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) diff -r 52fccca876b9 -r 4597e6572ba1 src/lib-storage/index/imapc/imapc-mail.c --- a/src/lib-storage/index/imapc/imapc-mail.c Sat Nov 16 20:06:34 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail.c Sat Nov 16 20:09:19 2013 +0200 @@ -296,6 +296,12 @@ if ((data->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0) (void)index_mail_get_received_date(_mail, &date); + if ((data->wanted_fields & MAIL_FETCH_SAVE_DATE) != 0) { + if (index_mail_get_save_date(_mail, &date) < 0) { + (void)index_mail_get_received_date(_mail, &date); + data->save_date = data->received_date; + } + } if ((data->wanted_fields & MAIL_FETCH_PHYSICAL_SIZE) != 0) { if (index_mail_get_physical_size(_mail, &size) < 0 && !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) From dovecot at dovecot.org Sat Nov 16 20:10:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 Nov 2013 20:10:36 +0200 Subject: dovecot-2.2: imapc: When sending FETCH, always request also miss... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0c9b17f7cadf changeset: 16962:0c9b17f7cadf user: Timo Sirainen date: Sat Nov 16 20:10:27 2013 +0200 description: imapc: When sending FETCH, always request also missing wanted_fields. This avoids sending multiple FETCH commands when prefetch hadn't already fetched everything. diffstat: src/lib-storage/index/imapc/imapc-mail-fetch.c | 30 ++++++++++++++++++------- 1 files changed, 21 insertions(+), 9 deletions(-) diffs (58 lines): diff -r 4597e6572ba1 -r 0c9b17f7cadf src/lib-storage/index/imapc/imapc-mail-fetch.c --- a/src/lib-storage/index/imapc/imapc-mail-fetch.c Sat Nov 16 20:09:19 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c Sat Nov 16 20:10:27 2013 +0200 @@ -207,19 +207,14 @@ imapc_mail_init_stream(mail, TRUE); } -bool imapc_mail_prefetch(struct mail *_mail) +static enum mail_fetch_field +imapc_mail_get_wanted_fetch_fields(struct imapc_mail *mail) { - struct imapc_mail *mail = (struct imapc_mail *)_mail; - struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; + struct imapc_mailbox *mbox = + (struct imapc_mailbox *)mail->imail.mail.mail.box; struct index_mail_data *data = &mail->imail.data; enum mail_fetch_field fields = 0; - if (mbox->prev_mail_cache.uid == _mail->uid) - imapc_mail_cache_get(mail, &mbox->prev_mail_cache); - - /* try to get as much from cache as possible */ - imapc_mail_update_access_parts(&mail->imail); - if ((data->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0 && data->received_date == (time_t)-1) fields |= MAIL_FETCH_RECEIVED_DATE; @@ -240,6 +235,22 @@ else fields |= MAIL_FETCH_STREAM_HEADER; } + return fields; +} + +bool imapc_mail_prefetch(struct mail *_mail) +{ + struct imapc_mail *mail = (struct imapc_mail *)_mail; + struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; + struct index_mail_data *data = &mail->imail.data; + enum mail_fetch_field fields; + + if (mbox->prev_mail_cache.uid == _mail->uid) + imapc_mail_cache_get(mail, &mbox->prev_mail_cache); + /* try to get as much from cache as possible */ + imapc_mail_update_access_parts(&mail->imail); + + fields = imapc_mail_get_wanted_fetch_fields(mail); if (fields != 0) T_BEGIN { (void)imapc_mail_send_fetch(_mail, fields, data->wanted_headers == NULL ? NULL : @@ -292,6 +303,7 @@ return -1; } + fields |= imapc_mail_get_wanted_fetch_fields(imail); T_BEGIN { ret = imapc_mail_send_fetch(_mail, fields, headers); } T_END; From dovecot at dovecot.org Sat Nov 16 21:18:13 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 Nov 2013 21:18:13 +0200 Subject: dovecot-2.2: dsync: Give hints to lib-storage about what fields ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/da32044d4fc8 changeset: 16963:da32044d4fc8 user: Timo Sirainen date: Sat Nov 16 21:17:56 2013 +0200 description: dsync: Give hints to lib-storage about what fields we're going to fetch. diffstat: src/doveadm/dsync/dsync-mail.c | 6 ++++++ src/doveadm/dsync/dsync-mail.h | 4 ++++ src/doveadm/dsync/dsync-mailbox-export.c | 10 ++++++++++ 3 files changed, 20 insertions(+), 0 deletions(-) diffs (78 lines): diff -r 0c9b17f7cadf -r da32044d4fc8 src/doveadm/dsync/dsync-mail.c --- a/src/doveadm/dsync/dsync-mail.c Sat Nov 16 20:10:27 2013 +0200 +++ b/src/doveadm/dsync/dsync-mail.c Sat Nov 16 21:17:56 2013 +0200 @@ -18,6 +18,12 @@ "Date", "Message-ID", NULL }; +struct mailbox_header_lookup_ctx * +dsync_mail_get_hash_headers(struct mailbox *box) +{ + return mailbox_header_lookup_init(box, hashed_headers); +} + int dsync_mail_get_hdr_hash(struct mail *mail, const char **hdr_hash_r) { struct istream *hdr_input, *input; diff -r 0c9b17f7cadf -r da32044d4fc8 src/doveadm/dsync/dsync-mail.h --- a/src/doveadm/dsync/dsync-mail.h Sat Nov 16 20:10:27 2013 +0200 +++ b/src/doveadm/dsync/dsync-mail.h Sat Nov 16 21:17:56 2013 +0200 @@ -4,6 +4,7 @@ #include "mail-types.h" struct mail; +struct mailbox; struct dsync_mail { /* either GUID="" or uid=0 */ @@ -76,6 +77,9 @@ ARRAY_TYPE(const_string) keyword_changes; }; +struct mailbox_header_lookup_ctx * +dsync_mail_get_hash_headers(struct mailbox *box); + int dsync_mail_get_hdr_hash(struct mail *mail, const char **hdr_hash_r); int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r, const char **error_field_r); diff -r 0c9b17f7cadf -r da32044d4fc8 src/doveadm/dsync/dsync-mailbox-export.c --- a/src/doveadm/dsync/dsync-mailbox-export.c Sat Nov 16 20:10:27 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Sat Nov 16 21:17:56 2013 +0200 @@ -24,6 +24,7 @@ struct dsync_transaction_log_scan *log_scan; uint32_t last_common_uid; + struct mailbox_header_lookup_ctx *wanted_headers; struct mailbox_transaction_context *trans; struct mail_search_context *search_ctx; unsigned int search_pos, search_count; @@ -272,6 +273,10 @@ time_t save_timestamp; int ret; + mail_add_temp_wanted_fields(mail, MAIL_FETCH_GUID | + MAIL_FETCH_SAVE_DATE, + exporter->wanted_headers); + /* If message is already expunged here, just skip it */ if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) <= 0) return ret; @@ -468,6 +473,9 @@ p_array_init(&exporter->expunged_seqs, pool, 16); p_array_init(&exporter->expunged_guids, pool, 16); + if (!exporter->mails_have_guids) + exporter->wanted_headers = dsync_mail_get_hash_headers(box); + /* first scan transaction log and save any expunges and flag changes */ dsync_mailbox_export_log_scan(exporter, log_scan); /* get saves and also find GUIDs for flag changes */ @@ -843,6 +851,8 @@ dsync_mailbox_export_body_search_deinit(exporter); (void)mailbox_transaction_commit(&exporter->trans); + if (exporter->wanted_headers != NULL) + mailbox_header_lookup_unref(&exporter->wanted_headers); if (exporter->attr.value_stream != NULL) i_stream_unref(&exporter->attr.value_stream); From dovecot at dovecot.org Sat Nov 16 21:20:38 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 Nov 2013 21:20:38 +0200 Subject: dovecot-2.2: Use io_loop_set_current() instead of directly setti... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b740824110f4 changeset: 16964:b740824110f4 user: Timo Sirainen date: Sat Nov 16 21:20:31 2013 +0200 description: Use io_loop_set_current() instead of directly setting it. diffstat: src/doveadm/client-connection.c | 4 ++-- src/lib-dict/dict-memcached-ascii.c | 10 +++++----- src/lib-dict/dict-memcached.c | 4 ++-- src/lib-dict/dict-redis.c | 8 ++++---- src/lib-http/http-client-request.c | 4 ++-- src/lib-http/http-client.c | 4 ++-- src/lib-imap-client/imapc-client.c | 4 ++-- src/lib-sql/driver-pgsql.c | 4 ++-- src/lib-storage/index/pop3c/pop3c-client.c | 4 ++-- src/lib/ioloop.c | 5 ++--- 10 files changed, 25 insertions(+), 26 deletions(-) diffs (202 lines): diff -r da32044d4fc8 -r b740824110f4 src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Sat Nov 16 21:17:56 2013 +0200 +++ b/src/doveadm/client-connection.c Sat Nov 16 21:20:31 2013 +0200 @@ -127,10 +127,10 @@ doveadm_print_flush(); mail_storage_service_deinit(&ctx->storage_service); - current_ioloop = prev_ioloop; + io_loop_set_current(prev_ioloop); lib_signals_reset_ioloop(); o_stream_switch_ioloop(conn->output); - current_ioloop = ioloop; + io_loop_set_current(ioloop); io_loop_destroy(&ioloop); if (ret < 0) { diff -r da32044d4fc8 -r b740824110f4 src/lib-dict/dict-memcached-ascii.c --- a/src/lib-dict/dict-memcached-ascii.c Sat Nov 16 21:17:56 2013 +0200 +++ b/src/lib-dict/dict-memcached-ascii.c Sat Nov 16 21:20:31 2013 +0200 @@ -219,13 +219,13 @@ { struct ioloop *old_ioloop = current_ioloop; - current_ioloop = dict->ioloop; + io_loop_set_current(dict->ioloop); if (dict->to != NULL) dict->to = io_loop_move_timeout(&dict->to); connection_switch_ioloop(&dict->conn.conn); io_loop_run(dict->ioloop); - current_ioloop = old_ioloop; + io_loop_set_current(old_ioloop); if (dict->to != NULL) dict->to = io_loop_move_timeout(&dict->to); connection_switch_ioloop(&dict->conn.conn); @@ -406,7 +406,7 @@ i_array_init(&dict->replies, 4); dict->ioloop = io_loop_create(); - current_ioloop = old_ioloop; + io_loop_set_current(old_ioloop); *dict_r = &dict->dict; return 0; } @@ -421,9 +421,9 @@ (void)memcached_ascii_wait(dict); connection_deinit(&dict->conn.conn); - current_ioloop = dict->ioloop; + io_loop_set_current(dict->ioloop); io_loop_destroy(&dict->ioloop); - current_ioloop = old_ioloop; + io_loop_set_current(old_ioloop); str_free(&dict->conn.reply_str); array_free(&dict->replies); diff -r da32044d4fc8 -r b740824110f4 src/lib-dict/dict-memcached.c --- a/src/lib-dict/dict-memcached.c Sat Nov 16 21:17:56 2013 +0200 +++ b/src/lib-dict/dict-memcached.c Sat Nov 16 21:20:31 2013 +0200 @@ -328,9 +328,9 @@ timeout_remove(&to); } - current_ioloop = prev_ioloop; + io_loop_set_current(prev_ioloop); connection_switch_ioloop(&dict->conn.conn); - current_ioloop = dict->ioloop; + io_loop_set_current(dict->ioloop); io_loop_destroy(&dict->ioloop); if (!dict->conn.reply.reply_received) { diff -r da32044d4fc8 -r b740824110f4 src/lib-dict/dict-redis.c --- a/src/lib-dict/dict-redis.c Sat Nov 16 21:17:56 2013 +0200 +++ b/src/lib-dict/dict-redis.c Sat Nov 16 21:20:31 2013 +0200 @@ -109,9 +109,9 @@ io_loop_run(dict->ioloop); } while (array_count(&dict->input_states) > 0); - current_ioloop = prev_ioloop; + io_loop_set_current(prev_ioloop); connection_switch_ioloop(&dict->conn.conn); - current_ioloop = dict->ioloop; + io_loop_set_current(dict->ioloop); io_loop_destroy(&dict->ioloop); } @@ -464,9 +464,9 @@ timeout_remove(&to); } - current_ioloop = prev_ioloop; + io_loop_set_current(prev_ioloop); connection_switch_ioloop(&dict->conn.conn); - current_ioloop = dict->ioloop; + io_loop_set_current(dict->ioloop); io_loop_destroy(&dict->ioloop); if (!dict->conn.value_received) { diff -r da32044d4fc8 -r b740824110f4 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sat Nov 16 21:17:56 2013 +0200 +++ b/src/lib-http/http-client-request.c Sat Nov 16 21:20:31 2013 +0200 @@ -435,9 +435,9 @@ } } - current_ioloop = prev_ioloop; + io_loop_set_current(prev_ioloop); http_client_switch_ioloop(client); - current_ioloop = client->ioloop; + io_loop_set_current(client->ioloop); io_loop_destroy(&client->ioloop); if (req->state == HTTP_REQUEST_STATE_FINISHED) diff -r da32044d4fc8 -r b740824110f4 src/lib-http/http-client.c --- a/src/lib-http/http-client.c Sat Nov 16 21:17:56 2013 +0200 +++ b/src/lib-http/http-client.c Sat Nov 16 21:20:31 2013 +0200 @@ -199,9 +199,9 @@ http_client_debug(client, "All requests finished"); - current_ioloop = prev_ioloop; + io_loop_set_current(prev_ioloop); http_client_switch_ioloop(client); - current_ioloop = client->ioloop; + io_loop_set_current(client->ioloop); io_loop_destroy(&client->ioloop); } diff -r da32044d4fc8 -r b740824110f4 src/lib-imap-client/imapc-client.c --- a/src/lib-imap-client/imapc-client.c Sat Nov 16 21:17:56 2013 +0200 +++ b/src/lib-imap-client/imapc-client.c Sat Nov 16 21:20:31 2013 +0200 @@ -159,7 +159,7 @@ if (io_loop_is_running(client->ioloop)) io_loop_run(client->ioloop); - current_ioloop = prev_ioloop; + io_loop_set_current(prev_ioloop); } static void imapc_client_run_post(struct imapc_client *client) @@ -171,7 +171,7 @@ array_foreach(&client->conns, connp) imapc_connection_ioloop_changed((*connp)->conn); - current_ioloop = ioloop; + io_loop_set_current(ioloop); io_loop_destroy(&ioloop); } diff -r da32044d4fc8 -r b740824110f4 src/lib-sql/driver-pgsql.c --- a/src/lib-sql/driver-pgsql.c Sat Nov 16 21:17:56 2013 +0200 +++ b/src/lib-sql/driver-pgsql.c Sat Nov 16 21:20:31 2013 +0200 @@ -86,10 +86,10 @@ /* switch back to original ioloop in case the caller wants to add/remove timeouts */ if (db->ioloop != NULL) - current_ioloop = db->orig_ioloop; + io_loop_set_current(db->orig_ioloop); sql_db_set_state(&db->api, state); if (db->ioloop != NULL) - current_ioloop = db->ioloop; + io_loop_set_current(db->ioloop); } static void driver_pgsql_stop_io(struct pgsql_db *db) diff -r da32044d4fc8 -r b740824110f4 src/lib-storage/index/pop3c/pop3c-client.c --- a/src/lib-storage/index/pop3c/pop3c-client.c Sat Nov 16 21:17:56 2013 +0200 +++ b/src/lib-storage/index/pop3c/pop3c-client.c Sat Nov 16 21:20:31 2013 +0200 @@ -236,9 +236,9 @@ if (timeout_added && client->to != NULL) timeout_remove(&client->to); - current_ioloop = prev_ioloop; + io_loop_set_current(prev_ioloop); pop3c_client_ioloop_changed(client); - current_ioloop = ioloop; + io_loop_set_current(ioloop); io_loop_destroy(&ioloop); } diff -r da32044d4fc8 -r b740824110f4 src/lib/ioloop.c --- a/src/lib/ioloop.c Sat Nov 16 21:17:56 2013 +0200 +++ b/src/lib/ioloop.c Sat Nov 16 21:20:31 2013 +0200 @@ -456,8 +456,7 @@ io_loop_default_time_moved; ioloop->prev = current_ioloop; - current_ioloop = ioloop; - + io_loop_set_current(ioloop); return ioloop; } @@ -498,7 +497,7 @@ /* ->prev won't work unless loops are destroyed in create order */ i_assert(ioloop == current_ioloop); - current_ioloop = current_ioloop->prev; + io_loop_set_current(current_ioloop->prev); i_free(ioloop); } From dovecot at dovecot.org Sat Nov 16 21:58:39 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 16 Nov 2013 21:58:39 +0200 Subject: dovecot-2.2: lib-index: Recent change broke cache transactions f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/94a467b0cb34 changeset: 16965:94a467b0cb34 user: Timo Sirainen date: Sat Nov 16 21:58:31 2013 +0200 description: lib-index: Recent change broke cache transactions for more than 1 mail. diffstat: src/lib-index/mail-cache-transaction.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r b740824110f4 -r 94a467b0cb34 src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Sat Nov 16 21:20:31 2013 +0200 +++ b/src/lib-index/mail-cache-transaction.c Sat Nov 16 21:58:31 2013 +0200 @@ -485,7 +485,7 @@ trans_rec = array_append_space(&ctx->cache_data_seq); trans_rec->seq = ctx->prev_seq; trans_rec->cache_data_pos = ctx->last_rec_pos; - ctx->last_rec_pos = size; + ctx->last_rec_pos = ctx->cache_data->used; } static void From dovecot at dovecot.org Sun Nov 17 00:41:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 17 Nov 2013 00:41:35 +0200 Subject: dovecot-2.2: imapc: Fixes to imapc_feature fetch-headers Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/be0e619a018b changeset: 16966:be0e619a018b user: Timo Sirainen date: Sun Nov 17 00:41:23 2013 +0200 description: imapc: Fixes to imapc_feature fetch-headers diffstat: src/lib-storage/index/imapc/imapc-mail-fetch.c | 2 +- src/lib-storage/index/imapc/imapc-mail.c | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diffs (41 lines): diff -r 94a467b0cb34 -r be0e619a018b src/lib-storage/index/imapc/imapc-mail-fetch.c --- a/src/lib-storage/index/imapc/imapc-mail-fetch.c Sat Nov 16 21:58:31 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c Sun Nov 17 00:41:23 2013 +0200 @@ -156,7 +156,7 @@ str_append(str, "BODY.PEEK[HEADER] "); else if (headers != NULL) { mail->fetching_headers = - headers_merge(mail->imail.mail.pool, headers, + headers_merge(mail->imail.mail.data_pool, headers, mail->fetching_headers); str_append(str, "BODY.PEEK[HEADER.FIELDS ("); for (i = 0; mail->fetching_headers[i] != NULL; i++) { diff -r 94a467b0cb34 -r be0e619a018b src/lib-storage/index/imapc/imapc-mail.c --- a/src/lib-storage/index/imapc/imapc-mail.c Sat Nov 16 21:58:31 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail.c Sun Nov 17 00:41:23 2013 +0200 @@ -308,13 +308,15 @@ data->access_part |= READ_HDR | READ_BODY; } - if (data->access_part == 0 && data->wanted_headers != NULL) { + if (data->access_part == 0 && data->wanted_headers != NULL && + !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS)) { /* see if all wanted headers exist in cache */ if (!imapc_mail_has_headers_in_cache(mail, data->wanted_headers)) data->access_part |= PARSE_HDR; } if (data->access_part == 0 && - (data->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) != 0) { + (data->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) != 0 && + !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS)) { /* the common code already checked this partially, but we need a guaranteed correct answer */ header_ctx = mailbox_header_lookup_init(_mail->box, @@ -360,6 +362,7 @@ index_mail_close(_mail); + mail->fetching_headers = NULL; if (mail->body_fetched) { imapc_mail_cache_free(cache); cache->uid = _mail->uid; From dovecot at dovecot.org Sun Nov 17 01:02:11 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 17 Nov 2013 01:02:11 +0200 Subject: dovecot-2.2: lib-imap-client: Make sure command timeout is alway... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f80bd0ace609 changeset: 16967:f80bd0ace609 user: Timo Sirainen date: Sun Nov 17 00:59:39 2013 +0200 description: lib-imap-client: Make sure command timeout is always set. diffstat: src/lib-imap-client/imapc-connection.c | 41 ++++++++++++++++----------------- 1 files changed, 20 insertions(+), 21 deletions(-) diffs (72 lines): diff -r be0e619a018b -r f80bd0ace609 src/lib-imap-client/imapc-connection.c --- a/src/lib-imap-client/imapc-connection.c Sun Nov 17 00:41:23 2013 +0200 +++ b/src/lib-imap-client/imapc-connection.c Sun Nov 17 00:59:39 2013 +0200 @@ -1481,6 +1481,19 @@ pool_unref(&cmd->pool); } +static void imapc_command_timeout(struct imapc_connection *conn) +{ + struct imapc_command *const *cmds; + unsigned int count; + + cmds = array_get(&conn->cmd_wait_list, &count); + i_assert(count > 0); + + i_error("imapc(%s): Command '%s' timed out, disconnecting", + conn->name, imapc_command_get_readable(cmds[0])); + imapc_connection_disconnect(conn); +} + static bool parse_sync_literal(const unsigned char *data, unsigned int pos, unsigned int *value_r) @@ -1632,6 +1645,13 @@ return; } + /* add timeout for commands if there's not one yet + (pre-login has its own timeout) */ + if (conn->to == NULL) { + conn->to = timeout_add(conn->client->set.cmd_timeout_msecs, + imapc_command_timeout, conn); + } + timeout_reset(conn->to_output); if ((ret = imapc_command_try_send_stream(conn, cmd)) == 0) return; @@ -1672,19 +1692,6 @@ } } -static void imapc_command_timeout(struct imapc_connection *conn) -{ - struct imapc_command *const *cmds; - unsigned int count; - - cmds = array_get(&conn->cmd_wait_list, &count); - i_assert(count > 0); - - i_error("imapc(%s): Command '%s' timed out, disconnecting", - conn->name, imapc_command_get_readable(cmds[0])); - imapc_connection_disconnect(conn); -} - static void imapc_connection_send_idle_done(struct imapc_connection *conn) { if ((conn->idling || conn->idle_plus_waiting) && !conn->idle_stopping) { @@ -1707,14 +1714,6 @@ return; } - if (conn->state == IMAPC_CONNECTION_STATE_DONE) { - /* add timeout for commands if there's not one yet - (pre-login has its own timeout) */ - if (conn->to == NULL) { - conn->to = timeout_add(conn->client->set.cmd_timeout_msecs, - imapc_command_timeout, conn); - } - } if ((cmd->flags & IMAPC_COMMAND_FLAG_SELECT) != 0 && conn->selected_box == NULL) { /* reopening the mailbox. add it before other From dovecot at dovecot.org Sun Nov 17 01:02:11 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 17 Nov 2013 01:02:11 +0200 Subject: dovecot-2.2: lib-imap-client: Don't send NOOP if we're still wai... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3937b2e39963 changeset: 16968:3937b2e39963 user: Timo Sirainen date: Sun Nov 17 01:01:50 2013 +0200 description: lib-imap-client: Don't send NOOP if we're still waiting for a command reply. diffstat: src/lib-imap-client/imapc-connection.c | 10 +++++++--- 1 files changed, 7 insertions(+), 3 deletions(-) diffs (21 lines): diff -r f80bd0ace609 -r 3937b2e39963 src/lib-imap-client/imapc-connection.c --- a/src/lib-imap-client/imapc-connection.c Sun Nov 17 00:59:39 2013 +0200 +++ b/src/lib-imap-client/imapc-connection.c Sun Nov 17 01:01:50 2013 +0200 @@ -1301,10 +1301,14 @@ { struct imapc_command *cmd; - if (!conn->idling) + if (conn->idling) + cmd = imapc_connection_cmd(conn, imapc_reidle_callback, conn); + else if (array_count(&conn->cmd_wait_list) == 0) cmd = imapc_connection_cmd(conn, imapc_noop_callback, NULL); - else - cmd = imapc_connection_cmd(conn, imapc_reidle_callback, conn); + else { + /* IMAP command reply is taking a long time */ + return; + } imapc_command_send(cmd, "NOOP"); } From dovecot at dovecot.org Sun Nov 17 01:04:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 17 Nov 2013 01:04:58 +0200 Subject: dovecot-2.2: Added io_loop_add/remove_switch_callback() to call ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/54eb87b7761c changeset: 16969:54eb87b7761c user: Timo Sirainen date: Sun Nov 17 01:04:50 2013 +0200 description: Added io_loop_add/remove_switch_callback() to call a callback when ioloop is changed. diffstat: src/lib/ioloop.c | 45 +++++++++++++++++++++++++++++++++++++++++---- src/lib/ioloop.h | 4 ++++ 2 files changed, 45 insertions(+), 4 deletions(-) diffs (100 lines): diff -r 3937b2e39963 -r 54eb87b7761c src/lib/ioloop.c --- a/src/lib/ioloop.c Sun Nov 17 01:01:50 2013 +0200 +++ b/src/lib/ioloop.c Sun Nov 17 01:04:50 2013 +0200 @@ -16,6 +16,7 @@ struct timeval ioloop_timeval; struct ioloop *current_ioloop = NULL; +static ARRAY(io_switch_callback_t *) io_switch_callbacks = ARRAY_INIT; static void io_loop_initialize_handler(struct ioloop *ioloop) { @@ -467,6 +468,10 @@ *_ioloop = NULL; + /* ->prev won't work unless loops are destroyed in create order */ + i_assert(ioloop == current_ioloop); + io_loop_set_current(current_ioloop->prev); + if (ioloop->notify_handler_context != NULL) io_loop_notify_handler_deinit(ioloop); @@ -495,10 +500,6 @@ if (ioloop->cur_ctx != NULL) io_loop_context_deactivate(ioloop->cur_ctx); - /* ->prev won't work unless loops are destroyed in create order */ - i_assert(ioloop == current_ioloop); - io_loop_set_current(current_ioloop->prev); - i_free(ioloop); } @@ -508,9 +509,45 @@ ioloop->time_moved_callback = callback; } +static void io_switch_callbacks_free(void) +{ + array_free(&io_switch_callbacks); +} + void io_loop_set_current(struct ioloop *ioloop) { + io_switch_callback_t *const *callbackp; + struct ioloop *prev_ioloop = current_ioloop; + current_ioloop = ioloop; + if (array_is_created(&io_switch_callbacks)) { + array_foreach(&io_switch_callbacks, callbackp) + (*callbackp)(prev_ioloop); + } +} + +void io_loop_add_switch_callback(io_switch_callback_t *callback) +{ + if (!array_is_created(&io_switch_callbacks)) { + i_array_init(&io_switch_callbacks, 4); + lib_atexit(io_switch_callbacks_free); + } + array_append(&io_switch_callbacks, &callback, 1); +} + +void io_loop_remove_switch_callback(io_switch_callback_t *callback) +{ + io_switch_callback_t *const *callbackp; + unsigned int idx; + + array_foreach(&io_switch_callbacks, callbackp) { + if (*callbackp == callback) { + idx = array_foreach_idx(&io_switch_callbacks, callbackp); + array_delete(&io_switch_callbacks, idx, 1); + return; + } + } + i_unreached(); } struct ioloop_context *io_loop_context_new(struct ioloop *ioloop) diff -r 3937b2e39963 -r 54eb87b7761c src/lib/ioloop.h --- a/src/lib/ioloop.h Sun Nov 17 01:01:50 2013 +0200 +++ b/src/lib/ioloop.h Sun Nov 17 01:04:50 2013 +0200 @@ -32,6 +32,7 @@ typedef void io_callback_t(void *context); typedef void timeout_callback_t(void *context); typedef void io_loop_time_moved_callback_t(time_t old_time, time_t new_time); +typedef void io_switch_callback_t(struct ioloop *prev_ioloop); /* Time when the I/O loop started calling handlers. Can be used instead of time(NULL). */ @@ -112,6 +113,9 @@ /* Change the current_ioloop. */ void io_loop_set_current(struct ioloop *ioloop); +/* Call the callback whenever ioloop is changed. */ +void io_loop_add_switch_callback(io_switch_callback_t *callback); +void io_loop_remove_switch_callback(io_switch_callback_t *callback); /* This context is used for all further I/O and timeout callbacks that are added until returning to ioloop. When a callback is called, this context is From dovecot at dovecot.org Sun Nov 17 14:04:33 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 17 Nov 2013 14:04:33 +0200 Subject: dovecot-2.2: lib-imap-client: Make sure command timeout is used ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5726807ed06d changeset: 16970:5726807ed06d user: Timo Sirainen date: Sun Nov 17 14:04:22 2013 +0200 description: lib-imap-client: Make sure command timeout is used also for IDLE DONE. diffstat: src/lib-imap-client/imapc-connection.c | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diffs (23 lines): diff -r 54eb87b7761c -r 5726807ed06d src/lib-imap-client/imapc-connection.c --- a/src/lib-imap-client/imapc-connection.c Sun Nov 17 01:04:50 2013 +0200 +++ b/src/lib-imap-client/imapc-connection.c Sun Nov 17 14:04:22 2013 +0200 @@ -953,7 +953,7 @@ conn->idle_plus_waiting = FALSE; conn->idling = TRUE; /* no timeouting while IDLEing */ - if (conn->to != NULL) + if (conn->to != NULL && !conn->idle_stopping) timeout_remove(&conn->to); } else if (cmds_count > 0 && cmds[0]->wait_for_literal) { /* reply for literal */ @@ -1701,6 +1701,10 @@ if ((conn->idling || conn->idle_plus_waiting) && !conn->idle_stopping) { conn->idle_stopping = TRUE; o_stream_nsend_str(conn->output, "DONE\r\n"); + if (conn->to == NULL) { + conn->to = timeout_add(conn->client->set.cmd_timeout_msecs, + imapc_command_timeout, conn); + } } } From dovecot at dovecot.org Sun Nov 17 15:03:45 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 17 Nov 2013 15:03:45 +0200 Subject: dovecot-2.2: lib-imap-client: Don't reset timeout while waiting ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fa4c87b510ea changeset: 16971:fa4c87b510ea user: Timo Sirainen date: Sun Nov 17 15:03:31 2013 +0200 description: lib-imap-client: Don't reset timeout while waiting for IDLE DONE reply. diffstat: src/lib-imap-client/imapc-connection.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 5726807ed06d -r fa4c87b510ea src/lib-imap-client/imapc-connection.c --- a/src/lib-imap-client/imapc-connection.c Sun Nov 17 14:04:22 2013 +0200 +++ b/src/lib-imap-client/imapc-connection.c Sun Nov 17 15:03:31 2013 +0200 @@ -1439,7 +1439,7 @@ if (conn->input == NULL) return; - if (conn->to != NULL) + if (conn->to != NULL && !conn->idle_stopping) timeout_reset(conn->to); o_stream_cork(conn->output); From dovecot at dovecot.org Sun Nov 17 15:16:43 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 17 Nov 2013 15:16:43 +0200 Subject: dovecot-2.2: imapc: Avoid sending unnecessary IDLEs that are imm... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ef649e2a980e changeset: 16972:ef649e2a980e user: Timo Sirainen date: Sun Nov 17 15:13:58 2013 +0200 description: imapc: Avoid sending unnecessary IDLEs that are immediately aborted. diffstat: src/lib-imap-client/imapc-client-private.h | 3 +++ src/lib-imap-client/imapc-client.c | 18 ++++++++++++++++-- src/lib-storage/index/imapc/imapc-sync.c | 10 +++++++--- 3 files changed, 26 insertions(+), 5 deletions(-) diffs (77 lines): diff -r fa4c87b510ea -r ef649e2a980e src/lib-imap-client/imapc-client-private.h --- a/src/lib-imap-client/imapc-client-private.h Sun Nov 17 15:03:31 2013 +0200 +++ b/src/lib-imap-client/imapc-client-private.h Sun Nov 17 15:13:58 2013 +0200 @@ -3,6 +3,8 @@ #include "imapc-client.h" +#define IMAPC_CLIENT_IDLE_SEND_DELAY_MSECS 100 + struct imapc_client_connection { struct imapc_connection *conn; struct imapc_client_mailbox *box; @@ -27,6 +29,7 @@ struct imapc_client *client; struct imapc_connection *conn; struct imapc_msgmap *msgmap; + struct timeout *to_send_idle; void (*reopen_callback)(void *context); void *reopen_context; diff -r fa4c87b510ea -r ef649e2a980e src/lib-imap-client/imapc-client.c --- a/src/lib-imap-client/imapc-client.c Sun Nov 17 15:03:31 2013 +0200 +++ b/src/lib-imap-client/imapc-client.c Sun Nov 17 15:13:58 2013 +0200 @@ -337,6 +337,8 @@ } imapc_msgmap_deinit(&box->msgmap); + if (box->to_send_idle != NULL) + timeout_remove(&box->to_send_idle); i_free(box); } @@ -359,10 +361,22 @@ return box->msgmap; } +static void imapc_client_mailbox_idle_send(struct imapc_client_mailbox *box) +{ + timeout_remove(&box->to_send_idle); + if (imapc_client_mailbox_is_opened(box)) + imapc_connection_idle(box->conn); +} + void imapc_client_mailbox_idle(struct imapc_client_mailbox *box) { - if (imapc_client_mailbox_is_opened(box)) - imapc_connection_idle(box->conn); + /* send the IDLE with a delay to avoid unnecessary IDLEs that are + immediately aborted */ + if (box->to_send_idle == NULL && imapc_client_mailbox_is_opened(box)) { + box->to_send_idle = + timeout_add_short(IMAPC_CLIENT_IDLE_SEND_DELAY_MSECS, + imapc_client_mailbox_idle_send, box); + } box->reconnect_ok = TRUE; } diff -r fa4c87b510ea -r ef649e2a980e src/lib-storage/index/imapc/imapc-sync.c --- a/src/lib-storage/index/imapc/imapc-sync.c Sun Nov 17 15:03:31 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-sync.c Sun Nov 17 15:13:58 2013 +0200 @@ -458,9 +458,13 @@ } capabilities = imapc_client_get_capabilities(mbox->storage->client->client); - if ((capabilities & IMAPC_CAPABILITY_IDLE) == 0) { - /* IDLE not supported. do NOOP to get latest changes - before starting sync. */ + if ((capabilities & IMAPC_CAPABILITY_IDLE) == 0 || + (flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0) { + /* do NOOP to make sure we have the latest changes before + starting sync. this is necessary either because se don't + support IDLE at all, or because we want to be sure that we + have the latest changes (IDLE is started with a small delay, + so we might not actually even be in IDLE right not) */ imapc_mailbox_noop(mbox); } From dovecot at dovecot.org Sun Nov 17 16:11:45 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 17 Nov 2013 16:11:45 +0200 Subject: dovecot-2.2: dsync: Set wanted_fields/headers for the whole sear... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/12b6698e91a9 changeset: 16973:12b6698e91a9 user: Timo Sirainen date: Sun Nov 17 16:11:31 2013 +0200 description: dsync: Set wanted_fields/headers for the whole search request when possible. diffstat: src/doveadm/dsync/dsync-mailbox-export.c | 13 ++++++++++++- 1 files changed, 12 insertions(+), 1 deletions(-) diffs (39 lines): diff -r ef649e2a980e -r 12b6698e91a9 src/doveadm/dsync/dsync-mailbox-export.c --- a/src/doveadm/dsync/dsync-mailbox-export.c Sun Nov 17 15:13:58 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Sun Nov 17 16:11:31 2013 +0200 @@ -273,6 +273,8 @@ time_t save_timestamp; int ret; + /* update wanted fields in case we didn't already set them for the + search */ mail_add_temp_wanted_fields(mail, MAIL_FETCH_GUID | MAIL_FETCH_SAVE_DATE, exporter->wanted_headers); @@ -338,6 +340,8 @@ struct mail_search_args *search_args; struct mail_search_arg *sarg; struct mail *mail; + enum mail_fetch_field wanted_fields = 0; + struct mailbox_header_lookup_ctx *wanted_headers = NULL; int ret; search_args = mail_search_build_init(); @@ -357,9 +361,16 @@ (uint32_t)-1); } + if (exporter->last_common_uid == 0) { + /* we're syncing all mails, so we can request the wanted + fields for all the mails */ + wanted_fields = MAIL_FETCH_GUID | MAIL_FETCH_SAVE_DATE; + wanted_headers = exporter->wanted_headers; + } + exporter->trans = mailbox_transaction_begin(exporter->box, 0); search_ctx = mailbox_search_init(exporter->trans, search_args, NULL, - 0, NULL); + wanted_fields, wanted_headers); mail_search_args_unref(&search_args); while (mailbox_search_next(search_ctx, &mail)) { From dovecot at dovecot.org Sun Nov 17 17:15:55 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 17 Nov 2013 17:15:55 +0200 Subject: dovecot-2.2: dsync: Tell search that we want to fetch mail strea... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aec5d4170e74 changeset: 16974:aec5d4170e74 user: Timo Sirainen date: Sun Nov 17 16:58:42 2013 +0200 description: dsync: Tell search that we want to fetch mail streams so prefetching works. diffstat: src/doveadm/dsync/dsync-mailbox-export.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r 12b6698e91a9 -r aec5d4170e74 src/doveadm/dsync/dsync-mailbox-export.c --- a/src/doveadm/dsync/dsync-mailbox-export.c Sun Nov 17 16:11:31 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Sun Nov 17 16:58:42 2013 +0200 @@ -724,7 +724,9 @@ MAIL_FETCH_GUID | MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_POP3_ORDER | - MAIL_FETCH_RECEIVED_DATE, NULL); + MAIL_FETCH_RECEIVED_DATE | + MAIL_FETCH_STREAM_HEADER | + MAIL_FETCH_STREAM_BODY, NULL); mail_search_args_unref(&search_args); return array_count(&sarg->value.seqset) > 0 ? 1 : 0; } From dovecot at dovecot.org Sun Nov 17 17:15:55 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 17 Nov 2013 17:15:55 +0200 Subject: dovecot-2.2: imapc: Merge identical FETCH commands together (onl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/10462996fa32 changeset: 16975:10462996fa32 user: Timo Sirainen date: Sun Nov 17 16:59:33 2013 +0200 description: imapc: Merge identical FETCH commands together (only updating UID range) diffstat: src/lib-storage/index/imapc/imapc-mail-fetch.c | 137 +++++++++++++++++++----- src/lib-storage/index/imapc/imapc-mail.c | 2 +- src/lib-storage/index/imapc/imapc-mail.h | 3 + src/lib-storage/index/imapc/imapc-mailbox.c | 11 +- src/lib-storage/index/imapc/imapc-save.c | 6 +- src/lib-storage/index/imapc/imapc-storage.c | 13 +- src/lib-storage/index/imapc/imapc-storage.h | 14 ++- src/lib-storage/index/imapc/imapc-sync.c | 2 +- 8 files changed, 140 insertions(+), 48 deletions(-) diffs (truncated from 399 to 300 lines): diff -r aec5d4170e74 -r 10462996fa32 src/lib-storage/index/imapc/imapc-mail-fetch.c --- a/src/lib-storage/index/imapc/imapc-mail-fetch.c Sun Nov 17 16:58:42 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c Sun Nov 17 16:59:33 2013 +0200 @@ -2,6 +2,7 @@ #include "lib.h" #include "str.h" +#include "ioloop.h" #include "istream.h" #include "istream-header-filter.h" #include "message-header-parser.h" @@ -13,29 +14,37 @@ #include "imapc-storage.h" static void -imapc_mail_prefetch_callback(const struct imapc_command_reply *reply, - void *context) +imapc_mail_fetch_callback(const struct imapc_command_reply *reply, + void *context) { - struct imapc_mail *mail = context; - struct imapc_mailbox *mbox = - (struct imapc_mailbox *)mail->imail.mail.mail.box; + struct imapc_fetch_request *request = context; + struct imapc_fetch_request *const *requests; + struct imapc_mail *const *mailp; + struct imapc_mailbox *mbox = NULL; + unsigned int i, count; - i_assert(mail->fetch_count > 0); + array_foreach(&request->mails, mailp) { + struct imapc_mail *mail = *mailp; - if (--mail->fetch_count == 0) { - struct imapc_mail *const *fetch_mails; - unsigned int i, count; + i_assert(mail->fetch_count > 0); + if (--mail->fetch_count == 0) + mail->fetching_fields = 0; + pool_unref(&mail->imail.mail.pool); + mbox = (struct imapc_mailbox *)mail->imail.mail.mail.box; + } + i_assert(mbox != NULL); - fetch_mails = array_get(&mbox->fetch_mails, &count); - for (i = 0; i < count; i++) { - if (fetch_mails[i] == mail) { - array_delete(&mbox->fetch_mails, i, 1); - break; - } + requests = array_get(&mbox->fetch_requests, &count); + for (i = 0; i < count; i++) { + if (requests[i] == request) { + array_delete(&mbox->fetch_requests, i, 1); + break; } - i_assert(i != count); - mail->fetching_fields = 0; } + i_assert(i < count); + + array_free(&request->mails); + i_free(request); if (reply->state == IMAPC_COMMAND_STATE_OK) ; @@ -47,9 +56,8 @@ mail_storage_set_internal_error(&mbox->storage->storage); } else { mail_storage_set_critical(&mbox->storage->storage, - "imapc: Mail prefetch failed: %s", reply->text_full); + "imapc: Mail FETCH failed: %s", reply->text_full); } - pool_unref(&mail->imail.mail.pool); imapc_client_stop(mbox->storage->client->client); } @@ -95,13 +103,61 @@ return array_idx(&headers, 0); } +static bool +imapc_mail_try_merge_fetch(struct imapc_mailbox *mbox, string_t *str) +{ + const char *s1 = str_c(str); + const char *s2 = str_c(mbox->pending_fetch_cmd); + const char *p1, *p2; + + i_assert(strncmp(s1, "UID FETCH ", 10) == 0); + i_assert(strncmp(s2, "UID FETCH ", 10) == 0); + + /* skip over UID range */ + p1 = strchr(s1+10, ' '); + p2 = strchr(s2+10, ' '); + + if (null_strcmp(p1, p2) != 0) + return FALSE; + /* append the new UID to the pending FETCH UID range */ + str_truncate(str, p1-s1); + str_insert(mbox->pending_fetch_cmd, p2-s2, ","); + str_insert(mbox->pending_fetch_cmd, p2-s2+1, str_c(str) + 10); + return TRUE; +} + +static void +imapc_mail_delayed_send_or_merge(struct imapc_mail *mail, string_t *str) +{ + struct imapc_mailbox *mbox = + (struct imapc_mailbox *)mail->imail.mail.mail.box; + + if (mbox->pending_fetch_request != NULL && + !imapc_mail_try_merge_fetch(mbox, str)) { + /* send the previous FETCH and create a new one */ + imapc_mail_fetch_flush(mbox); + } + if (mbox->pending_fetch_request == NULL) { + mbox->pending_fetch_request = + i_new(struct imapc_fetch_request, 1); + i_array_init(&mbox->pending_fetch_request->mails, 4); + i_assert(mbox->pending_fetch_cmd->used == 0); + str_append_str(mbox->pending_fetch_cmd, str); + } + array_append(&mbox->pending_fetch_request->mails, &mail, 1); + + if (mbox->to_pending_fetch_send == NULL) { + mbox->to_pending_fetch_send = + timeout_add_short(0, imapc_mail_fetch_flush, mbox); + } +} + static int imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields, const char *const *headers) { struct imapc_mail *mail = (struct imapc_mail *)_mail; struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; - struct imapc_command *cmd; struct mail_index_view *view; string_t *str; uint32_t seq; @@ -172,15 +228,10 @@ pool_ref(mail->imail.mail.pool); mail->fetching_fields |= fields; - if (mail->fetch_count++ == 0) - array_append(&mbox->fetch_mails, &mail, 1); + mail->fetch_count++; - cmd = imapc_client_mailbox_cmd(mbox->client_box, - imapc_mail_prefetch_callback, mail); - imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); - imapc_command_send(cmd, str_c(str)); - mail->imail.data.prefetch_sent = TRUE; - return 0; + imapc_mail_delayed_send_or_merge(mail, str); + return 1; } static void imapc_mail_cache_get(struct imapc_mail *mail, @@ -252,9 +303,10 @@ fields = imapc_mail_get_wanted_fetch_fields(mail); if (fields != 0) T_BEGIN { - (void)imapc_mail_send_fetch(_mail, fields, - data->wanted_headers == NULL ? NULL : - data->wanted_headers->name); + if (imapc_mail_send_fetch(_mail, fields, + data->wanted_headers == NULL ? NULL : + data->wanted_headers->name) > 0) + mail->imail.data.prefetch_sent = TRUE; } T_END; return !mail->imail.data.prefetch_sent; } @@ -316,10 +368,31 @@ while (imail->fetch_count > 0 && (!imapc_mail_have_fields(imail, fields) || !imail->header_list_fetched)) - imapc_storage_run(mbox->storage); + imapc_mailbox_run(mbox); return 0; } +void imapc_mail_fetch_flush(struct imapc_mailbox *mbox) +{ + struct imapc_command *cmd; + + if (mbox->pending_fetch_request == NULL) { + i_assert(mbox->to_pending_fetch_send == NULL); + return; + } + + cmd = imapc_client_mailbox_cmd(mbox->client_box, + imapc_mail_fetch_callback, + mbox->pending_fetch_request); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + imapc_command_send(cmd, str_c(mbox->pending_fetch_cmd)); + + array_append(&mbox->fetch_requests, &mbox->pending_fetch_request, 1); + mbox->pending_fetch_request = NULL; + timeout_remove(&mbox->to_pending_fetch_send); + str_truncate(mbox->pending_fetch_cmd, 0); +} + static bool imapc_find_lfile_arg(const struct imapc_untagged_reply *reply, const struct imap_arg *arg, int *fd_r) { diff -r aec5d4170e74 -r 10462996fa32 src/lib-storage/index/imapc/imapc-mail.c --- a/src/lib-storage/index/imapc/imapc-mail.c Sun Nov 17 16:58:42 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail.c Sun Nov 17 16:59:33 2013 +0200 @@ -358,7 +358,7 @@ struct imapc_mail_cache *cache = &mbox->prev_mail_cache; while (mail->fetch_count > 0) - imapc_storage_run(mbox->storage); + imapc_mailbox_run(mbox); index_mail_close(_mail); diff -r aec5d4170e74 -r 10462996fa32 src/lib-storage/index/imapc/imapc-mail.h --- a/src/lib-storage/index/imapc/imapc-mail.h Sun Nov 17 16:58:42 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail.h Sun Nov 17 16:59:33 2013 +0200 @@ -5,6 +5,7 @@ struct imap_arg; struct imapc_untagged_reply; +struct imapc_mailbox; struct imapc_mail { struct index_mail imail; @@ -28,11 +29,13 @@ int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields, const char *const *headers); bool imapc_mail_prefetch(struct mail *mail); +void imapc_mail_fetch_flush(struct imapc_mailbox *mbox); void imapc_mail_init_stream(struct imapc_mail *mail, bool have_body); void imapc_mail_fetch_update(struct imapc_mail *mail, const struct imapc_untagged_reply *reply, const struct imap_arg *args); void imapc_mail_update_access_parts(struct index_mail *mail); +void imapc_mail_command_flush(struct imapc_mailbox *mbox); #endif diff -r aec5d4170e74 -r 10462996fa32 src/lib-storage/index/imapc/imapc-mailbox.c --- a/src/lib-storage/index/imapc/imapc-mailbox.c Sun Nov 17 16:58:42 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mailbox.c Sun Nov 17 16:59:33 2013 +0200 @@ -254,6 +254,7 @@ struct imapc_mailbox *mbox) { uint32_t lseq, rseq = reply->num; + struct imapc_fetch_request *const *fetch_requestp; struct imapc_mail *const *mailp; const struct imap_arg *list, *flags_list; const char *atom; @@ -303,11 +304,13 @@ return; /* if this is a reply to some FETCH request, update the mail's fields */ - array_foreach(&mbox->fetch_mails, mailp) { - struct imapc_mail *mail = *mailp; + array_foreach(&mbox->fetch_requests, fetch_requestp) { + array_foreach(&(*fetch_requestp)->mails, mailp) { + struct imapc_mail *mail = *mailp; - if (mail->imail.mail.mail.uid == uid) - imapc_mail_fetch_update(mail, reply, list); + if (mail->imail.mail.mail.uid == uid) + imapc_mail_fetch_update(mail, reply, list); + } } if (lseq == 0) { diff -r aec5d4170e74 -r 10462996fa32 src/lib-storage/index/imapc/imapc-save.c --- a/src/lib-storage/index/imapc/imapc-save.c Sun Nov 17 16:58:42 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-save.c Sun Nov 17 16:59:33 2013 +0200 @@ -246,7 +246,7 @@ ctx->mbox->box.name, flags, internaldate, input); i_stream_unref(&input); while (sctx.ret == -2) - imapc_storage_run(ctx->mbox->storage); + imapc_mailbox_run(ctx->mbox); if (sctx.ret == 0 && ctx->mbox->selected && !ctx->mbox->exists_received) { @@ -259,7 +259,7 @@ imapc_save_noop_callback, &sctx); imapc_command_send(cmd, "NOOP"); while (sctx.ret == -2) - imapc_storage_run(ctx->mbox->storage); + imapc_mailbox_run(ctx->mbox); } return sctx.ret; } @@ -421,7 +421,7 @@ imapc_command_sendf(cmd, "UID COPY %u %s", mail->uid, _t->box->name); while (sctx.ret == -2) - imapc_storage_run(src_mbox->storage); + imapc_mailbox_run(src_mbox); ctx->finished = TRUE; From dovecot at dovecot.org Sun Nov 17 17:15:55 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 17 Nov 2013 17:15:55 +0200 Subject: dovecot-2.2: lib-storage: Fixed memory leak in mail_add_temp_wan... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a04c7365299e changeset: 16976:a04c7365299e user: Timo Sirainen date: Sun Nov 17 17:15:42 2013 +0200 description: lib-storage: Fixed memory leak in mail_add_temp_wanted_fields() diffstat: src/lib-storage/index/index-mail.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 10462996fa32 -r a04c7365299e src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Sun Nov 17 16:59:33 2013 +0200 +++ b/src/lib-storage/index/index-mail.c Sun Nov 17 17:15:42 2013 +0200 @@ -1603,6 +1603,8 @@ for (i = 0; i < headers->count; i++) array_append(&names, &headers->name[i], 1); array_append_zero(&names); + if (data->wanted_headers != NULL) + mailbox_header_lookup_unref(&data->wanted_headers); data->wanted_headers = mailbox_header_lookup_init(_mail->box, array_idx(&names, 0)); From dovecot at dovecot.org Sun Nov 17 17:59:21 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 17 Nov 2013 17:59:21 +0200 Subject: dovecot-2.2: dsync: Support syncing storages with 128bit GUIDs <... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8a7ecddca4ad changeset: 16977:8a7ecddca4ad user: Timo Sirainen date: Sun Nov 17 17:59:05 2013 +0200 description: dsync: Support syncing storages with 128bit GUIDs <-> string GUIDs. diffstat: src/doveadm/dsync/dsync-brain-mailbox.c | 3 +++ src/doveadm/dsync/dsync-ibc-stream.c | 6 +++++- src/doveadm/dsync/dsync-mailbox-import.c | 22 +++++++++++++++------- src/doveadm/dsync/dsync-mailbox-import.h | 3 ++- src/doveadm/dsync/dsync-mailbox.h | 2 +- src/lib-storage/mail-storage-private.h | 5 ++++- src/lib-storage/mail-storage.c | 2 ++ src/lib-storage/mail-storage.h | 2 ++ 8 files changed, 34 insertions(+), 11 deletions(-) diffs (192 lines): diff -r a04c7365299e -r 8a7ecddca4ad src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Sun Nov 17 17:15:42 2013 +0200 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Sun Nov 17 17:59:05 2013 +0200 @@ -184,6 +184,9 @@ (remote_dsync_box->have_save_guids || (brain->backup_recv && remote_dsync_box->have_guids))) import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS; + if (brain->local_dsync_box.have_only_guid128 || + remote_dsync_box->have_only_guid128) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128; brain->box_importer = brain->backup_send ? NULL : dsync_mailbox_import_init(brain->box, brain->log_scan, diff -r a04c7365299e -r 8a7ecddca4ad src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Sun Nov 17 17:15:42 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Sun Nov 17 17:59:05 2013 +0200 @@ -98,7 +98,7 @@ .chr = 'B', .required_keys = "mailbox_guid uid_validity uid_next messages_count " "first_recent_uid highest_modseq highest_pvt_modseq", - .optional_keys = "mailbox_lost cache_fields have_guids have_save_guids" + .optional_keys = "mailbox_lost cache_fields have_guids have_save_guids have_only_guid128" }, { .name = "mailbox_attribute", .chr = 'A', @@ -1178,6 +1178,8 @@ dsync_serializer_encode_add(encoder, "have_guids", ""); if (dsync_box->have_save_guids) dsync_serializer_encode_add(encoder, "have_save_guids", ""); + if (dsync_box->have_only_guid128) + dsync_serializer_encode_add(encoder, "have_only_guid128", ""); dsync_serializer_encode_add(encoder, "uid_validity", dec2str(dsync_box->uid_validity)); dsync_serializer_encode_add(encoder, "uid_next", @@ -1281,6 +1283,8 @@ if (dsync_deserializer_decode_try(decoder, "have_save_guids", &value) || (box->have_guids && ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_SAVE_GUID)) box->have_save_guids = TRUE; + if (dsync_deserializer_decode_try(decoder, "have_only_guid128", &value)) + box->have_only_guid128 = TRUE; value = dsync_deserializer_decode_get(decoder, "uid_validity"); if (str_to_uint32(value, &box->uid_validity) < 0) { dsync_ibc_input_error(ibc, decoder, "Invalid uid_validity"); diff -r a04c7365299e -r 8a7ecddca4ad src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Sun Nov 17 17:15:42 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Sun Nov 17 17:59:05 2013 +0200 @@ -103,6 +103,7 @@ unsigned int master_brain:1; unsigned int revert_local_changes:1; unsigned int mails_have_guids:1; + unsigned int mails_use_guid128:1; }; static void dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer, @@ -218,6 +219,8 @@ importer->debug = (flags & DSYNC_MAILBOX_IMPORT_FLAG_DEBUG) != 0; importer->mails_have_guids = (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS) != 0; + importer->mails_use_guid128 = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128) != 0; mailbox_get_open_status(importer->box, STATUS_UIDNEXT | STATUS_HIGHESTMODSEQ | STATUS_HIGHESTPVTMODSEQ, @@ -463,12 +466,14 @@ } static bool -dsync_mail_change_guid_equals(const struct dsync_mail_change *change, +dsync_mail_change_guid_equals(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change, const char *guid, const char **cmp_guid_r) { guid_128_t guid_128, change_guid_128; - if (change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + if (change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE && + !importer->mails_use_guid128) { if (cmp_guid_r != NULL) *cmp_guid_r = change->guid; return strcmp(change->guid, guid) == 0; @@ -480,7 +485,7 @@ mail_generate_guid_128_hash(guid, guid_128); if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) { if (cmp_guid_r != NULL) { - *cmp_guid_r = t_strdup_printf("%s(expunged, orig=%s)", + *cmp_guid_r = t_strdup_printf("%s(guid128, orig=%s)", binary_to_hex(change_guid_128, sizeof(change_guid_128)), change->guid); } @@ -760,7 +765,7 @@ dsync_mail_error(importer, importer->mail, "GUID"); return FALSE; } - if (!dsync_mail_change_guid_equals(change, guid, &cmp_guid)) { + if (!dsync_mail_change_guid_equals(importer, change, guid, &cmp_guid)) { dsync_import_unexpected_state(importer, t_strdup_printf( "Unexpected GUID mismatch for UID=%u: %s != %s", change->uid, guid, cmp_guid)); @@ -778,7 +783,8 @@ if (change->guid == NULL || change->guid[0] == '\0' || importer->cur_guid[0] == '\0') return TRUE; - if (!dsync_mail_change_guid_equals(change, importer->cur_guid, &cmp_guid)) { + if (!dsync_mail_change_guid_equals(importer, change, + importer->cur_guid, &cmp_guid)) { dsync_import_unexpected_state(importer, t_strdup_printf( "Unexpected GUID mismatch (2) for UID=%u: %s != %s", change->uid, importer->cur_guid, cmp_guid)); @@ -1294,7 +1300,8 @@ if (*change->guid != '\0' && *importer->cur_guid != '\0') { /* we have GUIDs, verify them */ - if (dsync_mail_change_guid_equals(change, importer->cur_guid, NULL)) + if (dsync_mail_change_guid_equals(importer, change, + importer->cur_guid, NULL)) return 1; else return 0; @@ -1343,7 +1350,8 @@ return FALSE; i_assert(local_change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE); - if (dsync_mail_change_guid_equals(local_change, change->guid, NULL)) + if (dsync_mail_change_guid_equals(importer, local_change, + change->guid, NULL)) importer->last_common_uid = change->uid; else if (change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) dsync_mailbox_common_uid_found(importer); diff -r a04c7365299e -r 8a7ecddca4ad src/doveadm/dsync/dsync-mailbox-import.h --- a/src/doveadm/dsync/dsync-mailbox-import.h Sun Nov 17 17:15:42 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.h Sun Nov 17 17:59:05 2013 +0200 @@ -6,7 +6,8 @@ DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS = 0x02, DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES = 0x04, DSYNC_MAILBOX_IMPORT_FLAG_DEBUG = 0x08, - DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS = 0x10 + DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS = 0x10, + DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128 = 0x20 }; struct mailbox; diff -r a04c7365299e -r 8a7ecddca4ad src/doveadm/dsync/dsync-mailbox.h --- a/src/doveadm/dsync/dsync-mailbox.h Sun Nov 17 17:15:42 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox.h Sun Nov 17 17:59:05 2013 +0200 @@ -8,7 +8,7 @@ struct dsync_mailbox { guid_128_t mailbox_guid; bool mailbox_lost; - bool have_guids, have_save_guids; + bool have_guids, have_save_guids, have_only_guid128; uint32_t uid_validity, uid_next, messages_count, first_recent_uid; uint64_t highest_modseq, highest_pvt_modseq; diff -r a04c7365299e -r 8a7ecddca4ad src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Sun Nov 17 17:15:42 2013 +0200 +++ b/src/lib-storage/mail-storage-private.h Sun Nov 17 17:59:05 2013 +0200 @@ -72,7 +72,10 @@ MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS = 0x80, /* message content can be unstructured binary data (e.g. zlib plugin is allowed to compress/decompress mails) */ - MAIL_STORAGE_CLASS_FLAG_BINARY_DATA = 0x100 + MAIL_STORAGE_CLASS_FLAG_BINARY_DATA = 0x100, + /* Message GUIDs can only be 128bit (always set + mailbox_status.have_only_guid128) */ + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUID128 = 0x200 }; struct mail_binary_cache { diff -r a04c7365299e -r 8a7ecddca4ad src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Sun Nov 17 17:15:42 2013 +0200 +++ b/src/lib-storage/mail-storage.c Sun Nov 17 17:59:05 2013 +0200 @@ -1502,6 +1502,8 @@ status_r->have_guids = TRUE; if ((box->storage->class_flags & MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS) != 0) status_r->have_save_guids = TRUE; + if ((box->storage->class_flags & MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUID128) != 0) + status_r->have_only_guid128 = TRUE; } int mailbox_get_status(struct mailbox *box, diff -r a04c7365299e -r 8a7ecddca4ad src/lib-storage/mail-storage.h --- a/src/lib-storage/mail-storage.h Sun Nov 17 17:15:42 2013 +0200 +++ b/src/lib-storage/mail-storage.h Sun Nov 17 17:59:05 2013 +0200 @@ -282,6 +282,8 @@ unsigned int have_guids:1; /* mailbox_save_set_guid() works (always set) */ unsigned int have_save_guids:1; + /* GUIDs are always 128bit (always set) */ + unsigned int have_only_guid128:1; }; struct mailbox_cache_field { From dovecot at dovecot.org Sun Nov 17 19:01:17 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 17 Nov 2013 19:01:17 +0200 Subject: dovecot-2.2: man: Added doveadm-batch.1. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0ba1ca4552b7 changeset: 16978:0ba1ca4552b7 user: Pascal Volk date: Sun Nov 17 16:58:20 2013 +0000 description: man: Added doveadm-batch.1. diffstat: .hgignore | 2 +- doc/man/Makefile.am | 2 + doc/man/doveadm-batch.1.in | 88 ++++++++++++++++++++++++++++++++++++++++++++++ doc/man/doveadm.1.in | 5 ++ 4 files changed, 96 insertions(+), 1 deletions(-) diffs (137 lines): diff -r 8a7ecddca4ad -r 0ba1ca4552b7 .hgignore --- a/.hgignore Sun Nov 17 17:59:05 2013 +0200 +++ b/.hgignore Sun Nov 17 16:58:20 2013 +0000 @@ -105,5 +105,5 @@ syntax: regexp src/.*/test-[^\.]*$ -doc/man/doveadm-(altmove|auth|deduplicate|director|dump|exec|expunge|fetch|flags|import|instance|index|force-resync|help|kick|log|mailbox|mount|move|penalty|purge|pw|quota|search|user|who)\.1$ +doc/man/doveadm-(altmove|auth|batch|deduplicate|director|dump|exec|expunge|fetch|flags|import|instance|index|force-resync|help|kick|log|mailbox|mount|move|penalty|purge|pw|quota|search|user|who)\.1$ doc/man/(doveadm|doveconf|dovecot-lda|dovecot|dsync)\.1$ diff -r 8a7ecddca4ad -r 0ba1ca4552b7 doc/man/Makefile.am --- a/doc/man/Makefile.am Sun Nov 17 17:59:05 2013 +0200 +++ b/doc/man/Makefile.am Sun Nov 17 16:58:20 2013 +0000 @@ -16,6 +16,7 @@ doveadm.1 \ doveadm-altmove.1 \ doveadm-auth.1 \ + doveadm-batch.1 \ doveadm-deduplicate.1 \ doveadm-director.1 \ doveadm-dump.1 \ @@ -57,6 +58,7 @@ doveadm.1.in \ doveadm-altmove.1.in \ doveadm-auth.1.in \ + doveadm-batch.1.in \ doveadm-deduplicate.1.in \ doveadm-director.1.in \ doveadm-dump.1.in \ diff -r 8a7ecddca4ad -r 0ba1ca4552b7 doc/man/doveadm-batch.1.in --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/man/doveadm-batch.1.in Sun Nov 17 16:58:20 2013 +0000 @@ -0,0 +1,88 @@ +.\" Copyright (c) 2013 Dovecot authors, see the included COPYING file +.TH DOVEADM\-BATCH 1 "2013-11-17" "Dovecot v2.2" "Dovecot" +.SH NAME +doveadm\-batch \- Execute multiple commands for multiple users +.\"------------------------------------------------------------------------ +.SH SYNOPSIS +.BR doveadm " [" \-Dv "] " batch " [" \-S +.IR socket_path "] " +.BI \-A " sep command sep command" +[...] +.br +.\"------------------------------------- +.BR doveadm " [" \-Dv "] " batch " [" \-S +.IR socket_path "] " +.BI \-u " usermask sep command sep command" +[...] +.\"------------------------------------------------------------------------ +.SH DESCRIPTION +The command +.B doveadm batch +is used to execute multiple +.BR doveadm (1) +.IR command s +sequentially for multiple users. +This is primarily an optimization where Dovecot can do all the mailbox +operations for the same user at once, instead of having to go through +the users multiple times. +.PP +In the first form +.BR doveadm (1) +will loop over all users, defined in the configured +.IR user_db (s), +and execute the sequence of +.IR command s +for each of them. +.PP +In the second form the +.IR command s +will be executed for each user matching the given +.IR usermask . +.\"------------------------------------------------------------------------ + at INCLUDE:global-options@ +.\" --- command specific options --- "/. +.PP +Command specific +.IR options : +.\"------------------------------------- + at INCLUDE:option-A@ +.\"------------------------------------- + at INCLUDE:option-S-socket@ +.\"------------------------------------- + at INCLUDE:option-u-user@ +.\"------------------------------------------------------------------------ +.SH ARGUMENTS +.TP +.I command +Any +.BR doveadm (1) +command, with arguments and options \(em except for the +.BR \-A ", " \-S " and " \-u " options." +.\"------------------------------------- +.TP +.I sep +A single character used as command separator, e.g. \(aq:\(aq. +Be careful, unquoted characters, like +.BR ; " or " & , +may be interpreted by your shell and +.BR doveadm (1) +will never see them. +.\"------------------------------------------------------------------------ +.SH EXAMPLE +In this example we move seen mails, older than one month, for all +example.net users to the alternative storage under /nfsmount: +.br +.nf +mail_location = mdbox:~/mdbox:ALT=/nfsmount/%h/mdbox +.fi +The second command will remove messages with refcount=0 from the +primary storage. +.PP +.nf +.B doveadm batch \-u \(rs*@example.net : altmove seen savedbefore 30d : purge +.fi +.\"------------------------------------------------------------------------ + at INCLUDE:reporting-bugs@ +.\"------------------------------------------------------------------------ +.SH SEE ALSO +.BR doveadm (1) \ No newline at end of file diff -r 8a7ecddca4ad -r 0ba1ca4552b7 doc/man/doveadm.1.in --- a/doc/man/doveadm.1.in Sun Nov 17 17:59:05 2013 +0200 +++ b/doc/man/doveadm.1.in Sun Nov 17 16:58:20 2013 +0000 @@ -102,6 +102,11 @@ Move matching mails to the alternative storage. .\"------------------------------------- .TP +.B doveadm batch +.BR doveadm\-batch (1), +Execute multiple commands for multiple users. +.\"------------------------------------- +.TP .B doveadm copy .BR doveadm\-copy (1), Copy messages matching the given search query into another mailbox. From dovecot at dovecot.org Mon Nov 18 15:47:01 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 18 Nov 2013 15:47:01 +0200 Subject: dovecot-2.2: doveadm user: Fixed looking up multiple users when ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5870a1656b9f changeset: 16979:5870a1656b9f user: Timo Sirainen date: Mon Nov 18 15:46:46 2013 +0200 description: doveadm user: Fixed looking up multiple users when they had different UIDs diffstat: src/doveadm/doveadm-auth.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 0ba1ca4552b7 -r 5870a1656b9f src/doveadm/doveadm-auth.c --- a/src/doveadm/doveadm-auth.c Sun Nov 17 16:58:20 2013 +0000 +++ b/src/doveadm/doveadm-auth.c Mon Nov 18 15:46:46 2013 +0200 @@ -478,7 +478,8 @@ MAIL_STORAGE_SERVICE_FLAG_NO_CHDIR | MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT | MAIL_STORAGE_SERVICE_FLAG_NO_PLUGINS | - MAIL_STORAGE_SERVICE_FLAG_NO_NAMESPACES); + MAIL_STORAGE_SERVICE_FLAG_NO_NAMESPACES | + MAIL_STORAGE_SERVICE_FLAG_TEMP_PRIV_DROP); } while ((input.username = argv[optind++]) != NULL) { From dovecot at dovecot.org Mon Nov 18 16:19:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 18 Nov 2013 16:19:14 +0200 Subject: dovecot-2.2: lib-storage: Added mail_storage_service_set_auth_co... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c6f1909e52af changeset: 16980:c6f1909e52af user: Timo Sirainen date: Mon Nov 18 16:15:38 2013 +0200 description: lib-storage: Added mail_storage_service_set_auth_conn() diffstat: src/lib-storage/mail-storage-service.c | 18 ++++++++++++------ src/lib-storage/mail-storage-service.h | 3 +++ 2 files changed, 15 insertions(+), 6 deletions(-) diffs (53 lines): diff -r 5870a1656b9f -r c6f1909e52af src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Mon Nov 18 15:46:46 2013 +0200 +++ b/src/lib-storage/mail-storage-service.c Mon Nov 18 16:15:38 2013 +0200 @@ -913,6 +913,16 @@ return -1; } +void mail_storage_service_set_auth_conn(struct mail_storage_service_ctx *ctx, + struct auth_master_connection *conn) +{ + i_assert(ctx->conn == NULL); + i_assert(mail_user_auth_master_conn == NULL); + + ctx->conn = conn; + mail_user_auth_master_conn = conn; +} + static void mail_storage_service_first_init(struct mail_storage_service_ctx *ctx, const struct setting_parser_info *user_info, @@ -920,17 +930,13 @@ { enum auth_master_flags flags = 0; - i_assert(ctx->conn == NULL); - ctx->debug = mail_user_set_get_mail_debug(user_info, user_set); if (ctx->debug) flags |= AUTH_MASTER_FLAG_DEBUG; if ((ctx->flags & MAIL_STORAGE_SERVICE_FLAG_NO_IDLE_TIMEOUT) != 0) flags |= AUTH_MASTER_FLAG_NO_IDLE_TIMEOUT; - ctx->conn = auth_master_init(user_set->auth_socket_path, flags); - - i_assert(mail_user_auth_master_conn == NULL); - mail_user_auth_master_conn = ctx->conn; + mail_storage_service_set_auth_conn(ctx, + auth_master_init(user_set->auth_socket_path, flags)); } static void diff -r 5870a1656b9f -r c6f1909e52af src/lib-storage/mail-storage-service.h --- a/src/lib-storage/mail-storage-service.h Mon Nov 18 15:46:46 2013 +0200 +++ b/src/lib-storage/mail-storage-service.h Mon Nov 18 16:15:38 2013 +0200 @@ -62,6 +62,9 @@ enum mail_storage_service_flags flags) ATTR_NULL(2); struct auth_master_connection * mail_storage_service_get_auth_conn(struct mail_storage_service_ctx *ctx); +/* Set auth connection (instead of creating a new one automatically). */ +void mail_storage_service_set_auth_conn(struct mail_storage_service_ctx *ctx, + struct auth_master_connection *conn); int mail_storage_service_read_settings(struct mail_storage_service_ctx *ctx, const struct mail_storage_service_input *input, pool_t pool, From dovecot at dovecot.org Mon Nov 18 16:19:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 18 Nov 2013 16:19:14 +0200 Subject: dovecot-2.2: doveadm auth/user: Fixes to auth_socket_path handling. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c41700c4a9bb changeset: 16981:c41700c4a9bb user: Timo Sirainen date: Mon Nov 18 16:18:53 2013 +0200 description: doveadm auth/user: Fixes to auth_socket_path handling. doveadm user (without -u parameter) was ignoring "-a" parameter entirely. Others weren't using auth_socket_path setting. diffstat: src/doveadm/doveadm-auth.c | 67 +++++++++++++++++++++++------------------ src/doveadm/doveadm-settings.c | 3 + src/doveadm/doveadm-settings.h | 1 + 3 files changed, 42 insertions(+), 29 deletions(-) diffs (252 lines): diff -r c6f1909e52af -r c41700c4a9bb src/doveadm/doveadm-auth.c --- a/src/doveadm/doveadm-auth.c Mon Nov 18 16:15:38 2013 +0200 +++ b/src/doveadm/doveadm-auth.c Mon Nov 18 16:18:53 2013 +0200 @@ -7,7 +7,9 @@ #include "base64.h" #include "str.h" #include "wildcard-match.h" +#include "settings-parser.h" #include "master-service.h" +#include "master-service-settings.h" #include "auth-client.h" #include "auth-master.h" #include "auth-server-connection.h" @@ -29,24 +31,28 @@ static void auth_cmd_help(doveadm_command_t *cmd); +static struct auth_master_connection * +doveadm_get_auth_master_conn(const char *auth_socket_path) +{ + enum auth_master_flags flags = 0; + + if (doveadm_debug) + flags |= AUTH_MASTER_FLAG_DEBUG; + return auth_master_init(auth_socket_path, flags); +} + static int -cmd_user_input(const char *auth_socket_path, const struct authtest_input *input, +cmd_user_input(struct auth_master_connection *conn, + const struct authtest_input *input, const char *show_field, bool userdb) { const char *lookup_name = userdb ? "userdb lookup" : "passdb lookup"; - struct auth_master_connection *conn; pool_t pool; const char *username, *const *fields, *p; int ret; - if (auth_socket_path == NULL) { - auth_socket_path = t_strconcat(doveadm_settings->base_dir, - "/auth-userdb", NULL); - } - pool = pool_alloconly_create("auth master lookup", 1024); - conn = auth_master_init(auth_socket_path, 0); if (userdb) { ret = auth_master_user_lookup(conn, input->username, &input->info, pool, &username, &fields); @@ -86,7 +92,6 @@ } } } - auth_master_deinit(&conn); return ret; } @@ -191,23 +196,17 @@ } static void -cmd_user_list(const char *auth_socket_path, const struct authtest_input *input, +cmd_user_list(struct auth_master_connection *conn, + const struct authtest_input *input, char *const *users) { struct auth_master_user_list_ctx *ctx; - struct auth_master_connection *conn; const char *username, *user_mask = NULL; unsigned int i; - if (auth_socket_path == NULL) { - auth_socket_path = t_strconcat(doveadm_settings->base_dir, - "/auth-userdb", NULL); - } - if (users[0] != NULL && users[1] == NULL) user_mask = users[0]; - conn = auth_master_init(auth_socket_path, 0); ctx = auth_master_user_list_init(conn, user_mask, &input->info); while ((username = auth_master_user_list_next(ctx)) != NULL) { for (i = 0; users[i] != NULL; i++) { @@ -219,12 +218,11 @@ } if (auth_master_user_list_deinit(&ctx) < 0) i_fatal("user listing failed"); - auth_master_deinit(&conn); } static void cmd_auth_cache_flush(int argc, char *argv[]) { - const char *auth_socket_path = NULL; + const char *master_socket_path = NULL; struct auth_master_connection *conn; unsigned int count; int c; @@ -232,7 +230,7 @@ while ((c = getopt(argc, argv, "a:")) > 0) { switch (c) { case 'a': - auth_socket_path = optarg; + master_socket_path = optarg; break; default: auth_cmd_help(cmd_auth_cache_flush); @@ -240,12 +238,12 @@ } argv += optind; - if (auth_socket_path == NULL) { - auth_socket_path = t_strconcat(doveadm_settings->base_dir, - "/auth-master", NULL); + if (master_socket_path == NULL) { + master_socket_path = t_strconcat(doveadm_settings->base_dir, + "auth-master", NULL); } - conn = auth_master_init(auth_socket_path, 0); + conn = doveadm_get_auth_master_conn(master_socket_path); if (auth_master_cache_flush(conn, (void *)argv, &count) < 0) { i_error("Cache flush failed"); doveadm_exit_code = EX_TEMPFAIL; @@ -292,7 +290,8 @@ static void cmd_auth_lookup(int argc, char *argv[]) { - const char *auth_socket_path = NULL; + const char *auth_socket_path = doveadm_settings->auth_socket_path; + struct auth_master_connection *conn; struct authtest_input input; const char *show_field = NULL; bool first = TRUE; @@ -320,13 +319,14 @@ if (optind == argc) auth_cmd_help(cmd_auth_lookup); + conn = doveadm_get_auth_master_conn(auth_socket_path); while ((input.username = argv[optind++]) != NULL) { if (first) first = FALSE; else putchar('\n'); - ret = cmd_user_input(auth_socket_path, &input, show_field, FALSE); + ret = cmd_user_input(conn, &input, show_field, FALSE); switch (ret) { case -1: doveadm_exit_code = EX_TEMPFAIL; @@ -336,6 +336,7 @@ break; } } + auth_master_deinit(&conn); } static void cmd_user_mail_input_field(const char *key, const char *value, @@ -425,7 +426,8 @@ static void cmd_user(int argc, char *argv[]) { - const char *auth_socket_path = NULL; + const char *auth_socket_path = doveadm_settings->auth_socket_path; + struct auth_master_connection *conn; struct authtest_input input; const char *show_field = NULL; struct mail_storage_service_ctx *storage_service = NULL; @@ -458,6 +460,8 @@ if (optind == argc) auth_cmd_help(cmd_user); + conn = doveadm_get_auth_master_conn(auth_socket_path); + have_wildcards = FALSE; for (i = optind; argv[i] != NULL; i++) { if (strchr(argv[i], '*') != NULL || @@ -468,7 +472,8 @@ } if (have_wildcards) { - cmd_user_list(auth_socket_path, &input, argv + optind); + cmd_user_list(conn, &input, argv + optind); + auth_master_deinit(&conn); return; } @@ -480,6 +485,8 @@ MAIL_STORAGE_SERVICE_FLAG_NO_PLUGINS | MAIL_STORAGE_SERVICE_FLAG_NO_NAMESPACES | MAIL_STORAGE_SERVICE_FLAG_TEMP_PRIV_DROP); + mail_storage_service_set_auth_conn(storage_service, conn); + conn = NULL; } while ((input.username = argv[optind++]) != NULL) { @@ -490,7 +497,7 @@ ret = !userdb_only ? cmd_user_mail_input(storage_service, &input, show_field) : - cmd_user_input(auth_socket_path, &input, show_field, TRUE); + cmd_user_input(conn, &input, show_field, TRUE); switch (ret) { case -1: doveadm_exit_code = EX_TEMPFAIL; @@ -502,6 +509,8 @@ } if (storage_service != NULL) mail_storage_service_deinit(&storage_service); + if (conn != NULL) + auth_master_deinit(&conn); } struct doveadm_cmd doveadm_cmd_auth[] = { diff -r c6f1909e52af -r c41700c4a9bb src/doveadm/doveadm-settings.c --- a/src/doveadm/doveadm-settings.c Mon Nov 18 16:15:38 2013 +0200 +++ b/src/doveadm/doveadm-settings.c Mon Nov 18 16:18:53 2013 +0200 @@ -56,6 +56,7 @@ DEF(SET_STR, libexec_dir), DEF(SET_STR, mail_plugins), DEF(SET_STR, mail_plugin_dir), + DEF(SET_STR, auth_socket_path), DEF(SET_STR, doveadm_socket_path), DEF(SET_UINT, doveadm_worker_count), DEF(SET_UINT, doveadm_port), @@ -77,6 +78,7 @@ .libexec_dir = PKG_LIBEXECDIR, .mail_plugins = "", .mail_plugin_dir = MODULEDIR, + .auth_socket_path = "auth-userdb", .doveadm_socket_path = "doveadm-server", .doveadm_worker_count = 0, .doveadm_port = 0, @@ -125,6 +127,7 @@ struct doveadm_settings *set = _set; #ifndef CONFIG_BINARY + fix_base_path(set, pool, &set->auth_socket_path); fix_base_path(set, pool, &set->doveadm_socket_path); #endif if (*set->dsync_alt_char == '\0') { diff -r c6f1909e52af -r c41700c4a9bb src/doveadm/doveadm-settings.h --- a/src/doveadm/doveadm-settings.h Mon Nov 18 16:15:38 2013 +0200 +++ b/src/doveadm/doveadm-settings.h Mon Nov 18 16:18:53 2013 +0200 @@ -6,6 +6,7 @@ const char *libexec_dir; const char *mail_plugins; const char *mail_plugin_dir; + const char *auth_socket_path; const char *doveadm_socket_path; unsigned int doveadm_worker_count; unsigned int doveadm_port; From dovecot at dovecot.org Mon Nov 18 16:23:52 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 18 Nov 2013 16:23:52 +0200 Subject: dovecot-2.2: doveadm user: Fixed printing multiple users Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2503785cf989 changeset: 16982:2503785cf989 user: Timo Sirainen date: Mon Nov 18 16:23:42 2013 +0200 description: doveadm user: Fixed printing multiple users diffstat: src/doveadm/doveadm-auth.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r c41700c4a9bb -r 2503785cf989 src/doveadm/doveadm-auth.c --- a/src/doveadm/doveadm-auth.c Mon Nov 18 16:18:53 2013 +0200 +++ b/src/doveadm/doveadm-auth.c Mon Nov 18 16:23:42 2013 +0200 @@ -388,7 +388,7 @@ return 0; } - if (show_field == NULL) { + if (show_field == NULL && !doveadm_print_is_initialized()) { doveadm_print_init(DOVEADM_PRINT_TYPE_TAB); doveadm_print_header_simple("field"); doveadm_print_header_simple("value"); From dovecot at dovecot.org Mon Nov 18 16:26:42 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 18 Nov 2013 16:26:42 +0200 Subject: dovecot-2.2: doveadm user: Fixed now printing with a non-default... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d925674c4329 changeset: 16983:d925674c4329 user: Timo Sirainen date: Mon Nov 18 16:26:31 2013 +0200 description: doveadm user: Fixed now printing with a non-default formatter diffstat: src/doveadm/doveadm-auth.c | 11 +++++------ 1 files changed, 5 insertions(+), 6 deletions(-) diffs (28 lines): diff -r 2503785cf989 -r d925674c4329 src/doveadm/doveadm-auth.c --- a/src/doveadm/doveadm-auth.c Mon Nov 18 16:23:42 2013 +0200 +++ b/src/doveadm/doveadm-auth.c Mon Nov 18 16:26:31 2013 +0200 @@ -388,12 +388,6 @@ return 0; } - if (show_field == NULL && !doveadm_print_is_initialized()) { - doveadm_print_init(DOVEADM_PRINT_TYPE_TAB); - doveadm_print_header_simple("field"); - doveadm_print_header_simple("value"); - } - cmd_user_mail_input_field("uid", user->set->mail_uid, show_field); cmd_user_mail_input_field("gid", user->set->mail_gid, show_field); cmd_user_mail_input_field("home", user->set->mail_home, show_field); @@ -487,6 +481,11 @@ MAIL_STORAGE_SERVICE_FLAG_TEMP_PRIV_DROP); mail_storage_service_set_auth_conn(storage_service, conn); conn = NULL; + if (show_field == NULL) { + doveadm_print_init(DOVEADM_PRINT_TYPE_TAB); + doveadm_print_header_simple("field"); + doveadm_print_header_simple("value"); + } } while ((input.username = argv[optind++]) != NULL) { From dovecot at dovecot.org Tue Nov 19 16:13:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 19 Nov 2013 16:13:34 +0200 Subject: dovecot-2.2: lib-storage: Error handling fix for layout=index wh... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/908283e74473 changeset: 16984:908283e74473 user: Timo Sirainen date: Tue Nov 19 16:13:12 2013 +0200 description: lib-storage: Error handling fix for layout=index when mailbox creation fails. diffstat: src/lib-storage/list/mailbox-list-index-backend.c | 11 +++++++---- 1 files changed, 7 insertions(+), 4 deletions(-) diffs (38 lines): diff -r d925674c4329 -r 908283e74473 src/lib-storage/list/mailbox-list-index-backend.c --- a/src/lib-storage/list/mailbox-list-index-backend.c Mon Nov 18 16:26:31 2013 +0200 +++ b/src/lib-storage/list/mailbox-list-index-backend.c Tue Nov 19 16:13:12 2013 +0200 @@ -244,6 +244,7 @@ struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box); struct index_mailbox_list *list = (struct index_mailbox_list *)box->list; + struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(box->list); struct mailbox_list_index_sync_context *sync_ctx; struct mailbox_list_index_record rec; struct mailbox_list_index_node *node; @@ -263,7 +264,7 @@ return 0; } - mail_index_lookup_ext(sync_ctx->view, seq, sync_ctx->ilist->ext_id, + mail_index_lookup_ext(sync_ctx->view, seq, ilist->ext_id, &data, &expunged); i_assert(data != NULL && !expunged); memcpy(&rec, data, sizeof(rec)); @@ -274,12 +275,14 @@ mail_index_update_flags(sync_ctx->trans, seq, MODIFY_REPLACE, 0); memcpy(rec.guid, update->mailbox_guid, sizeof(rec.guid)); - mail_index_update_ext(sync_ctx->trans, seq, sync_ctx->ilist->ext_id, - &rec, NULL); + mail_index_update_ext(sync_ctx->trans, seq, ilist->ext_id, &rec, NULL); success = ibox->module_ctx.super.create_box(box, update, FALSE) == 0; - if (mailbox_list_index_sync_end(&sync_ctx, success) < 0) + if (mailbox_list_index_sync_end(&sync_ctx, success) < 0) { + /* make sure we forget any changes done internally */ + mailbox_list_index_reset(ilist); return -1; + } return 1; } From dovecot at dovecot.org Tue Nov 19 17:21:40 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 19 Nov 2013 17:21:40 +0200 Subject: dovecot-2.2: lmtp: Make sure output is flushed before disconnect... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5f946b807706 changeset: 16985:5f946b807706 user: Timo Sirainen date: Tue Nov 19 17:21:21 2013 +0200 description: lmtp: Make sure output is flushed before disconnection. diffstat: src/lmtp/client.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 908283e74473 -r 5f946b807706 src/lmtp/client.c --- a/src/lmtp/client.c Tue Nov 19 16:13:12 2013 +0200 +++ b/src/lmtp/client.c Tue Nov 19 17:21:21 2013 +0200 @@ -259,6 +259,7 @@ const char *reason) { client_disconnect(client, prefix, reason); + o_stream_uncork(client->output); clients_count--; DLLIST_REMOVE(&clients, client); From dovecot at dovecot.org Tue Nov 19 23:17:21 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 19 Nov 2013 23:17:21 +0200 Subject: dovecot-2.2: auth: Fixed potential crash. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dd321a3304fa changeset: 16986:dd321a3304fa user: Timo Sirainen date: Tue Nov 19 23:17:07 2013 +0200 description: auth: Fixed potential crash. diffstat: src/auth/auth-request-handler.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 5f946b807706 -r dd321a3304fa src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Tue Nov 19 17:21:21 2013 +0200 +++ b/src/auth/auth-request-handler.c Tue Nov 19 23:17:07 2013 +0200 @@ -174,7 +174,7 @@ AUTH_FIELD_FLAG_HIDDEN, 0); if (request->original_username != NULL && - strcmp(request->original_username, request->user) != 0) { + null_strcmp(request->original_username, request->user) != 0) { auth_str_add_keyvalue(dest, "original_user", request->original_username); } From dovecot at dovecot.org Tue Nov 19 23:24:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 19 Nov 2013 23:24:08 +0200 Subject: dovecot-2.2: Released v2.2.8. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/095a777edc26 changeset: 16987:095a777edc26 user: Timo Sirainen date: Tue Nov 19 23:18:01 2013 +0200 description: Released v2.2.8. diffstat: NEWS | 28 ++++++++++++++++++++++++++++ configure.ac | 2 +- 2 files changed, 29 insertions(+), 1 deletions(-) diffs (47 lines): diff -r dd321a3304fa -r 095a777edc26 NEWS --- a/NEWS Tue Nov 19 23:17:07 2013 +0200 +++ b/NEWS Tue Nov 19 23:18:01 2013 +0200 @@ -1,3 +1,31 @@ +v2.2.8 2013-11-19 Timo Sirainen + + + Mail cache lookups work for the mail being saved. This improves + performance by avoiding the need to parse the mail multiple times + when using some plugins (e.g. mail_log). + + Mail cache works for recently cached data also with in-memory + indexes. + + imapc: Many performance improvements, especially when working with + dsync. Also added imapc_feature=fetch-headers which allows using + FETCH BODY.PEEK[HEADER.FIELDS (..)] to avoid reading the entire + header. + + mail_location = ..:FULLDIRNAME=dbox-Mails is the same as + :DIRNAME=dbox-Mails, but it will also be used for + :INDEX and :CONTROL directories. (It should have worked this way + from the beginning, but can't be changed anymore without breaking + existing installations). + - Fixed infinite loop in message parsing if message ends with + "--boundary" and CR (without LF). Messages saved via SMTP/LMTP can't + trigger this, because messages must end with an "LF.". A user could + trigger this for him/herself though. + - lmtp: Client was sometimes disconnected before all the output was + sent to it. + - imap_zlib plugin caused crashes during client disconnection in + v2.2.7 + - replicator: Database wasn't being exported to disk every 15 minutes + as it should have. Instead it was being imported, causing "doveadm + replicator remove" commands to not work very well. + v2.2.7 2013-11-03 Timo Sirainen * Some usage of passdb checkpassword could have been exploitable by diff -r dd321a3304fa -r 095a777edc26 configure.ac --- a/configure.ac Tue Nov 19 23:17:07 2013 +0200 +++ b/configure.ac Tue Nov 19 23:18:01 2013 +0200 @@ -2,7 +2,7 @@ # Be sure to update ABI version also if anything changes that might require # recompiling plugins. Most importantly that means if any structs are changed. -AC_INIT([Dovecot],[2.2.7],[dovecot at dovecot.org]) +AC_INIT([Dovecot],[2.2.8],[dovecot at dovecot.org]) AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv7($PACKAGE_VERSION)", [Dovecot ABI version]) AC_CONFIG_SRCDIR([src]) From dovecot at dovecot.org Tue Nov 19 23:24:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 19 Nov 2013 23:24:08 +0200 Subject: dovecot-2.2: Added tag 2.2.8 for changeset 095a777edc26 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/112defd08e14 changeset: 16988:112defd08e14 user: Timo Sirainen date: Tue Nov 19 23:18:01 2013 +0200 description: Added tag 2.2.8 for changeset 095a777edc26 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 095a777edc26 -r 112defd08e14 .hgtags --- a/.hgtags Tue Nov 19 23:18:01 2013 +0200 +++ b/.hgtags Tue Nov 19 23:18:01 2013 +0200 @@ -111,3 +111,4 @@ 71f0696749ab347119278d7f68a01f81ed584c53 2.2.5 1de9a494cb2520986da97fa73cf87a3781ea2d17 2.2.6 b5b118701aa83e7b047e53f51aa5a4622ee812df 2.2.7 +095a777edc2623fad2ad9d6321ed93a804e85889 2.2.8 From dovecot at dovecot.org Tue Nov 19 23:24:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 19 Nov 2013 23:24:08 +0200 Subject: dovecot-2.2: Added signature for changeset 095a777edc26 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4a0c03f98ca3 changeset: 16989:4a0c03f98ca3 user: Timo Sirainen date: Tue Nov 19 23:18:11 2013 +0200 description: Added signature for changeset 095a777edc26 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 112defd08e14 -r 4a0c03f98ca3 .hgsigs --- a/.hgsigs Tue Nov 19 23:18:01 2013 +0200 +++ b/.hgsigs Tue Nov 19 23:18:11 2013 +0200 @@ -74,3 +74,4 @@ 71f0696749ab347119278d7f68a01f81ed584c53 0 iEYEABECAAYFAlH/+4cACgkQyUhSUUBVislEUwCfc7GeL9I5US5yJltYpWe8ZU7MAIkAnRlUGvhpFMBnehwSlUDRsssRCJ+a 1de9a494cb2520986da97fa73cf87a3781ea2d17 0 iEYEABECAAYFAlJCi2cACgkQyUhSUUBVisnoWQCfecKzvsHw/MC/5DDuZXIF/CmkaBUAn2DfADK7V0fIWZ49GPyL+BPbYCXy b5b118701aa83e7b047e53f51aa5a4622ee812df 0 iEYEABECAAYFAlJ2rGIACgkQyUhSUUBVislFRwCfTgg6BYRGpKAOAvE3hhhIskLwxLUAoJjRlth0ob0ahrqIhg7du4+Z0C3j +095a777edc2623fad2ad9d6321ed93a804e85889 0 iEYEABECAAYFAlKL1YoACgkQyUhSUUBVismrpQCeP/ab1oKZp0VJ6PwmnRgDthf2hi8AoJZB+KuQIgKA5iBx3NpGcaZjmlzc From dovecot at dovecot.org Wed Nov 20 14:39:15 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 20 Nov 2013 14:39:15 +0200 Subject: dovecot-2.2: lib-storage: mail_get_*header*() didn't remove lead... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f4eb4b5884b2 changeset: 16990:f4eb4b5884b2 user: Timo Sirainen date: Wed Nov 20 14:37:44 2013 +0200 description: lib-storage: mail_get_*header*() didn't remove leading whitespace for headers from cache. But it was removed when the headers were found by parsing the full header. diffstat: src/lib-storage/index/index-mail-headers.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 4a0c03f98ca3 -r f4eb4b5884b2 src/lib-storage/index/index-mail-headers.c --- a/src/lib-storage/index/index-mail-headers.c Tue Nov 19 23:18:11 2013 +0200 +++ b/src/lib-storage/index/index-mail-headers.c Wed Nov 20 14:37:44 2013 +0200 @@ -653,7 +653,8 @@ /* cached. skip "header name: " parts in dest. */ for (i = 0; i < len; i++) { if (data[i] == ':') { - if (i+1 != len && data[++i] == ' ') i++; + while (IS_LWSP(data[i+1])) i++; + if (i+1 != len) i++; /* @UNSAFE */ len = get_header_size(dest, i); From dovecot at dovecot.org Wed Nov 20 14:39:15 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 20 Nov 2013 14:39:15 +0200 Subject: dovecot-2.2: lib-storage: mail_get_*header*() were unnecessarily... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a91437fe94b6 changeset: 16991:a91437fe94b6 user: Timo Sirainen date: Wed Nov 20 14:38:55 2013 +0200 description: lib-storage: mail_get_*header*() were unnecessarily looking up the headers twice. diffstat: src/lib-storage/index/index-mail-headers.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (23 lines): diff -r f4eb4b5884b2 -r a91437fe94b6 src/lib-storage/index/index-mail-headers.c --- a/src/lib-storage/index/index-mail-headers.c Wed Nov 20 14:37:44 2013 +0200 +++ b/src/lib-storage/index/index-mail-headers.c Wed Nov 20 14:38:55 2013 +0200 @@ -760,6 +760,9 @@ mail_cache_set_corrupted(_mail->box->cache, "Broken header %s for mail UID %u", field, _mail->uid); + /* retry by parsing the full header */ + } else { + break; } } return ret; @@ -788,6 +791,9 @@ mail_cache_set_corrupted(_mail->box->cache, "Broken header %s for mail UID %u", field, _mail->uid); + /* retry by parsing the full header */ + } else { + break; } } *value_r = list[0]; From dovecot at dovecot.org Wed Nov 20 14:52:07 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 20 Nov 2013 14:52:07 +0200 Subject: dovecot-2.2: lib-storage: mail_get_*header*() still didn't handl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4ef184875799 changeset: 16992:4ef184875799 user: Timo Sirainen date: Wed Nov 20 14:50:53 2013 +0200 description: lib-storage: mail_get_*header*() still didn't handle cached values correctly. If header ends with ":", the ":" shouldn't be returned as part of the value. diffstat: src/lib-storage/index/index-mail-headers.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r a91437fe94b6 -r 4ef184875799 src/lib-storage/index/index-mail-headers.c --- a/src/lib-storage/index/index-mail-headers.c Wed Nov 20 14:38:55 2013 +0200 +++ b/src/lib-storage/index/index-mail-headers.c Wed Nov 20 14:50:53 2013 +0200 @@ -653,8 +653,8 @@ /* cached. skip "header name: " parts in dest. */ for (i = 0; i < len; i++) { if (data[i] == ':') { - while (IS_LWSP(data[i+1])) i++; - if (i+1 != len) i++; + i++; + while (i < len && IS_LWSP(data[i])) i++; /* @UNSAFE */ len = get_header_size(dest, i); From dovecot at dovecot.org Wed Nov 20 15:07:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 20 Nov 2013 15:07:36 +0200 Subject: dovecot-2.2: lib-storage: mail_get_headers*() returned only the ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/47923cfd4b56 changeset: 16993:47923cfd4b56 user: Timo Sirainen date: Wed Nov 20 15:07:26 2013 +0200 description: lib-storage: mail_get_headers*() returned only the first header from cache. diffstat: src/lib-storage/index/index-mail-headers.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diffs (26 lines): diff -r 4ef184875799 -r 47923cfd4b56 src/lib-storage/index/index-mail-headers.c --- a/src/lib-storage/index/index-mail-headers.c Wed Nov 20 14:50:53 2013 +0200 +++ b/src/lib-storage/index/index-mail-headers.c Wed Nov 20 15:07:26 2013 +0200 @@ -603,7 +603,7 @@ unsigned char *data; unsigned int field_idx; string_t *dest; - size_t i, len; + size_t i, len, len2; int ret; ARRAY(const char *) header_values; @@ -657,10 +657,10 @@ while (i < len && IS_LWSP(data[i])) i++; /* @UNSAFE */ - len = get_header_size(dest, i); - data[i + len] = '\0'; + len2 = get_header_size(dest, i); + data[i + len2] = '\0'; value = (const char *)data + i; - i += len + 1; + i += len2 + 1; array_append(&header_values, &value, 1); } From pigeonhole at rename-it.nl Wed Nov 20 20:34:39 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 20 Nov 2013 19:34:39 +0100 Subject: dovecot-2.2-pigeonhole: Small improvements to the test suite. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/bb3fbd27372a changeset: 1821:bb3fbd27372a user: Stephan Bosch date: Wed Nov 20 19:34:28 2013 +0100 description: Small improvements to the test suite. diffstat: tests/test-header.svtest | 18 ++++++++++++++---- 1 files changed, 14 insertions(+), 4 deletions(-) diffs (46 lines): diff -r b7434a02fc51 -r bb3fbd27372a tests/test-header.svtest --- a/tests/test-header.svtest Fri Nov 15 23:13:32 2013 +0100 +++ b/tests/test-header.svtest Wed Nov 20 19:34:28 2013 +0100 @@ -1,4 +1,5 @@ require "vnd.dovecot.testsuite"; +require "variables"; /* * ## RFC 5228, Section 5.7. Test header (page 29) ## @@ -145,7 +146,7 @@ To: nico at frop.example.com Subject: Help X-A: Text -X-B: Text +X-B: Text Text . @@ -153,15 +154,24 @@ test "Ignoring whitespace" { if not header :is "x-a" "Text" { - test_fail "header test does not strip leading whitespace"; + if header :matches "x-a" "*" { + set "header" "${1}"; + } + test_fail "header test does not strip leading whitespace (header=`${header}`)"; } if not header :is "x-b" "Text" { - test_fail "header test does not strip trailing whitespace"; + if header :matches "x-b" "*" { + set "header" "${1}"; + } + test_fail "header test does not strip trailing whitespace (header=`${header}`)"; } if not header :is "subject" "Help" { - test_fail "header test does not strip both leading and trailing whitespace"; + if header :matches "subject" "*" { + set "header" "${1}"; + } + test_fail "header test does not strip both leading and trailing whitespace (header=`${header}`)"; } } From dovecot at dovecot.org Wed Nov 20 21:25:33 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 20 Nov 2013 21:25:33 +0200 Subject: dovecot-2.2: dsync: have_only_guid128 was never actually being set. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fbf77af0233c changeset: 16994:fbf77af0233c user: Timo Sirainen date: Wed Nov 20 21:25:19 2013 +0200 description: dsync: have_only_guid128 was never actually being set. diffstat: src/doveadm/dsync/dsync-brain-mailbox.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 47923cfd4b56 -r fbf77af0233c src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Wed Nov 20 15:07:26 2013 +0200 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Wed Nov 20 21:25:19 2013 +0200 @@ -317,6 +317,7 @@ dsync_box_r->cache_fields = *metadata.cache_fields; dsync_box_r->have_guids = status.have_guids; dsync_box_r->have_save_guids = status.have_save_guids; + dsync_box_r->have_only_guid128 = status.have_only_guid128; return 1; } From dovecot at dovecot.org Wed Nov 20 22:20:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 20 Nov 2013 22:20:34 +0200 Subject: dovecot-2.2: dsync: Fix to 128bit GUID syncing Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/df2b27885c9d changeset: 16995:df2b27885c9d user: Timo Sirainen date: Wed Nov 20 22:20:06 2013 +0200 description: dsync: Fix to 128bit GUID syncing diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 11 ++++++----- 1 files changed, 6 insertions(+), 5 deletions(-) diffs (26 lines): diff -r fbf77af0233c -r df2b27885c9d src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Wed Nov 20 21:25:19 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Wed Nov 20 22:20:06 2013 +0200 @@ -472,16 +472,17 @@ { guid_128_t guid_128, change_guid_128; - if (change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE && - !importer->mails_use_guid128) { + if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + if (guid_128_from_string(change->guid, change_guid_128) < 0) + i_unreached(); + } else if (importer->mails_use_guid128) { + mail_generate_guid_128_hash(change->guid, change_guid_128); + } else { if (cmp_guid_r != NULL) *cmp_guid_r = change->guid; return strcmp(change->guid, guid) == 0; } - if (guid_128_from_string(change->guid, change_guid_128) < 0) - i_unreached(); - mail_generate_guid_128_hash(guid, guid_128); if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) { if (cmp_guid_r != NULL) { From dovecot at dovecot.org Wed Nov 20 23:29:26 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 20 Nov 2013 23:29:26 +0200 Subject: dovecot-2.2: doveadm: Added "fs metadata" command Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d2e95af17305 changeset: 16996:d2e95af17305 user: Timo Sirainen date: Wed Nov 20 23:29:13 2013 +0200 description: doveadm: Added "fs metadata" command diffstat: src/doveadm/doveadm-fs.c | 26 ++++++++++++++++++++++++++ 1 files changed, 26 insertions(+), 0 deletions(-) diffs (43 lines): diff -r df2b27885c9d -r d2e95af17305 src/doveadm/doveadm-fs.c --- a/src/doveadm/doveadm-fs.c Wed Nov 20 22:20:06 2013 +0200 +++ b/src/doveadm/doveadm-fs.c Wed Nov 20 23:29:13 2013 +0200 @@ -156,6 +156,31 @@ fs_deinit(&fs); } +static void cmd_fs_metadata(int argc, char *argv[]) +{ + struct fs *fs; + struct fs_file *file; + const struct fs_metadata *m; + const ARRAY_TYPE(fs_metadata) *metadata; + + fs = cmd_fs_init(&argc, &argv, 1, cmd_fs_metadata); + + file = fs_file_init(fs, argv[0], FS_OPEN_MODE_READONLY); + if (fs_get_metadata(file, &metadata) == 0) { + array_foreach(metadata, m) + printf("%s=%s\n", m->key, m->value); + } else if (errno == ENOENT) { + i_error("%s doesn't exist", fs_file_path(file)); + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + } else { + i_error("fs_stat(%s) failed: %s", + fs_file_path(file), fs_file_last_error(file)); + doveadm_exit_code = EX_TEMPFAIL; + } + fs_file_deinit(&file); + fs_deinit(&fs); +} + static void cmd_fs_delete_dir_recursive(struct fs *fs, const char *path) { struct fs_iter *iter; @@ -271,6 +296,7 @@ { cmd_fs_put, "fs put", " " }, { cmd_fs_copy, "fs copy", " " }, { cmd_fs_stat, "fs stat", " " }, + { cmd_fs_metadata, "fs metadata", " " }, { cmd_fs_delete, "fs delete", "[-R] " }, { cmd_fs_iter, "fs iter", " " }, { cmd_fs_iter_dirs, "fs iter-dirs", " " }, From dovecot at dovecot.org Thu Nov 21 23:02:46 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 21 Nov 2013 23:02:46 +0200 Subject: dovecot-2.2: lib-storage: Improved EACCES error messages for som... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/af96719b5d9c changeset: 16997:af96719b5d9c user: Timo Sirainen date: Thu Nov 21 23:02:33 2013 +0200 description: lib-storage: Improved EACCES error messages for some stat() calls diffstat: src/lib-storage/mailbox-list.c | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diffs (26 lines): diff -r d2e95af17305 -r af96719b5d9c src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Wed Nov 20 23:29:13 2013 +0200 +++ b/src/lib-storage/mailbox-list.c Thu Nov 21 23:02:33 2013 +0200 @@ -868,7 +868,10 @@ if (path == NULL) { /* no filesystem support in storage */ } else if (stat(path, &st) < 0) { - if (!ENOTFOUND(errno)) { + if (errno == EACCES) { + mailbox_list_set_critical(list, "%s", + mail_error_eacces_msg("stat", path)); + } else if (!ENOTFOUND(errno)) { mailbox_list_set_critical(list, "stat(%s) failed: %m", path); } else if (list->mail_set->mail_debug) { @@ -1052,7 +1055,9 @@ /* get the first existing parent directory's permissions */ if (stat_first_parent(expanded, &root_dir, &st) < 0) { - *error_r = t_strdup_printf("stat(%s) failed: %m", root_dir); + *error_r = errno == EACCES ? + mail_error_eacces_msg("stat", root_dir) : + t_strdup_printf("stat(%s) failed: %m", root_dir); return -1; } From dovecot at dovecot.org Fri Nov 22 13:47:53 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 Nov 2013 13:47:53 +0200 Subject: dovecot-2.2: ostream: Don't mark the stream closed too early aft... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e3a87c25e0b2 changeset: 16998:e3a87c25e0b2 user: Timo Sirainen date: Fri Nov 22 13:47:36 2013 +0200 description: ostream: Don't mark the stream closed too early after all. Use another flag instead to avoid losing the last data written to the stream before closing. diffstat: src/lib/ostream-private.h | 1 + src/lib/ostream.c | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diffs (31 lines): diff -r af96719b5d9c -r e3a87c25e0b2 src/lib/ostream-private.h --- a/src/lib/ostream-private.h Thu Nov 21 23:02:33 2013 +0200 +++ b/src/lib/ostream-private.h Fri Nov 22 13:47:36 2013 +0200 @@ -37,6 +37,7 @@ void *context; unsigned int corked:1; + unsigned int closing:1; unsigned int last_errors_not_checked:1; unsigned int error_handling_disabled:1; }; diff -r af96719b5d9c -r e3a87c25e0b2 src/lib/ostream.c --- a/src/lib/ostream.c Thu Nov 21 23:02:33 2013 +0200 +++ b/src/lib/ostream.c Fri Nov 22 13:47:36 2013 +0200 @@ -45,9 +45,14 @@ static void o_stream_close_full(struct ostream *stream, bool close_parents) { - if (!stream->closed) { + if (!stream->closed && !stream->real_stream->closing) { + /* first mark the stream as being closed so the + o_stream_copy_error_from_parent() won't recurse us back + here. but don't immediately mark the stream closed, because + we may still want to write something to it. */ + stream->real_stream->closing = TRUE; + io_stream_close(&stream->real_stream->iostream, close_parents); stream->closed = TRUE; - io_stream_close(&stream->real_stream->iostream, close_parents); } if (stream->stream_errno == 0) From dovecot at dovecot.org Fri Nov 22 19:32:19 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 Nov 2013 19:32:19 +0200 Subject: dovecot-2.2: doveadm acl debug: Improved output Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/157b428d4ea4 changeset: 16999:157b428d4ea4 user: Timo Sirainen date: Fri Nov 22 19:32:06 2013 +0200 description: doveadm acl debug: Improved output diffstat: src/plugins/acl/doveadm-acl.c | 11 ++++++++--- 1 files changed, 8 insertions(+), 3 deletions(-) diffs (35 lines): diff -r e3a87c25e0b2 -r 157b428d4ea4 src/plugins/acl/doveadm-acl.c --- a/src/plugins/acl/doveadm-acl.c Fri Nov 22 13:47:36 2013 +0200 +++ b/src/plugins/acl/doveadm-acl.c Fri Nov 22 19:32:06 2013 +0200 @@ -372,8 +372,8 @@ &path) <= 0) i_error("Can't open mailbox %s: %s", mailbox, errstr); else { - i_error("Mailbox '%s' doesn't exist in %s", - mailbox, path); + i_error("Mailbox '%s' in namespace '%s' doesn't exist in %s", + box->name, ns->prefix, path); } mailbox_free(&box); return -1; @@ -399,7 +399,7 @@ struct acl_backend *backend = acl_mailbox_list_get_backend(box->list); struct acl_mailbox_list_context *iter; struct acl_lookup_dict_iter *diter; - const char *const *rights, *name; + const char *const *rights, *name, *path; enum mail_flags private_flags_mask; string_t *str; int ret; @@ -407,6 +407,11 @@ *retry_r = FALSE; + i_info("Mailbox '%s' is in namespace '%s'", + box->name, box->list->ns->prefix); + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, &path) > 0) + i_info("Mailbox path: %s", path); + private_flags_mask = mailbox_get_private_flags_mask(box); if (private_flags_mask == 0) i_info("All message flags are shared across users in mailbox"); From dovecot at dovecot.org Fri Nov 22 19:39:28 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 Nov 2013 19:39:28 +0200 Subject: dovecot-2.2: Added mail-filter plugin. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/84ccfa053bcd changeset: 17000:84ccfa053bcd user: Timo Sirainen date: Fri Nov 22 19:39:13 2013 +0200 description: Added mail-filter plugin. diffstat: configure.ac | 1 + src/plugins/Makefile.am | 1 + src/plugins/mail-filter/Makefile.am | 20 ++ src/plugins/mail-filter/istream-ext-filter.c | 197 ++++++++++++++++++++++++ src/plugins/mail-filter/istream-ext-filter.h | 8 + src/plugins/mail-filter/mail-filter-plugin.c | 213 +++++++++++++++++++++++++++ src/plugins/mail-filter/mail-filter-plugin.h | 7 + src/plugins/mail-filter/ostream-ext-filter.c | 162 ++++++++++++++++++++ src/plugins/mail-filter/ostream-ext-filter.h | 8 + 9 files changed, 617 insertions(+), 0 deletions(-) diffs (truncated from 665 to 300 lines): diff -r 157b428d4ea4 -r 84ccfa053bcd configure.ac --- a/configure.ac Fri Nov 22 19:32:06 2013 +0200 +++ b/configure.ac Fri Nov 22 19:39:13 2013 +0200 @@ -2884,6 +2884,7 @@ src/plugins/fts-squat/Makefile src/plugins/lazy-expunge/Makefile src/plugins/listescape/Makefile +src/plugins/mail-filter/Makefile src/plugins/mail-log/Makefile src/plugins/mailbox-alias/Makefile src/plugins/notify/Makefile diff -r 157b428d4ea4 -r 84ccfa053bcd src/plugins/Makefile.am --- a/src/plugins/Makefile.am Fri Nov 22 19:32:06 2013 +0200 +++ b/src/plugins/Makefile.am Fri Nov 22 19:39:13 2013 +0200 @@ -20,6 +20,7 @@ lazy-expunge \ listescape \ notify \ + mail-filter \ mail-log \ mailbox-alias \ quota \ diff -r 157b428d4ea4 -r 84ccfa053bcd src/plugins/mail-filter/Makefile.am --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/mail-filter/Makefile.am Fri Nov 22 19:39:13 2013 +0200 @@ -0,0 +1,20 @@ +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage + +NOPLUGIN_LDFLAGS = +lib10_mail_filter_plugin_la_LDFLAGS = -module -avoid-version + +module_LTLIBRARIES = \ + lib10_mail_filter_plugin.la + +lib10_mail_filter_plugin_la_SOURCES = \ + mail-filter-plugin.c \ + istream-ext-filter.c \ + ostream-ext-filter.c + +noinst_HEADERS = \ + mail-filter-plugin.h diff -r 157b428d4ea4 -r 84ccfa053bcd src/plugins/mail-filter/istream-ext-filter.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/mail-filter/istream-ext-filter.c Fri Nov 22 19:39:13 2013 +0200 @@ -0,0 +1,197 @@ +/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "net.h" +#include "eacces-error.h" +#include "fd-set-nonblock.h" +#include "ostream.h" +#include "istream-private.h" +#include "istream-ext-filter.h" + +#include + +struct mail_filter_istream { + struct istream_private istream; + + int fd; + struct istream *ext_in; + struct ostream *ext_out; + size_t prev_ret; +}; + +static void +i_stream_mail_filter_close(struct iostream_private *stream, bool close_parent) +{ + struct mail_filter_istream *mstream = + (struct mail_filter_istream *)stream; + + if (mstream->ext_in != NULL) + i_stream_destroy(&mstream->ext_in); + if (mstream->ext_out != NULL) + o_stream_destroy(&mstream->ext_out); + if (mstream->fd != -1) { + if (close(mstream->fd) < 0) + i_error("ext-filter: close() failed: %m"); + mstream->fd = -1; + } + if (close_parent) + i_stream_close(mstream->istream.parent); +} + +static ssize_t +i_stream_read_copy_from(struct istream *istream, struct istream *source) +{ + struct istream_private *stream = istream->real_stream; + size_t pos; + ssize_t ret; + + stream->pos -= stream->skip; + stream->skip = 0; + + stream->buffer = i_stream_get_data(source, &pos); + if (pos > stream->pos) + ret = 0; + else do { + if ((ret = i_stream_read(source)) == -2) + return -2; + + stream->istream.stream_errno = source->stream_errno; + stream->istream.eof = source->eof; + stream->buffer = i_stream_get_data(source, &pos); + /* check again, in case the source stream had been seeked + backwards and the previous read() didn't get us far + enough. */ + } while (pos <= stream->pos && ret > 0); + + ret = pos > stream->pos ? (ssize_t)(pos - stream->pos) : + (ret == 0 ? 0 : -1); + stream->pos = pos; + i_assert(ret != -1 || stream->istream.eof || + stream->istream.stream_errno != 0); + return ret; +} + +static ssize_t +i_stream_mail_filter_read_once(struct mail_filter_istream *mstream) +{ + struct istream_private *stream = &mstream->istream; + ssize_t ret; + + if (mstream->ext_out != NULL) { + /* we haven't sent everything yet */ + (void)o_stream_send_istream(mstream->ext_out, stream->parent); + if (mstream->ext_out->stream_errno != 0) { + stream->istream.stream_errno = + mstream->ext_out->stream_errno; + return -1; + } + if (i_stream_is_eof(stream->parent)) { + o_stream_destroy(&mstream->ext_out); + /* if we wanted to be a blocking stream, + from now on the rest of the reads are */ + if (stream->istream.blocking) + net_set_nonblock(mstream->fd, FALSE); + if (shutdown(mstream->fd, SHUT_WR) < 0) + i_error("ext-filter: shutdown() failed: %m"); + } + } + + i_stream_skip(mstream->ext_in, mstream->prev_ret); + ret = i_stream_read_copy_from(&stream->istream, mstream->ext_in); + mstream->prev_ret = ret < 0 ? 0 : ret; + return ret; +} + +static ssize_t i_stream_mail_filter_read(struct istream_private *stream) +{ + struct mail_filter_istream *mstream = + (struct mail_filter_istream *)stream; + ssize_t ret; + + if (mstream->ext_in == NULL) { + stream->istream.stream_errno = EIO; + return -1; + } + + while ((ret = i_stream_mail_filter_read_once(mstream)) == 0) { + if (!stream->istream.blocking) + break; + } + return ret; +} + +static int +i_stream_mail_filter_stat(struct istream_private *stream, bool exact) +{ + const struct stat *st; + + i_assert(!exact); + + if (i_stream_stat(stream->parent, exact, &st) < 0) + return -1; + stream->statbuf = *st; + return 0; +} + +static int filter_connect(struct mail_filter_istream *mstream, + const char *socket_path, const char *args) +{ + const char **argv; + string_t *str; + int fd; + + argv = t_strsplit(args, " "); + + if ((fd = net_connect_unix_with_retries(socket_path, 1000)) < 0) { + if (errno == EACCES) { + i_error("ext-filter: %s", + eacces_error_get("net_connect_unix", + socket_path)); + } else { + i_error("ext-filter: net_connect_unix(%s) failed: %m", + socket_path); + } + return -1; + } + if (mstream->istream.istream.blocking) + net_set_nonblock(fd, FALSE); + + mstream->fd = fd; + mstream->ext_in = + i_stream_create_fd(fd, mstream->istream.max_buffer_size, FALSE); + mstream->ext_out = o_stream_create_fd(fd, 0, FALSE); + + str = t_str_new(256); + str_append(str, "VERSION\tscript\t3\t0\nnoreply\n"); + for (; *argv != NULL; argv++) { + str_append(str, *argv); + str_append_c(str, '\n'); + } + str_append_c(str, '\n'); + + o_stream_send(mstream->ext_out, str_data(str), str_len(str)); + return 0; +} + +struct istream * +i_stream_create_ext_filter(struct istream *input, const char *socket_path, + const char *args) +{ + struct mail_filter_istream *mstream; + + mstream = i_new(struct mail_filter_istream, 1); + mstream->istream.iostream.close = i_stream_mail_filter_close; + mstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + mstream->istream.read = i_stream_mail_filter_read; + mstream->istream.stat = i_stream_mail_filter_stat; + + mstream->istream.istream.readable_fd = FALSE; + mstream->istream.istream.blocking = input->blocking; + mstream->istream.istream.seekable = FALSE; + + mstream->fd = -1; + (void)filter_connect(mstream, socket_path, args); + + return i_stream_create(&mstream->istream, input, mstream->fd); +} diff -r 157b428d4ea4 -r 84ccfa053bcd src/plugins/mail-filter/istream-ext-filter.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/mail-filter/istream-ext-filter.h Fri Nov 22 19:39:13 2013 +0200 @@ -0,0 +1,8 @@ +#ifndef ISTREAM_MAIL_FILTER_H +#define ISTREAM_MAIL_FILTER_H + +struct istream * +i_stream_create_ext_filter(struct istream *input, const char *socket_path, + const char *args); + +#endif diff -r 157b428d4ea4 -r 84ccfa053bcd src/plugins/mail-filter/mail-filter-plugin.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/mail-filter/mail-filter-plugin.c Fri Nov 22 19:39:13 2013 +0200 @@ -0,0 +1,213 @@ +/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "safe-mkstemp.h" +#include "mail-user.h" +#include "mail-storage-private.h" +#include "istream.h" +#include "istream-seekable.h" +#include "istream-ext-filter.h" +#include "ostream-ext-filter.h" +#include "mail-filter-plugin.h" + +/* After buffer grows larger than this, create a temporary file to /tmp + where to read the mail. */ +#define MAIL_MAX_MEMORY_BUFFER (1024*128) + +#define MAIL_FILTER_MAIL_CONTEXT(obj) \ + MODULE_CONTEXT(obj, mail_filter_mail_module) +#define MAIL_FILTER_CONTEXT(obj) \ + MODULE_CONTEXT(obj, mail_filter_storage_module) +#define MAIL_FILTER_USER_CONTEXT(obj) \ + MODULE_CONTEXT(obj, mail_filter_user_module) + +struct mail_filter_user { + union mail_user_module_context module_ctx; + + const char *socket_path, *args; + const char *out_socket_path, *out_args; +}; + +const char *mail_filter_plugin_version = DOVECOT_ABI_VERSION; + +static MODULE_CONTEXT_DEFINE_INIT(mail_filter_user_module, + &mail_user_module_register); +static MODULE_CONTEXT_DEFINE_INIT(mail_filter_storage_module, From dovecot at dovecot.org Fri Nov 22 22:06:11 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 Nov 2013 22:06:11 +0200 Subject: dovecot-2.2: iostream-rawlog: Timestamp prefix is now optional. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cf38c8eb8493 changeset: 17001:cf38c8eb8493 user: Timo Sirainen date: Fri Nov 22 22:05:52 2013 +0200 description: iostream-rawlog: Timestamp prefix is now optional. diffstat: src/lib/iostream-rawlog-private.h | 3 ++- src/lib/iostream-rawlog.c | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diffs (57 lines): diff -r 84ccfa053bcd -r cf38c8eb8493 src/lib/iostream-rawlog-private.h --- a/src/lib/iostream-rawlog-private.h Fri Nov 22 19:39:13 2013 +0200 +++ b/src/lib/iostream-rawlog-private.h Fri Nov 22 22:05:52 2013 +0200 @@ -5,7 +5,8 @@ enum iostream_rawlog_flags { IOSTREAM_RAWLOG_FLAG_AUTOCLOSE = 0x01, - IOSTREAM_RAWLOG_FLAG_BUFFERED = 0x02 + IOSTREAM_RAWLOG_FLAG_BUFFERED = 0x02, + IOSTREAM_RAWLOG_FLAG_TIMESTAMP = 0x04 }; struct rawlog_iostream { diff -r 84ccfa053bcd -r cf38c8eb8493 src/lib/iostream-rawlog.c --- a/src/lib/iostream-rawlog.c Fri Nov 22 19:39:13 2013 +0200 +++ b/src/lib/iostream-rawlog.c Fri Nov 22 22:05:52 2013 +0200 @@ -41,6 +41,9 @@ unsigned char data[MAX_INT_STRLEN + 6 + 1 + 3]; buffer_t buf; + if ((rstream->flags & IOSTREAM_RAWLOG_FLAG_TIMESTAMP) == 0) + return 0; + buffer_create_from_data(&buf, data, sizeof(data)); str_printfa(&buf, "%lu.%06u ", (unsigned long)ioloop_timeval.tv_sec, @@ -187,10 +190,13 @@ old_input = *input; old_output = *output; + *input = i_stream_create_rawlog(old_input, in_path, in_fd, - IOSTREAM_RAWLOG_FLAG_AUTOCLOSE); + IOSTREAM_RAWLOG_FLAG_AUTOCLOSE | + IOSTREAM_RAWLOG_FLAG_TIMESTAMP); *output = o_stream_create_rawlog(old_output, out_path, out_fd, - IOSTREAM_RAWLOG_FLAG_AUTOCLOSE); + IOSTREAM_RAWLOG_FLAG_AUTOCLOSE | + IOSTREAM_RAWLOG_FLAG_TIMESTAMP); i_stream_unref(&old_input); o_stream_unref(&old_output); return 0; @@ -212,10 +218,12 @@ old_input = *input; old_output = *output; *input = i_stream_create_rawlog(old_input, path, fd, - IOSTREAM_RAWLOG_FLAG_BUFFERED); + IOSTREAM_RAWLOG_FLAG_BUFFERED | + IOSTREAM_RAWLOG_FLAG_TIMESTAMP); *output = o_stream_create_rawlog(old_output, path, fd, IOSTREAM_RAWLOG_FLAG_AUTOCLOSE | - IOSTREAM_RAWLOG_FLAG_BUFFERED); + IOSTREAM_RAWLOG_FLAG_BUFFERED | + IOSTREAM_RAWLOG_FLAG_TIMESTAMP); i_stream_unref(&old_input); o_stream_unref(&old_output); return 0; From dovecot at dovecot.org Fri Nov 22 22:13:13 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 Nov 2013 22:13:13 +0200 Subject: dovecot-2.2: http-client: Changed struct http_client_host_port i... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/160e489d7c12 changeset: 17002:160e489d7c12 user: Stephan Bosch date: Fri Nov 22 22:07:41 2013 +0200 description: http-client: Changed struct http_client_host_port into a struct http_client_queue object. Peer and request objects now reference the queue object directly rather than the host object. This way, there is no need to find the matching host:port in the host anymore. This makes the queueing structure more intuitive and more efficient. This is a first step towards support for connecting to HTTP services through unix sockets or directing requests at specific hosts (so not from the URL). This patch also fixes a potential timeout leak (to_connect) in http_client_host_port (now http_client_queue) and makes sure it is moved during switch_ioloop(). Finally it updates the structure comment at the top of http-client.c. diffstat: src/lib-http/Makefile.am | 1 + src/lib-http/http-client-host.c | 422 ++---------------------------------- src/lib-http/http-client-peer.c | 63 ++-- src/lib-http/http-client-private.h | 354 ++++++++++++++++-------------- src/lib-http/http-client-queue.c | 375 ++++++++++++++++++++++++++++++++ src/lib-http/http-client-request.c | 7 +- src/lib-http/http-client.c | 57 +++- 7 files changed, 668 insertions(+), 611 deletions(-) diffs (truncated from 1540 to 300 lines): diff -r cf38c8eb8493 -r 160e489d7c12 src/lib-http/Makefile.am --- a/src/lib-http/Makefile.am Fri Nov 22 22:05:52 2013 +0200 +++ b/src/lib-http/Makefile.am Fri Nov 22 22:07:41 2013 +0200 @@ -21,6 +21,7 @@ http-client-request.c \ http-client-connection.c \ http-client-peer.c \ + http-client-queue.c \ http-client-host.c \ http-client.c diff -r cf38c8eb8493 -r 160e489d7c12 src/lib-http/http-client-host.c --- a/src/lib-http/http-client-host.c Fri Nov 22 22:05:52 2013 +0200 +++ b/src/lib-http/http-client-host.c Fri Nov 22 22:07:41 2013 +0200 @@ -38,312 +38,19 @@ } /* - * Host:port + * Host */ static void -http_client_host_port_connection_setup(struct http_client_host_port *hport); - -static struct http_client_host_port * -http_client_host_port_find(struct http_client_host *host, - const struct http_client_peer_addr *addr) +http_client_host_lookup_failure(struct http_client_host *host, + const char *error) { - struct http_client_host_port *hport; - - array_foreach_modifiable(&host->ports, hport) { - if (hport->addr.type == addr->type && hport->addr.port == addr->port && - null_strcmp(hport->addr.https_name, addr->https_name) == 0) - return hport; - } - - return NULL; -} - -static struct http_client_host_port * -http_client_host_port_init(struct http_client_host *host, - const struct http_client_peer_addr *addr) -{ - struct http_client_host_port *hport; - - hport = http_client_host_port_find(host, addr); - if (hport == NULL) { - hport = array_append_space(&host->ports); - hport->host = host; - hport->addr = *addr; - hport->https_name = i_strdup(addr->https_name); - hport->addr.https_name = hport->https_name; - hport->ips_connect_idx = 0; - i_array_init(&hport->request_queue, 16); - } - - return hport; -} - -static void http_client_host_port_error(struct http_client_host_port *hport, - unsigned int status, const char *error) -{ - struct http_client_request **req; - - /* abort all pending requests */ - array_foreach_modifiable(&hport->request_queue, req) { - http_client_request_error(*req, status, error); - } - array_clear(&hport->request_queue); -} - -static void http_client_host_port_deinit(struct http_client_host_port *hport) -{ - http_client_host_port_error - (hport, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Aborted"); - i_free(hport->https_name); - if (array_is_created(&hport->pending_peers)) - array_free(&hport->pending_peers); - array_free(&hport->request_queue); -} - -static void -http_client_host_port_drop_request(struct http_client_host_port *hport, - struct http_client_request *req) -{ - ARRAY_TYPE(http_client_request) *req_arr = &hport->request_queue; - struct http_client_request **req_idx; - - array_foreach_modifiable(req_arr, req_idx) { - if (*req_idx == req) { - array_delete(req_arr, array_foreach_idx(req_arr, req_idx), 1); - break; - } - } -} - -static bool -http_client_hport_is_last_connect_ip(struct http_client_host_port *hport) -{ - i_assert(hport->ips_connect_idx < hport->host->ips_count); - i_assert(hport->ips_connect_start_idx < hport->host->ips_count); - - /* we'll always go through all the IPs. we don't necessarily start - connecting from the first IP, so we'll need to treat the IPs as - a ring buffer where we automatically wrap back to the first IP - when necessary. */ - return (hport->ips_connect_idx + 1) % hport->host->ips_count == - hport->ips_connect_start_idx; -} - -static void -http_client_host_port_soft_connect_timeout(struct http_client_host_port *hport) -{ - struct http_client_host *host = hport->host; - const struct http_client_peer_addr *addr = &hport->addr; - - if (hport->to_connect != NULL) - timeout_remove(&hport->to_connect); - - if (http_client_hport_is_last_connect_ip(hport)) { - /* no more IPs to try */ - return; - } - - /* if our our previous connection attempt takes longer than the - soft_connect_timeout, we start a connection attempt to the next IP in - parallel */ - http_client_host_debug(host, "Connection to %s%s is taking a long time; " - "starting parallel connection attempt to next IP", - http_client_peer_addr2str(addr), addr->https_name == NULL ? "" : - t_strdup_printf(" (SSL=%s)", addr->https_name)); - - /* next IP */ - hport->ips_connect_idx = (hport->ips_connect_idx + 1) % host->ips_count; - - /* setup connection to new peer (can start new soft timeout) */ - http_client_host_port_connection_setup(hport); -} - -static void -http_client_host_port_connection_setup(struct http_client_host_port *hport) -{ - struct http_client_host *host = hport->host; - struct http_client_peer *peer = NULL; - const struct http_client_peer_addr *addr = &hport->addr; - unsigned int num_requests = array_count(&hport->request_queue); - - if (num_requests == 0) - return; - - /* update our peer address */ - hport->addr.ip = host->ips[hport->ips_connect_idx]; - - http_client_host_debug(host, "Setting up connection to %s%s " - "(%u requests pending)", http_client_peer_addr2str(addr), - (addr->https_name == NULL ? "" : - t_strdup_printf(" (SSL=%s)", addr->https_name)), num_requests); - - /* create/get peer */ - peer = http_client_peer_get(host->client, addr); - http_client_peer_add_host(peer, host); - - /* handle requests; creates new connections when needed/possible */ - http_client_peer_trigger_request_handler(peer); - - if (!http_client_peer_is_connected(peer)) { - unsigned int msecs; - - /* not already connected, wait for connections */ - if (!array_is_created(&hport->pending_peers)) - i_array_init(&hport->pending_peers, 8); - array_append(&hport->pending_peers, &peer, 1); - - /* start soft connect time-out (but only if we have another IP left) */ - msecs = host->client->set.soft_connect_timeout_msecs; - if (!http_client_hport_is_last_connect_ip(hport) && msecs > 0 && - hport->to_connect == NULL) { - hport->to_connect = - timeout_add(msecs, http_client_host_port_soft_connect_timeout, hport); - } - } -} - -static unsigned int -http_client_host_get_ip_idx(struct http_client_host *host, - const struct ip_addr *ip) -{ - unsigned int i; - - for (i = 0; i < host->ips_count; i++) { - if (net_ip_compare(&host->ips[i], ip)) - return i; - } - i_unreached(); -} - -static void -http_client_host_port_connection_success(struct http_client_host_port *hport, - const struct http_client_peer_addr *addr) -{ - /* we achieved at least one connection the the addr->ip */ - hport->ips_connect_start_idx = - http_client_host_get_ip_idx(hport->host, &addr->ip); - - /* stop soft connect time-out */ - if (hport->to_connect != NULL) - timeout_remove(&hport->to_connect); - - /* drop all other attempts to the hport. note that we get here whenever - a connection is successfully created, so pending_peers array - may be empty. */ - if (array_is_created(&hport->pending_peers) && - array_count(&hport->pending_peers) > 0) { - struct http_client_peer *const *peer_idx; - - array_foreach(&hport->pending_peers, peer_idx) { - if (http_client_peer_addr_cmp(&(*peer_idx)->addr, addr) == 0) { - /* don't drop any connections to the successfully - connected peer, even if some of the connections - are pending. they may be intended for urgent - requests. */ - continue; - } - /* remove this host from the peer; if this was the last/only host, the - peer will be freed, closing all connections. - */ - http_client_peer_remove_host(*peer_idx, hport->host); - } - array_clear(&hport->pending_peers); - } -} - -static bool -http_client_host_port_connection_failure(struct http_client_host_port *hport, - const struct http_client_peer_addr *addr, const char *reason) -{ - struct http_client_host *host = hport->host; - - if (array_is_created(&hport->pending_peers) && - array_count(&hport->pending_peers) > 0) { - struct http_client_peer *const *peer_idx; - - /* we're still doing the initial connections to this hport. if - we're also doing parallel connections with soft timeouts - (pending_peer_count>1), wait for them to finish - first. */ - array_foreach(&hport->pending_peers, peer_idx) { - if (http_client_peer_addr_cmp(&(*peer_idx)->addr, addr) == 0) { - array_delete(&hport->pending_peers, - array_foreach_idx(&hport->pending_peers, peer_idx), 1); - break; - } - } - if (array_count(&hport->pending_peers) > 0) - return TRUE; - } - - /* one of the connections failed. if we're not using soft timeouts, - we need to try to connect to the next IP. if we are using soft - timeouts, we've already tried all of the IPs by now. */ - if (hport->to_connect != NULL) - timeout_remove(&hport->to_connect); - - if (http_client_hport_is_last_connect_ip(hport)) { - /* all IPs failed, but retry all of them again on the - next request. */ - hport->ips_connect_idx = hport->ips_connect_start_idx = - (hport->ips_connect_idx + 1) % host->ips_count; - http_client_host_port_error(hport, - HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, reason); - return FALSE; - } - hport->ips_connect_idx = (hport->ips_connect_idx + 1) % host->ips_count; - http_client_host_port_connection_setup(hport); - return TRUE; -} - -/* - * Host - */ - -void http_client_host_connection_success(struct http_client_host *host, - const struct http_client_peer_addr *addr) -{ - struct http_client_host_port *hport; - - http_client_host_debug(host, "Successfully connected to %s", - http_client_peer_addr2str(addr)); - - hport = http_client_host_port_find(host, addr); - if (hport == NULL) - return; - - http_client_host_port_connection_success(hport, addr); -} - -void http_client_host_connection_failure(struct http_client_host *host, From dovecot at dovecot.org Fri Nov 22 22:13:13 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 Nov 2013 22:13:13 +0200 Subject: dovecot-2.2: http-client: Improved handing of delayed request er... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6b48a5675512 changeset: 17003:6b48a5675512 user: Stephan Bosch date: Fri Nov 22 22:08:20 2013 +0200 description: http-client: Improved handing of delayed request errors. Originally each request would have its own zero timeout, but now only one timeout is put in the host object. diffstat: src/lib-http/http-client-host.c | 59 ++++++++++++++++++++++++++++++++----- src/lib-http/http-client-private.h | 7 +++- src/lib-http/http-client-request.c | 33 ++++---------------- 3 files changed, 64 insertions(+), 35 deletions(-) diffs (199 lines): diff -r 160e489d7c12 -r 6b48a5675512 src/lib-http/http-client-host.c --- a/src/lib-http/http-client-host.c Fri Nov 22 22:07:41 2013 +0200 +++ b/src/lib-http/http-client-host.c Fri Nov 22 22:08:20 2013 +0200 @@ -205,7 +205,7 @@ { struct http_client_host *host = *_host; struct http_client_queue *const *queue_idx; - struct http_client_request *req, *const *reqp; + struct http_client_request *req, *const *req_idx; const char *hostname = host->name; http_client_host_debug(host, "Host destroy"); @@ -223,30 +223,73 @@ array_free(&host->queues); while (array_count(&host->delayed_failing_requests) > 0) { - reqp = array_idx(&host->delayed_failing_requests, 0); - req = *reqp; + req_idx = array_idx(&host->delayed_failing_requests, 0); + req = *req_idx; i_assert(req->refcount == 1); - http_client_request_unref(&req); + http_client_request_error_delayed(&req); } array_free(&host->delayed_failing_requests); + if (host->to_failing_requests != NULL) + timeout_remove(&host->to_failing_requests); + i_free(host->ips); i_free(host->name); i_free(host); } +static void +http_client_host_handle_request_errors(struct http_client_host *host) +{ + timeout_remove(&host->to_failing_requests); + + while (array_count(&host->delayed_failing_requests) > 0) { + struct http_client_request *const *req_idx = + array_idx(&host->delayed_failing_requests, 0); + struct http_client_request *req = *req_idx; + + i_assert(req->refcount == 1); + http_client_request_error_delayed(&req); + } + array_clear(&host->delayed_failing_requests); +} + +void http_client_host_delay_request_error(struct http_client_host *host, + struct http_client_request *req) +{ + if (host->to_failing_requests == NULL) { + host->to_failing_requests = timeout_add_short(0, + http_client_host_handle_request_errors, host); + } + array_append(&host->delayed_failing_requests, &req, 1); +} + +void http_client_host_remove_request_error(struct http_client_host *host, + struct http_client_request *req) +{ + struct http_client_request *const *reqs; + unsigned int i, count; + + reqs = array_get(&host->delayed_failing_requests, &count); + for (i = 0; i < count; i++) { + if (reqs[i] == req) { + array_delete(&host->delayed_failing_requests, i, 1); + return; + } + } +} + void http_client_host_switch_ioloop(struct http_client_host *host) { struct http_client_queue *const *queue_idx; - struct http_client_request *const *req_idx; if (host->dns_lookup != NULL) dns_lookup_switch_ioloop(host->dns_lookup); array_foreach(&host->queues, queue_idx) http_client_queue_switch_ioloop(*queue_idx); - array_foreach(&host->delayed_failing_requests, req_idx) { - (*req_idx)->to_delayed_error = - io_loop_move_timeout(&(*req_idx)->to_delayed_error); + if (host->to_failing_requests != NULL) { + host->to_failing_requests = + io_loop_move_timeout(&host->to_failing_requests); } } diff -r 160e489d7c12 -r 6b48a5675512 src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Fri Nov 22 22:07:41 2013 +0200 +++ b/src/lib-http/http-client-private.h Fri Nov 22 22:08:20 2013 +0200 @@ -75,7 +75,6 @@ unsigned int delayed_error_status; const char *delayed_error; - struct timeout *to_delayed_error; http_client_request_callback_t *callback; void *context; @@ -200,6 +199,7 @@ /* list of requests in this host that are waiting for ioloop */ ARRAY(struct http_client_request *) delayed_failing_requests; + struct timeout *to_failing_requests; /* requests are managed on a per-port basis */ ARRAY_TYPE(http_client_queue) queues; @@ -247,6 +247,7 @@ struct http_response *response); void http_client_request_send_error(struct http_client_request *req, unsigned int status, const char *error); +void http_client_request_error_delayed(struct http_client_request **_req); void http_client_request_error(struct http_client_request *req, unsigned int status, const char *error); void http_client_request_redirect(struct http_client_request *req, @@ -330,6 +331,10 @@ void http_client_host_free(struct http_client_host **_host); void http_client_host_submit_request(struct http_client_host *host, struct http_client_request *req); +void http_client_host_delay_request_error(struct http_client_host *host, + struct http_client_request *req); +void http_client_host_remove_request_error(struct http_client_host *host, + struct http_client_request *req); void http_client_host_switch_ioloop(struct http_client_host *host); static inline const char * diff -r 160e489d7c12 -r 6b48a5675512 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Fri Nov 22 22:07:41 2013 +0200 +++ b/src/lib-http/http-client-request.c Fri Nov 22 22:08:20 2013 +0200 @@ -50,7 +50,6 @@ /* * Request */ -static void http_client_request_remove_delayed(struct http_client_request *req); static struct http_client_request * http_client_request_new(struct http_client *client, const char *method, @@ -166,8 +165,8 @@ if (client->pending_requests == 0 && client->ioloop != NULL) io_loop_stop(client->ioloop); - if (req->to_delayed_error != NULL) - http_client_request_remove_delayed(req); + if (req->delayed_error != NULL) + http_client_host_remove_request_error(req->host, req); if (req->payload_input != NULL) i_stream_unref(&req->payload_input); if (req->payload_output != NULL) @@ -718,30 +717,14 @@ } } -static void http_client_request_remove_delayed(struct http_client_request *req) +void http_client_request_error_delayed(struct http_client_request **_req) { - struct http_client_request *const *reqs; - unsigned int i, count; + struct http_client_request *req = *_req; + i_assert(req->delayed_error != NULL && req->delayed_error_status != 0); http_client_request_send_error(req, req->delayed_error_status, req->delayed_error); - - timeout_remove(&req->to_delayed_error); - - reqs = array_get(&req->host->delayed_failing_requests, &count); - for (i = 0; i < count; i++) { - if (reqs[i] == req) { - array_delete(&req->host->delayed_failing_requests, i, 1); - return; - } - } - i_unreached(); -} - -static void http_client_request_error_delayed(struct http_client_request *req) -{ - http_client_request_remove_delayed(req); - http_client_request_unref(&req); + http_client_request_unref(_req); } void http_client_request_error(struct http_client_request *req, @@ -754,9 +737,7 @@ i_assert(req->delayed_error == NULL); req->delayed_error = p_strdup(req->pool, error); req->delayed_error_status = status; - req->to_delayed_error = timeout_add_short(0, - http_client_request_error_delayed, req); - array_append(&req->host->delayed_failing_requests, &req, 1); + http_client_host_delay_request_error(req->host, req); } else { http_client_request_send_error(req, status, error); http_client_request_unref(&req); From dovecot at dovecot.org Fri Nov 22 22:13:13 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 Nov 2013 22:13:13 +0200 Subject: dovecot-2.2: http: Preparse Retry-After header if response statu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c215131c2c18 changeset: 17004:c215131c2c18 user: Stephan Bosch date: Fri Nov 22 22:08:44 2013 +0200 description: http: Preparse Retry-After header if response status is 503 or 3xx. diffstat: src/lib-http/http-response-parser.c | 53 +++++++++++++++++++++++++++++++++++++ src/lib-http/http-response.h | 2 +- 2 files changed, 54 insertions(+), 1 deletions(-) diffs (100 lines): diff -r 6b48a5675512 -r c215131c2c18 src/lib-http/http-response-parser.c --- a/src/lib-http/http-response-parser.c Fri Nov 22 22:08:20 2013 +0200 +++ b/src/lib-http/http-response-parser.c Fri Nov 22 22:08:44 2013 +0200 @@ -3,6 +3,7 @@ #include "lib.h" #include "istream.h" #include "http-parser.h" +#include "http-date.h" #include "http-message-parser.h" #include "http-response-parser.h" @@ -232,10 +233,41 @@ return 0; } +static int +http_response_parse_retry_after(const char *hdrval, time_t resp_time, + time_t *retry_after_r) +{ + time_t delta; + + /* http://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-23 + Section 7.1.3: + + The value of this field can be either an HTTP-date or an integer + number of seconds (in decimal) after the time of the response. + Time spans are non-negative decimal integers, representing time in + seconds. + + Retry-After = HTTP-date / delta-seconds + delta-seconds = 1*DIGIT + */ + if (str_to_time(hdrval, &delta) >= 0) { + if (resp_time == (time_t)-1) { + return -1; + } + *retry_after_r = resp_time + delta; + return 0; + } + + return (http_date_parse + ((unsigned char *)hdrval, strlen(hdrval), retry_after_r) ? 0 : -1); +} + int http_response_parse_next(struct http_response_parser *parser, enum http_response_payload_type payload_type, struct http_response *response, const char **error_r) { + const char *hdrval; + time_t retry_after = (time_t)-1; int ret; /* make sure we finished streaming payload from previous response @@ -300,6 +332,26 @@ } } + /* http://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-23 + Section 7.1.3: + + Servers send the "Retry-After" header field to indicate how long the + user agent ought to wait before making a follow-up request. When + sent with a 503 (Service Unavailable) response, Retry-After indicates + how long the service is expected to be unavailable to the client. + When sent with any 3xx (Redirection) response, Retry-After indicates + the minimum time that the user agent is asked to wait before issuing + the redirected request. + */ + if (parser->response_status == 503 || (parser->response_status / 100) == 3) { + hdrval = http_header_field_get(parser->parser.msg.header, "Retry-After"); + if (hdrval != NULL) { + (void)http_response_parse_retry_after + (hdrval, parser->parser.msg.date, &retry_after); + /* broken Retry-After header is ignored */ + } + } + parser->state = HTTP_RESPONSE_PARSE_STATE_INIT; memset(response, 0, sizeof(*response)); @@ -309,6 +361,7 @@ response->version_minor = parser->parser.msg.version_minor; response->location = parser->parser.msg.location; response->date = parser->parser.msg.date; + response->retry_after = retry_after; response->payload = parser->parser.payload; response->header = parser->parser.msg.header; response->headers = *http_header_get_fields(response->header); /* FIXME: remove in v2.3 */ diff -r 6b48a5675512 -r c215131c2c18 src/lib-http/http-response.h --- a/src/lib-http/http-response.h Fri Nov 22 22:08:20 2013 +0200 +++ b/src/lib-http/http-response.h Fri Nov 22 22:08:44 2013 +0200 @@ -22,7 +22,7 @@ const char *reason; const char *location; - time_t date; + time_t date, retry_after; const struct http_header *header; struct istream *payload; From dovecot at dovecot.org Fri Nov 22 22:13:13 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 Nov 2013 22:13:13 +0200 Subject: dovecot-2.2: http: Implemented delayed requests scheduling. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d2f2423615d7 changeset: 17005:d2f2423615d7 user: Stephan Bosch date: Fri Nov 22 22:12:08 2013 +0200 description: http: Implemented delayed requests scheduling. Requests can now be (re)submitted with a delay. The request is not sent until the delay time expires. This facilitates handling the Retry-After header in responses. This can either be performed automatically if the indicated delay is not too long or explicitly by the code using lib-http. diffstat: src/lib-http/http-client-connection.c | 27 +++++++- src/lib-http/http-client-private.h | 10 ++- src/lib-http/http-client-queue.c | 101 +++++++++++++++++++++++++++++++++- src/lib-http/http-client-request.c | 46 +++++++++++--- src/lib-http/http-client.c | 1 + src/lib-http/http-client.h | 11 +++ 6 files changed, 174 insertions(+), 22 deletions(-) diffs (truncated from 362 to 300 lines): diff -r c215131c2c18 -r d2f2423615d7 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Fri Nov 22 22:08:44 2013 +0200 +++ b/src/lib-http/http-client-connection.c Fri Nov 22 22:12:08 2013 +0200 @@ -595,19 +595,36 @@ conn->close_indicated = response.connection_close; if (!aborted) { + bool handled = FALSE; + + /* failed Expect: */ if (response.status == 417 && req->payload_sync) { /* drop Expect: continue */ req->payload_sync = FALSE; conn->output_locked = FALSE; conn->peer->no_payload_sync = TRUE; - http_client_request_retry_response(req, &response); - + if (http_client_request_try_retry(req)) + handled = TRUE; + /* redirection */ } else if (!req->client->set.no_auto_redirect && response.status / 100 == 3 && response.status != 304 && response.location != NULL) { - /* redirect */ - http_client_request_redirect(req, response.status, response.location); - } else { + /* redirect (possibly after delay) */ + if (http_client_request_delay_from_response(req, &response) >= 0) { + http_client_request_redirect + (req, response.status, response.location); + handled = TRUE; + } + /* service unavailable */ + } else if (response.status == 503) { + /* automatically retry after delay if indicated */ + if ( response.retry_after != (time_t)-1 && + http_client_request_delay_from_response(req, &response) > 0 && + http_client_request_try_retry(req)) + handled = TRUE; + } + + if (!handled) { /* response for application */ if (!http_client_connection_return_response(conn, req, &response)) return; diff -r c215131c2c18 -r d2f2423615d7 src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Fri Nov 22 22:08:44 2013 +0200 +++ b/src/lib-http/http-client-private.h Fri Nov 22 22:12:08 2013 +0200 @@ -70,6 +70,8 @@ uoff_t payload_size, payload_offset; struct ostream *payload_output; + struct timeval release_time; + unsigned int attempts; unsigned int redirects; @@ -182,9 +184,9 @@ ARRAY_TYPE(http_client_peer) pending_peers; /* requests pending in queue to be picked up by connections */ - ARRAY_TYPE(http_client_request) request_queue; + ARRAY_TYPE(http_client_request) request_queue, delayed_request_queue; - struct timeout *to_connect; + struct timeout *to_connect, *to_delayed; }; struct http_client_host { @@ -229,6 +231,8 @@ void http_client_request_ref(struct http_client_request *req); void http_client_request_unref(struct http_client_request **_req); +int http_client_request_delay_from_response(struct http_client_request *req, + const struct http_response *response); enum http_response_payload_type http_client_request_get_payload_type(struct http_client_request *req); int http_client_request_send(struct http_client_request *req, @@ -243,8 +247,6 @@ void http_client_request_resubmit(struct http_client_request *req); void http_client_request_retry(struct http_client_request *req, unsigned int status, const char *error); -void http_client_request_retry_response(struct http_client_request *req, - struct http_response *response); void http_client_request_send_error(struct http_client_request *req, unsigned int status, const char *error); void http_client_request_error_delayed(struct http_client_request **_req); diff -r c215131c2c18 -r d2f2423615d7 src/lib-http/http-client-queue.c --- a/src/lib-http/http-client-queue.c Fri Nov 22 22:08:44 2013 +0200 +++ b/src/lib-http/http-client-queue.c Fri Nov 22 22:12:08 2013 +0200 @@ -5,10 +5,12 @@ #include "str.h" #include "hash.h" #include "array.h" +#include "bsearch-insert-pos.h" #include "llist.h" #include "ioloop.h" #include "istream.h" #include "ostream.h" +#include "time-util.h" #include "dns-lookup.h" #include "http-response-parser.h" @@ -41,6 +43,10 @@ * Queue */ +static void +http_client_queue_set_delay_timer(struct http_client_queue *queue, + struct timeval time); + static struct http_client_queue * http_client_queue_find(struct http_client_host *host, const struct http_client_peer_addr *addr) @@ -92,6 +98,7 @@ queue->name = name; queue->ips_connect_idx = 0; i_array_init(&queue->request_queue, 16); + i_array_init(&queue->delayed_request_queue, 4); array_append(&host->queues, &queue, 1); } @@ -106,8 +113,11 @@ if (array_is_created(&queue->pending_peers)) array_free(&queue->pending_peers); array_free(&queue->request_queue); + array_free(&queue->delayed_request_queue); if (queue->to_connect != NULL) timeout_remove(&queue->to_connect); + if (queue->to_delayed != NULL) + timeout_remove(&queue->to_delayed); i_free(queue->name); i_free(queue); } @@ -122,6 +132,12 @@ http_client_request_error(*req, status, error); } array_clear(&queue->request_queue); + + /* abort all delayed requests */ + array_foreach_modifiable(&queue->delayed_request_queue, req) { + http_client_request_error(*req, status, error); + } + array_clear(&queue->delayed_request_queue); } void @@ -309,10 +325,11 @@ return TRUE; } -void http_client_queue_submit_request(struct http_client_queue *queue, +static void http_client_queue_submit_now(struct http_client_queue *queue, struct http_client_request *req) { - req->queue = queue; + req->release_time.tv_sec = 0; + req->release_time.tv_usec = 0; if (req->urgent) array_insert(&queue->request_queue, 0, &req, 1); @@ -320,6 +337,84 @@ array_append(&queue->request_queue, &req, 1); } +static void +http_client_queue_delay_timeout(struct http_client_queue *queue) +{ + struct http_client_request *const *reqs; + unsigned int count, i, finished; + + io_loop_time_refresh(); + + finished = 0; + reqs = array_get(&queue->delayed_request_queue, &count); + for (i = 0; i < count; i++) { + if (timeval_cmp(&reqs[i]->release_time, &ioloop_timeval) > 0) { + break; + } + + http_client_queue_debug(queue, + "Activated delayed request %s%s", + http_client_request_label(reqs[i]), + (reqs[i]->urgent ? " (urgent)" : "")); + http_client_queue_submit_now(queue, reqs[i]); + finished++; + } + i_assert(finished > 0); + if (i < count) { + http_client_queue_set_delay_timer(queue, reqs[i]->release_time); + } + array_delete(&queue->delayed_request_queue, 0, finished); + + http_client_queue_connection_setup(queue); +} + +static void +http_client_queue_set_delay_timer(struct http_client_queue *queue, + struct timeval time) +{ + int usecs = timeval_diff_usecs(&time, &ioloop_timeval); + int msecs; + + /* round up to nearest microsecond */ + msecs = (usecs + 999) / 1000; + + /* set timer */ + if (queue->to_delayed != NULL) + timeout_remove(&queue->to_delayed); + queue->to_delayed = timeout_add + (msecs, http_client_queue_delay_timeout, queue); +} + +static int +http_client_queue_delayed_cmp(struct http_client_request *const *req1, + struct http_client_request *const *req2) +{ + return timeval_cmp(&(*req1)->release_time, &(*req2)->release_time); +} + +void http_client_queue_submit_request(struct http_client_queue *queue, + struct http_client_request *req) +{ + unsigned int insert_idx; + + req->queue = queue; + + if (req->release_time.tv_sec > 0) { + io_loop_time_refresh(); + + if (timeval_cmp(&req->release_time, &ioloop_timeval) > 0) { + (void)array_bsearch_insert_pos(&queue->delayed_request_queue, + &req, http_client_queue_delayed_cmp, &insert_idx); + array_insert(&queue->delayed_request_queue, insert_idx, &req, 1); + if (insert_idx == 0) + http_client_queue_set_delay_timer(queue, req->release_time); + return; + } + } + + http_client_queue_submit_now(queue, req); +} + struct http_client_request * http_client_queue_claim_request(struct http_client_queue *queue, const struct http_client_peer_addr *addr, bool no_urgent) @@ -372,4 +467,6 @@ { if (queue->to_connect != NULL) queue->to_connect = io_loop_move_timeout(&queue->to_connect); + if (queue->to_delayed != NULL) + queue->to_delayed = io_loop_move_timeout(&queue->to_delayed); } diff -r c215131c2c18 -r d2f2423615d7 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Fri Nov 22 22:08:44 2013 +0200 +++ b/src/lib-http/http-client-request.c Fri Nov 22 22:12:08 2013 +0200 @@ -269,6 +269,40 @@ req->payload_sync = TRUE; } +void http_client_request_delay_until(struct http_client_request *req, + time_t time) +{ + req->release_time.tv_sec = time; + req->release_time.tv_usec = 0; +} + +void http_client_request_delay(struct http_client_request *req, + time_t seconds) +{ + req->release_time = ioloop_timeval; + req->release_time.tv_sec += seconds; +} + +int http_client_request_delay_from_response(struct http_client_request *req, + const struct http_response *response) +{ + time_t retry_after = response->retry_after; + unsigned int max; + + if (retry_after == (time_t)-1) + return 0; /* no delay */ + if (retry_after < ioloop_time) + return 0; /* delay already expired */ + max = (req->client->set.max_auto_retry_delay == 0 ? + req->client->set.request_timeout_msecs / 1000 : + req->client->set.max_auto_retry_delay); + if ((retry_after - ioloop_time) > max) + return -1; /* delay too long */ + req->release_time.tv_sec = retry_after; + req->release_time.tv_usec = 0; + return 1; /* valid delay */ +} + enum http_request_state http_client_request_get_state(struct http_client_request *req) { @@ -902,7 +936,7 @@ req->conn = NULL; req->peer = NULL; From dovecot at dovecot.org Fri Nov 22 23:15:22 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 Nov 2013 23:15:22 +0200 Subject: dovecot-2.2: lib-dns: Fixed handling write() failure to dns-look... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/75b6c3b8ee20 changeset: 17006:75b6c3b8ee20 user: Timo Sirainen date: Fri Nov 22 23:15:07 2013 +0200 description: lib-dns: Fixed handling write() failure to dns-lookup server process. diffstat: src/lib-dns/dns-lookup.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r d2f2423615d7 -r 75b6c3b8ee20 src/lib-dns/dns-lookup.c --- a/src/lib-dns/dns-lookup.c Fri Nov 22 22:12:08 2013 +0200 +++ b/src/lib-dns/dns-lookup.c Fri Nov 22 23:15:07 2013 +0200 @@ -319,8 +319,10 @@ return -1; } if (write_full(client->fd, cmd, strlen(cmd)) < 0) { - dns_client_disconnect(client, t_strdup_printf( - "write(%s) failed: %m", client->path)); + result.error = t_strdup_printf("write(%s) failed: %m", + client->path); + dns_client_disconnect(client, result.error); + callback(&result, context); return -1; } From dovecot at dovecot.org Fri Nov 22 23:31:54 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 Nov 2013 23:31:54 +0200 Subject: dovecot-2.2: dns: Don't idle-disconnect clients Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fd071679352a changeset: 17007:fd071679352a user: Timo Sirainen date: Fri Nov 22 23:31:36 2013 +0200 description: dns: Don't idle-disconnect clients It's the job for the clients. diffstat: src/dns/dns-client.c | 10 ---------- 1 files changed, 0 insertions(+), 10 deletions(-) diffs (46 lines): diff -r 75b6c3b8ee20 -r fd071679352a src/dns/dns-client.c --- a/src/dns/dns-client.c Fri Nov 22 23:15:07 2013 +0200 +++ b/src/dns/dns-client.c Fri Nov 22 23:31:36 2013 +0200 @@ -14,7 +14,6 @@ struct istream *input; struct ostream *output; struct io *io; - struct timeout *to; }; #define MAX_INBUF_SIZE 1024 @@ -83,17 +82,11 @@ } } o_stream_uncork(client->output); - timeout_reset(client->to); if (client->input->eof || client->input->stream_errno != 0 || ret < 0) dns_client_destroy(&client); } -static void dns_client_timeout(struct dns_client *client) -{ - dns_client_destroy(&client); -} - static struct dns_client *dns_client_create(int fd) { struct dns_client *client; @@ -104,8 +97,6 @@ client->output = o_stream_create_fd(fd, MAX_OUTBUF_SIZE, FALSE); o_stream_set_no_error_handling(client->output, TRUE); client->io = io_add(fd, IO_READ, dns_client_input, client); - client->to = timeout_add(INPUT_TIMEOUT_MSECS, dns_client_timeout, - client); return client; } @@ -115,7 +106,6 @@ *_client = NULL; - timeout_remove(&client->to); io_remove(&client->io); i_stream_destroy(&client->input); o_stream_destroy(&client->output); From dovecot at dovecot.org Fri Nov 22 23:52:52 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 22 Nov 2013 23:52:52 +0200 Subject: dovecot-2.2: lib-storage: Don't do filesystem permission lookups... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2b79848b013f changeset: 17008:2b79848b013f user: Timo Sirainen date: Fri Nov 22 23:52:37 2013 +0200 description: lib-storage: Don't do filesystem permission lookups if storage has no mailbox root dir. diffstat: src/lib-storage/mailbox-list.c | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diffs (23 lines): diff -r fd071679352a -r 2b79848b013f src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Fri Nov 22 23:31:36 2013 +0200 +++ b/src/lib-storage/mailbox-list.c Fri Nov 22 23:52:37 2013 +0200 @@ -843,6 +843,7 @@ const char *name, struct mailbox_permissions *permissions_r) { + struct mail_storage *storage; const char *path, *parent_name, *parent_path, *p; struct stat st; @@ -865,7 +866,10 @@ (void)mailbox_list_get_root_path(list, MAILBOX_LIST_PATH_TYPE_DIR, &path); } - if (path == NULL) { + mailbox_list_get_default_storage(list, &storage); + + if (path == NULL || + (storage->class_flags & MAIL_STORAGE_CLASS_FLAG_NO_ROOT) != 0) { /* no filesystem support in storage */ } else if (stat(path, &st) < 0) { if (errno == EACCES) { From dovecot at dovecot.org Sat Nov 23 00:00:57 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 23 Nov 2013 00:00:57 +0200 Subject: dovecot-2.2: lib-http: Fixed request hang on connect() timeout. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7df4141eb57a changeset: 17009:7df4141eb57a user: Timo Sirainen date: Sat Nov 23 00:00:43 2013 +0200 description: lib-http: Fixed request hang on connect() timeout. diffstat: src/lib-http/http-client-connection.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (13 lines): diff -r 2b79848b013f -r 7df4141eb57a src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Fri Nov 22 23:52:37 2013 +0200 +++ b/src/lib-http/http-client-connection.c Sat Nov 23 00:00:43 2013 +0200 @@ -354,8 +354,7 @@ _conn->name, msecs/1000, msecs%1000); } http_client_connection_debug(conn, "%s", error); - http_client_connection_retry_requests(conn, - HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT, error); + http_client_peer_connection_failure(conn->peer, error); break; case CONNECTION_DISCONNECT_CONN_CLOSED: /* retry pending requests if possible */ From dovecot at dovecot.org Sat Nov 23 00:38:54 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 23 Nov 2013 00:38:54 +0200 Subject: dovecot-2.2: lib-storage: Non-crashing way of avoiding fs lookup... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f1e8dc453fd2 changeset: 17010:f1e8dc453fd2 user: Timo Sirainen date: Sat Nov 23 00:38:41 2013 +0200 description: lib-storage: Non-crashing way of avoiding fs lookups for storages without mail root. diffstat: src/lib-storage/mailbox-list.c | 4 +--- 1 files changed, 1 insertions(+), 3 deletions(-) diffs (23 lines): diff -r 7df4141eb57a -r f1e8dc453fd2 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Sat Nov 23 00:00:43 2013 +0200 +++ b/src/lib-storage/mailbox-list.c Sat Nov 23 00:38:41 2013 +0200 @@ -843,7 +843,6 @@ const char *name, struct mailbox_permissions *permissions_r) { - struct mail_storage *storage; const char *path, *parent_name, *parent_path, *p; struct stat st; @@ -866,10 +865,9 @@ (void)mailbox_list_get_root_path(list, MAILBOX_LIST_PATH_TYPE_DIR, &path); } - mailbox_list_get_default_storage(list, &storage); if (path == NULL || - (storage->class_flags & MAIL_STORAGE_CLASS_FLAG_NO_ROOT) != 0) { + (list->flags & MAILBOX_LIST_FLAG_NO_MAIL_FILES) != 0) { /* no filesystem support in storage */ } else if (stat(path, &st) < 0) { if (errno == EACCES) { From dovecot at dovecot.org Sat Nov 23 12:20:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 23 Nov 2013 12:20:06 +0200 Subject: dovecot-2.2: lib-http: Compiler warning fix Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8bd6c378afd2 changeset: 17011:8bd6c378afd2 user: Timo Sirainen date: Sat Nov 23 12:19:49 2013 +0200 description: lib-http: Compiler warning fix diffstat: src/lib-http/http-client-request.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r f1e8dc453fd2 -r 8bd6c378afd2 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sat Nov 23 00:38:41 2013 +0200 +++ b/src/lib-http/http-client-request.c Sat Nov 23 12:19:49 2013 +0200 @@ -296,7 +296,7 @@ max = (req->client->set.max_auto_retry_delay == 0 ? req->client->set.request_timeout_msecs / 1000 : req->client->set.max_auto_retry_delay); - if ((retry_after - ioloop_time) > max) + if ((unsigned int)(retry_after - ioloop_time) > max) return -1; /* delay too long */ req->release_time.tv_sec = retry_after; req->release_time.tv_usec = 0; From dovecot at dovecot.org Sat Nov 23 18:35:27 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 23 Nov 2013 18:35:27 +0200 Subject: dovecot-2.2: lib-dns: If existing dns connection dies in write()... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/29e98eb92955 changeset: 17012:29e98eb92955 user: Timo Sirainen date: Sat Nov 23 18:35:08 2013 +0200 description: lib-dns: If existing dns connection dies in write(), retry connect once. diffstat: src/lib-dns/dns-lookup.c | 42 ++++++++++++++++++++++++++++++++---------- 1 files changed, 32 insertions(+), 10 deletions(-) diffs (63 lines): diff -r 8bd6c378afd2 -r 29e98eb92955 src/lib-dns/dns-lookup.c --- a/src/lib-dns/dns-lookup.c Sat Nov 23 12:19:49 2013 +0200 +++ b/src/lib-dns/dns-lookup.c Sat Nov 23 18:35:08 2013 +0200 @@ -303,6 +303,28 @@ } static int +dns_client_send_request(struct dns_client *client, const char *cmd, + const char **error_r) +{ + int ret; + + if (client->fd == -1) { + if (dns_client_connect(client, error_r) < 0) + return -1; + ret = -1; + } else { + /* already connected. if write() fails, retry connecting */ + ret = 0; + } + + if (write_full(client->fd, cmd, strlen(cmd)) < 0) { + *error_r = t_strdup_printf("write(%s) failed: %m", client->path); + return ret; + } + return 1; +} + +static int dns_client_lookup_common(struct dns_client *client, const char *cmd, bool ptr_lookup, dns_lookup_callback_t *callback, void *context, @@ -310,20 +332,20 @@ { struct dns_lookup *lookup; struct dns_lookup_result result; + int ret; memset(&result, 0, sizeof(result)); result.ret = EAI_FAIL; - if (dns_client_connect(client, &result.error) < 0) { - callback(&result, context); - return -1; - } - if (write_full(client->fd, cmd, strlen(cmd)) < 0) { - result.error = t_strdup_printf("write(%s) failed: %m", - client->path); - dns_client_disconnect(client, result.error); - callback(&result, context); - return -1; + if ((ret = dns_client_send_request(client, cmd, &result.error)) <= 0) { + if (ret == 0) { + /* retry once */ + ret = dns_client_send_request(client, cmd, &result.error); + } + if (ret <= 0) { + callback(&result, context); + return -1; + } } lookup = i_new(struct dns_lookup, 1); From dovecot at dovecot.org Sat Nov 23 18:35:27 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 23 Nov 2013 18:35:27 +0200 Subject: dovecot-2.2: man: doveadm-user.1: Added description for the -u o... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bab3e86bdb6a changeset: 17013:bab3e86bdb6a user: Pascal Volk date: Sat Nov 23 16:28:04 2013 +0000 description: man: doveadm-user.1: Added description for the -u option. diffstat: doc/man/doveadm-user.1.in | 15 +++++++++++++-- 1 files changed, 13 insertions(+), 2 deletions(-) diffs (36 lines): diff -r 29e98eb92955 -r bab3e86bdb6a doc/man/doveadm-user.1.in --- a/doc/man/doveadm-user.1.in Sat Nov 23 18:35:08 2013 +0200 +++ b/doc/man/doveadm-user.1.in Sat Nov 23 16:28:04 2013 +0000 @@ -1,5 +1,5 @@ -.\" Copyright (c) 2010-2011 Dovecot authors, see the included COPYING file -.TH DOVEADM\-USER 1 "2011-11-04" "Dovecot v2.2" "Dovecot" +.\" Copyright (c) 2010-2013 Dovecot authors, see the included COPYING file +.TH DOVEADM\-USER 1 "2013-11-23" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-user \- Perform a user lookup in Dovecot\(aqs userdbs .\"------------------------------------------------------------------------ @@ -7,6 +7,7 @@ .BR doveadm " [" \-Dv "] " user [\fB\-a\fP \fIuserdb_socket_path\fP] [\fB\-f\fP \fIfield\fP] +.RB [ \-u ] [\fB\-x\fP \fIauth_info\fP] .IR user\ ... .\"------------------------------------------------------------------------ @@ -65,6 +66,16 @@ will show only the value of the specified field. .\"------------------------------------- .TP +.B \-u +When this option is given, +.BR doveadm (1) +will only show values from the +.IR userdb . +Without \-u parameter if any of the +.IR uid ", " gid ", " home " or " mail +fields are missing, their defaults are taken from configuration file. +.\"------------------------------------- +.TP .BI \-x\ auth_info .I auth_info specifies additional conditions for the From dovecot at dovecot.org Sat Nov 23 19:29:02 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 23 Nov 2013 19:29:02 +0200 Subject: dovecot-2.2: fts: Added plugin { fts_autoindex_on_save } setting. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3aa27f41fc42 changeset: 17014:3aa27f41fc42 user: Timo Sirainen date: Sat Nov 23 19:28:48 2013 +0200 description: fts: Added plugin { fts_autoindex_on_save } setting. When enabled and a mail is saved/copied, the indexer process is requested to index the changed mailbox. fts_autoindex_max_recent_msgs setting can also be used to skip indexing mailboxes that have too many \Recent messages (implying that the mailbox is never actually accessed). This corresponds to doveadm index -n parameter. diffstat: src/plugins/fts/fts-storage.c | 76 ++++++++++++++++++++++++++++++++++++++++++- 1 files changed, 75 insertions(+), 1 deletions(-) diffs (131 lines): diff -r bab3e86bdb6a -r 3aa27f41fc42 src/plugins/fts/fts-storage.c --- a/src/plugins/fts/fts-storage.c Sat Nov 23 16:28:04 2013 +0000 +++ b/src/plugins/fts/fts-storage.c Sat Nov 23 19:28:48 2013 +0200 @@ -2,6 +2,10 @@ #include "lib.h" #include "array.h" +#include "net.h" +#include "str.h" +#include "strescape.h" +#include "write-full.h" #include "mail-search-build.h" #include "mail-storage-private.h" #include "mailbox-list-private.h" @@ -22,6 +26,9 @@ #define FTS_LIST_CONTEXT(obj) \ MODULE_CONTEXT(obj, fts_mailbox_list_module) +#define INDEXER_SOCKET_NAME "indexer" +#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n" + struct fts_mailbox_list { union mailbox_list_module_context module_ctx; struct fts_backend *backend; @@ -43,6 +50,7 @@ uint32_t highest_virtual_uid; unsigned int precached:1; + unsigned int mails_saved:1; unsigned int failed:1; }; @@ -527,17 +535,59 @@ fbox->module_ctx.super.transaction_rollback(t); } +static void fts_queue_index(struct mailbox *box) +{ + struct mail_user *user = box->storage->user; + string_t *str = t_str_new(256); + const char *path, *value; + unsigned int max_recent_msgs; + int fd; + + path = t_strconcat(user->set->base_dir, "/"INDEXER_SOCKET_NAME, NULL); + fd = net_connect_unix(path); + if (fd == -1) { + i_error("net_connect_unix(%s) failed: %m", path); + return; + } + + value = mail_user_plugin_getenv(user, "fts_autoindex_max_recent_msgs"); + if (value == NULL || str_to_uint(value, &max_recent_msgs) < 0) + max_recent_msgs = 0; + + str_append(str, INDEXER_HANDSHAKE); + str_append(str, "APPEND\t0\t"); + str_append_tabescaped(str, user->username); + str_append_c(str, '\t'); + str_append_tabescaped(str, box->vname); + str_printfa(str, "\t%u\n", max_recent_msgs); + if (write_full(fd, str_data(str), str_len(str)) < 0) + i_error("write(%s) failed: %m", path); + i_close_fd(&fd); +} + static int fts_transaction_commit(struct mailbox_transaction_context *t, struct mail_transaction_commit_changes *changes_r) { + struct fts_transaction_context *ft = FTS_CONTEXT(t); struct fts_mailbox *fbox = FTS_CONTEXT(t->box); + struct mailbox *box = t->box; + bool autoindex; int ret; + autoindex = ft->mails_saved && + mail_user_plugin_getenv(box->storage->user, + "fts_autoindex_on_save") != NULL; + ret = fts_transaction_end(t); if (fbox->module_ctx.super.transaction_commit(t, changes_r) < 0) ret = -1; - return ret; + if (ret < 0) + return -1; + + if (autoindex) + fts_queue_index(box); + return 0; } static void fts_mailbox_sync_notify(struct mailbox *box, uint32_t uid, @@ -596,6 +646,28 @@ return ret; } +static int fts_save_finish(struct mail_save_context *ctx) +{ + struct fts_transaction_context *ft = FTS_CONTEXT(ctx->transaction); + struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box); + + if (fbox->module_ctx.super.save_finish(ctx) < 0) + return -1; + ft->mails_saved = TRUE; + return 0; +} + +static int fts_copy(struct mail_save_context *ctx, struct mail *mail) +{ + struct fts_transaction_context *ft = FTS_CONTEXT(ctx->transaction); + struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box); + + if (fbox->module_ctx.super.copy(ctx, mail) < 0) + return -1; + ft->mails_saved = TRUE; + return 0; +} + void fts_mailbox_allocated(struct mailbox *box) { struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list); @@ -619,6 +691,8 @@ v->transaction_commit = fts_transaction_commit; v->sync_notify = fts_mailbox_sync_notify; v->sync_deinit = fts_sync_deinit; + v->save_finish = fts_save_finish; + v->copy = fts_copy; MODULE_CONTEXT_SET(box, fts_storage_module, fbox); } From dovecot at dovecot.org Sat Nov 23 20:27:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 23 Nov 2013 20:27:48 +0200 Subject: dovecot-2.2: man: doveadm-move.1: Added description for 'user so... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0870cca2b4fb changeset: 17015:0870cca2b4fb user: Pascal Volk date: Sat Nov 23 18:24:24 2013 +0000 description: man: doveadm-move.1: Added description for 'user source_user'. diffstat: doc/man/doveadm-move.1.in | 47 +++++++++++++++++++++++++++++++++++++++-------- 1 files changed, 39 insertions(+), 8 deletions(-) diffs (87 lines): diff -r 3aa27f41fc42 -r 0870cca2b4fb doc/man/doveadm-move.1.in --- a/doc/man/doveadm-move.1.in Sat Nov 23 19:28:48 2013 +0200 +++ b/doc/man/doveadm-move.1.in Sat Nov 23 18:24:24 2013 +0000 @@ -1,5 +1,5 @@ -.\" Copyright (c) 2011 Dovecot authors, see the included COPYING file -.TH DOVEADM\-MOVE 1 "2013-11-16" "Dovecot v2.2" "Dovecot" +.\" Copyright (c) 2011-2013 Dovecot authors, see the included COPYING file +.TH DOVEADM\-MOVE 1 "2013-11-23" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-move \- Move messages matching the given search query into another mailbox @@ -9,31 +9,43 @@ .\"------------------------------------------------------------------------ .SH SYNOPSIS .BR doveadm " [" \-Dv "] " move " [" \-S -.IR socket_path "] " "destination search_query" +.IR socket_path "] " destination +.RB [ user +.IR source_user "] " search_query .br .\"------------------------------------- .BR doveadm " [" \-Dv "] " move " [" \-S .IR socket_path "] " -.BI \-A " destination search_query" +.BI \-A " destination +.RB [ user +.IR source_user "] " search_query .br .\"------------------------------------- .BR doveadm " [" \-Dv "] " move " [" \-S .IR socket_path "] " -.BI \-u " user destination search_query" +.BI \-u " user destination +.RB [ user +.IR source_user "] " search_query .\"------------------------------------- .PP .BR doveadm " [" \-Dv "] " copy " [" \-S -.IR socket_path "] " "destination search_query" +.IR socket_path "] " "destination +.RB [ user +.IR source_user "] " search_query .br .\"------------------------------------- .BR doveadm " [" \-Dv "] " copy " [" \-S .IR socket_path "] " -.BI \-A " destination search_query" +.BI \-A " destination +.RB [ user +.IR source_user "] " search_query .br .\"------------------------------------- .BR doveadm " [" \-Dv "] " copy " [" \-S .IR socket_path "] " -.BI \-u " user destination search_query" +.BI \-u " user destination +.RB [ user +.IR source_user "] " search_query .\"------------------------------------------------------------------------ .SH DESCRIPTION .B doveadm move @@ -94,6 +106,25 @@ See .BR doveadm\-search\-query (7) for details. +.\"------------------------------------- +.TP +.BI user \ source_user +The keyword +.B user +followed by a valid user name. +When this argument is present, +.BR doveadm (1) +will apply the +.I search_query +to the +.IR source_user "\(aqs " mail_location . +.br +.B Limitation: +Currently the users, specified by +.BI \-u " user" +and +.BI user " source_user," +must share the same UID and GID. .\"------------------------------------------------------------------------ .SH EXAMPLE Move jane\(aqs messages \- received in September 2011 \- from her INBOX From dovecot at dovecot.org Sat Nov 23 21:12:53 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 23 Nov 2013 21:12:53 +0200 Subject: dovecot-2.2: replicator: Added replication_dsync_parameters sett... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/556e654cc9a0 changeset: 17016:556e654cc9a0 user: Timo Sirainen date: Sat Nov 23 21:12:39 2013 +0200 description: replicator: Added replication_dsync_parameters setting to pass "doveadm sync" parameters. -f and -s parameters are added automatically when needed. diffstat: src/replication/replicator/dsync-client.c | 26 ++++++++++++++++------- src/replication/replicator/dsync-client.h | 3 +- src/replication/replicator/replicator-brain.c | 3 +- src/replication/replicator/replicator-settings.c | 2 + src/replication/replicator/replicator-settings.h | 1 + 5 files changed, 25 insertions(+), 10 deletions(-) diffs (128 lines): diff -r 0870cca2b4fb -r 556e654cc9a0 src/replication/replicator/dsync-client.c --- a/src/replication/replicator/dsync-client.c Sat Nov 23 18:24:24 2013 +0000 +++ b/src/replication/replicator/dsync-client.c Sat Nov 23 21:12:39 2013 +0200 @@ -14,11 +14,6 @@ #define DSYNC_FAIL_TIMEOUT_MSECS (1000*5) #define DOVEADM_HANDSHAKE "VERSION\tdoveadm-server\t1\t0\n" -/* normally there shouldn't be any need for locking, since replicator doesn't - start dsync in parallel for the same user. we'll do locking just in case - anyway */ -#define DSYNC_LOCK_TIMEOUT_SECS 30 - struct dsync_client { char *path; int fd; @@ -27,6 +22,7 @@ struct ostream *output; struct timeout *to; + char *dsync_params; char *state; dsync_callback_t *callback; void *context; @@ -36,13 +32,15 @@ unsigned int cmd_sent:1; }; -struct dsync_client *dsync_client_init(const char *path) +struct dsync_client * +dsync_client_init(const char *path, const char *dsync_params) { struct dsync_client *client; client = i_new(struct dsync_client, 1); client->path = i_strdup(path); client->fd = -1; + client->dsync_params = i_strdup(dsync_params); return client; } @@ -190,6 +188,8 @@ dsync_callback_t *callback, void *context) { string_t *cmd; + unsigned int pos; + char *p; i_assert(callback != NULL); i_assert(!dsync_client_is_busy(client)); @@ -207,10 +207,20 @@ cmd = t_str_new(256); str_append_c(cmd, '\t'); str_append_tabescaped(cmd, username); - str_printfa(cmd, "\tsync\t-d\t-N\t-l\t%u", DSYNC_LOCK_TIMEOUT_SECS); + str_append(cmd, "\tsync\t"); + pos = str_len(cmd); + /* insert the parameters. we can do it simply by converting + spaces into tabs, it's unlikely we'll ever need anything + more complex here. */ + str_append(cmd, client->dsync_params); + p = str_c_modifiable(cmd) + pos; + for (; *p != '\0'; p++) { + if (*p == ' ') + *p = '\t'; + } if (full) str_append(cmd, "\t-f"); - str_append(cmd, "\t-U\t-s\t"); + str_append(cmd, "\t-s\t"); if (state != NULL) str_append(cmd, state); str_append_c(cmd, '\n'); diff -r 0870cca2b4fb -r 556e654cc9a0 src/replication/replicator/dsync-client.h --- a/src/replication/replicator/dsync-client.h Sat Nov 23 18:24:24 2013 +0000 +++ b/src/replication/replicator/dsync-client.h Sat Nov 23 21:12:39 2013 +0200 @@ -10,7 +10,8 @@ typedef void dsync_callback_t(enum dsync_reply reply, const char *state, void *context); -struct dsync_client *dsync_client_init(const char *path); +struct dsync_client * +dsync_client_init(const char *path, const char *dsync_params); void dsync_client_deinit(struct dsync_client **conn); void dsync_client_sync(struct dsync_client *conn, diff -r 0870cca2b4fb -r 556e654cc9a0 src/replication/replicator/replicator-brain.c --- a/src/replication/replicator/replicator-brain.c Sat Nov 23 18:24:24 2013 +0000 +++ b/src/replication/replicator/replicator-brain.c Sat Nov 23 21:12:39 2013 +0200 @@ -80,7 +80,8 @@ brain->set->replication_max_conns) return NULL; - conn = dsync_client_init(brain->set->doveadm_socket_path); + conn = dsync_client_init(brain->set->doveadm_socket_path, + brain->set->replication_dsync_parameters); array_append(&brain->dsync_clients, &conn, 1); return conn; } diff -r 0870cca2b4fb -r 556e654cc9a0 src/replication/replicator/replicator-settings.c --- a/src/replication/replicator/replicator-settings.c Sat Nov 23 18:24:24 2013 +0000 +++ b/src/replication/replicator/replicator-settings.c Sat Nov 23 21:12:39 2013 +0200 @@ -53,6 +53,7 @@ static const struct setting_define replicator_setting_defines[] = { DEF(SET_STR, auth_socket_path), DEF(SET_STR, doveadm_socket_path), + DEF(SET_STR, replication_dsync_parameters), DEF(SET_TIME, replication_full_sync_interval), DEF(SET_UINT, replication_max_conns), @@ -63,6 +64,7 @@ const struct replicator_settings replicator_default_settings = { .auth_socket_path = "auth-userdb", .doveadm_socket_path = "doveadm-server", + .replication_dsync_parameters = "-d -N -l 30 -U", .replication_full_sync_interval = 60*60*24, .replication_max_conns = 10 diff -r 0870cca2b4fb -r 556e654cc9a0 src/replication/replicator/replicator-settings.h --- a/src/replication/replicator/replicator-settings.h Sat Nov 23 18:24:24 2013 +0000 +++ b/src/replication/replicator/replicator-settings.h Sat Nov 23 21:12:39 2013 +0200 @@ -4,6 +4,7 @@ struct replicator_settings { const char *auth_socket_path; const char *doveadm_socket_path; + const char *replication_dsync_parameters; unsigned int replication_full_sync_interval; unsigned int replication_max_conns; From dovecot at dovecot.org Sat Nov 23 21:13:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 23 Nov 2013 21:13:35 +0200 Subject: dovecot-2.2: replicator: Memory leak fix for previous commit Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/faf8af1b375d changeset: 17017:faf8af1b375d user: Timo Sirainen date: Sat Nov 23 21:13:26 2013 +0200 description: replicator: Memory leak fix for previous commit diffstat: src/replication/replicator/dsync-client.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 556e654cc9a0 -r faf8af1b375d src/replication/replicator/dsync-client.c --- a/src/replication/replicator/dsync-client.c Sat Nov 23 21:12:39 2013 +0200 +++ b/src/replication/replicator/dsync-client.c Sat Nov 23 21:13:26 2013 +0200 @@ -95,6 +95,7 @@ *_client = NULL; dsync_disconnect(client); + i_free(client->dsync_params); i_free(client->path); i_free(client); } From dovecot at dovecot.org Sat Nov 23 21:19:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 23 Nov 2013 21:19:06 +0200 Subject: dovecot-2.2: fts: Renamed fts_autoindex_on_save to fts_autoindex Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/14be468c8963 changeset: 17018:14be468c8963 user: Timo Sirainen date: Sat Nov 23 21:18:30 2013 +0200 description: fts: Renamed fts_autoindex_on_save to fts_autoindex diffstat: src/plugins/fts/fts-storage.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r faf8af1b375d -r 14be468c8963 src/plugins/fts/fts-storage.c --- a/src/plugins/fts/fts-storage.c Sat Nov 23 21:13:26 2013 +0200 +++ b/src/plugins/fts/fts-storage.c Sat Nov 23 21:18:30 2013 +0200 @@ -577,7 +577,7 @@ autoindex = ft->mails_saved && mail_user_plugin_getenv(box->storage->user, - "fts_autoindex_on_save") != NULL; + "fts_autoindex") != NULL; ret = fts_transaction_end(t); if (fbox->module_ctx.super.transaction_commit(t, changes_r) < 0) From dovecot at dovecot.org Sat Nov 23 21:37:56 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 23 Nov 2013 21:37:56 +0200 Subject: dovecot-2.2: man: doveadm-index.1: Added description for options... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/151a2cfe3d34 changeset: 17019:151a2cfe3d34 user: Pascal Volk date: Sat Nov 23 19:35:09 2013 +0000 description: man: doveadm-index.1: Added description for options -n and -q. diffstat: doc/man/doveadm-index.1.in | 46 +++++++++++++++++++++++++++++++++++++++++----- 1 files changed, 41 insertions(+), 5 deletions(-) diffs (80 lines): diff -r 14be468c8963 -r 151a2cfe3d34 doc/man/doveadm-index.1.in --- a/doc/man/doveadm-index.1.in Sat Nov 23 21:18:30 2013 +0200 +++ b/doc/man/doveadm-index.1.in Sat Nov 23 19:35:09 2013 +0000 @@ -1,21 +1,27 @@ -.\" Copyright (c) 2010-2011 Dovecot authors, see the included COPYING file -.TH DOVEADM\-INDEX 1 "2011-05-11" "Dovecot v2.2" "Dovecot" +.\" Copyright (c) 2010-2013 Dovecot authors, see the included COPYING file +.TH DOVEADM\-INDEX 1 "2013-11-23" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-index \- Index mailboxes .\"------------------------------------------------------------------------ .SH SYNOPSIS .BR doveadm " [" \-Dv "] " index " [" \-S -.IR socket_path "] " mailbox +.IR socket_path ] +.RB [ \-q "] [" \-n +.IR max_recent "] " mailbox .\"------------------------------------- .br .BR doveadm " [" \-Dv "] " index " [" \-S .IR socket_path "] " -.BI \-A \ mailbox +.B \-A +.RB [ \-q "] [" \-n +.IR max_recent "] " mailbox .\"------------------------------------- .br .BR doveadm " [" \-Dv "] " index " [" \-S .IR socket_path "] " -.BI \-u " user mailbox" +.BI \-u \ user +.RB [ \-q "] [" \-n +.IR max_recent "] " mailbox .\"------------------------------------------------------------------------ .SH DESCRIPTION Add unindexed messages in a mailbox into index/cache file. If full text @@ -26,6 +32,17 @@ mailboxes that user\(aqs client hasn\(aqt accessed yet. You can use .B doveadm dump command to show a specific mailbox\(aqs current caching decisions. +.PP +Messages can also be added automatically to full text search index using: +.sp +.nf +plugin { + ... + fts_autoindex = yes +} +.fi +in +.IR @pkgsysconfdir@/conf.d/90\-plugin.conf . .\"------------------------------------------------------------------------ @INCLUDE:global-options@ .\" --- command specific options --- "/. @@ -35,6 +52,25 @@ .\"------------------------------------- @INCLUDE:option-A@ .\"------------------------------------- +.TP +.BI \-n \ max_recent +An integer value, which specifies the maximum number of \(rsRecent +messages in mailboxes. +If the mailbox contains more than +.I max_recent +messages with \(rsRecent flag set, the mailbox will not be indexed. +This may be useful to avoid unnecessary indexing for large mailboxes that +are never opened. +.\"------------------------------------- +.TP +.B \-q +Queues the indexing to be run by indexer process. +Without \-q the indexing is done directly by the +.BR doveadm (1) +process. +Some backends like fts\-lucene can\(aqt handle multiple processes updating +the indexes simultaneously, so \-q should usually be used on production. +.\"------------------------------------- @INCLUDE:option-S-socket@ .\"------------------------------------- @INCLUDE:option-u-user@ From dovecot at dovecot.org Sun Nov 24 01:29:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 Nov 2013 01:29:06 +0200 Subject: dovecot-2.2: man: doveadm-move.1: Fixed syntax errors, escaped s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8a3d0426f514 changeset: 17020:8a3d0426f514 user: Pascal Volk date: Sat Nov 23 23:22:09 2013 +0000 description: man: doveadm-move.1: Fixed syntax errors, escaped some dashes. diffstat: doc/man/doveadm-move.1.in | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 151a2cfe3d34 -r 8a3d0426f514 doc/man/doveadm-move.1.in --- a/doc/man/doveadm-move.1.in Sat Nov 23 19:35:09 2013 +0000 +++ b/doc/man/doveadm-move.1.in Sat Nov 23 23:22:09 2013 +0000 @@ -132,7 +132,7 @@ .PP .nf .B doveadm move \-u jane Archive/2011/09 mailbox INBOX BEFORE \(rs -.B 2011-10-01 SINCE 01-Sep-2011 +.B 2011\-10\-01 SINCE 01\-Sep\-2011 .fi .\"------------------------------------------------------------------------ @INCLUDE:reporting-bugs@ From dovecot at dovecot.org Sun Nov 24 20:27:57 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 Nov 2013 20:27:57 +0200 Subject: dovecot-2.2: auth: Fixed (non-exploitable) buffer overflow in DI... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2b3541c3e262 changeset: 17021:2b3541c3e262 user: Timo Sirainen date: Sun Nov 24 20:27:27 2013 +0200 description: auth: Fixed (non-exploitable) buffer overflow in DIGEST-MD5 data parsing. Found by Jann Horn. diffstat: src/auth/mech-digest-md5.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 8a3d0426f514 -r 2b3541c3e262 src/auth/mech-digest-md5.c --- a/src/auth/mech-digest-md5.c Sat Nov 23 23:22:09 2013 +0000 +++ b/src/auth/mech-digest-md5.c Sun Nov 24 20:27:27 2013 +0200 @@ -246,7 +246,7 @@ *value = p+1; /* skip trailing whitespace in key */ - while (IS_LWS(p[-1])) + while (p > *data && IS_LWS(p[-1])) p--; *p = '\0'; From dovecot at dovecot.org Sun Nov 24 20:42:39 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 Nov 2013 20:42:39 +0200 Subject: dovecot-2.2: Data stack has now an extra NULL pointer padding be... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2d2614769339 changeset: 17022:2d2614769339 user: Timo Sirainen date: Sun Nov 24 20:41:42 2013 +0200 description: Data stack has now an extra NULL pointer padding before its actual data starts. This is just an extra safety measure against buffer overflows. Patch by Jann Horn. diffstat: src/lib/data-stack.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (21 lines): diff -r 2b3541c3e262 -r 2d2614769339 src/lib/data-stack.c --- a/src/lib/data-stack.c Sun Nov 24 20:27:27 2013 +0200 +++ b/src/lib/data-stack.c Sun Nov 24 20:41:42 2013 +0200 @@ -32,6 +32,9 @@ struct stack_block *next; size_t size, left, lowwater; + /* always NULL and here just in case something accesses + the memory in front of an allocated area */ + char *nullpad; /* unsigned char data[]; */ }; @@ -299,6 +302,7 @@ block->left = 0; block->lowwater = block->size; block->next = NULL; + block->nullpad = NULL; #ifdef DEBUG memset(STACK_BLOCK_DATA(block), CLEAR_CHR, alloc_size); From dovecot at dovecot.org Sun Nov 24 20:42:39 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 Nov 2013 20:42:39 +0200 Subject: dovecot-2.2: man: doveadm-expunge.1: Added option -d also to sec... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/440acf9fd5ce changeset: 17023:440acf9fd5ce user: Pascal Volk date: Sun Nov 24 18:02:40 2013 +0000 description: man: doveadm-expunge.1: Added option -d also to section synopsis. diffstat: doc/man/doveadm-expunge.1.in | 16 ++++++++++------ 1 files changed, 10 insertions(+), 6 deletions(-) diffs (34 lines): diff -r 2d2614769339 -r 440acf9fd5ce doc/man/doveadm-expunge.1.in --- a/doc/man/doveadm-expunge.1.in Sun Nov 24 20:41:42 2013 +0200 +++ b/doc/man/doveadm-expunge.1.in Sun Nov 24 18:02:40 2013 +0000 @@ -1,20 +1,24 @@ -.\" Copyright (c) 2010-2012 Dovecot authors, see the included COPYING file -.TH DOVEADM\-EXPUNGE 1 "2012-11-27" "Dovecot v2.2" "Dovecot" +.\" Copyright (c) 2010-2013 Dovecot authors, see the included COPYING file +.TH DOVEADM\-EXPUNGE 1 "2013-11-24" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-expunge \- Expunge messages matching given search query .\"------------------------------------------------------------------------ .SH SYNOPSIS .BR doveadm " [" \-Dv "] " expunge " [" \-S -.IR socket_path "] " search_query +.IR socket_path ] +.RB [ \-d ] +.I search_query .br .\"------------------------------------- .BR doveadm " [" \-Dv "] " expunge " [" \-S -.IR socket_path "] " -.BI \-A " search_query" +.IR socket_path ] +.RB [ \-d ] +.BI \-A " search_query" .br .\"------------------------------------- .BR doveadm " [" \-Dv "] " expunge " [" \-S -.IR socket_path "] " +.IR socket_path ] +.RB [ \-d ] .BI \-u " user search_query" .\"------------------------------------------------------------------------ .SH DESCRIPTION From dovecot at dovecot.org Sun Nov 24 20:42:39 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 Nov 2013 20:42:39 +0200 Subject: dovecot-2.2: man: doveadm-import.1: Added description for option... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/be86adfff0ca changeset: 17024:be86adfff0ca user: Pascal Volk date: Sun Nov 24 18:29:55 2013 +0000 description: man: doveadm-import.1: Added description for option -s. diffstat: doc/man/doveadm-import.1.in | 24 +++++++++++++++++++----- 1 files changed, 19 insertions(+), 5 deletions(-) diffs (50 lines): diff -r 440acf9fd5ce -r be86adfff0ca doc/man/doveadm-import.1.in --- a/doc/man/doveadm-import.1.in Sun Nov 24 18:02:40 2013 +0000 +++ b/doc/man/doveadm-import.1.in Sun Nov 24 18:29:55 2013 +0000 @@ -1,20 +1,24 @@ -.\" Copyright (c) 2010 Dovecot authors, see the included COPYING file -.TH DOVEADM\-IMPORT 1 "2010-11-26" "Dovecot v2.2" "Dovecot" +.\" Copyright (c) 2010-2013 Dovecot authors, see the included COPYING file +.TH DOVEADM\-IMPORT 1 "2013-11-24" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-import \- Import messages matching given search query .\"------------------------------------------------------------------------ .SH SYNOPSIS .BR doveadm " [" \-Dv "] " import " [" \-S -.IR socket_path "] " "source_location dest_parent search_query" +.IR socket_path ] +.RB [ \-s ] +.I source_location dest_parent search_query .br .\"------------------------------------- .BR doveadm " [" \-Dv "] " import " [" \-S -.IR socket_path "] " +.IR socket_path ] +.RB [ \-s ] .BI \-A " source_location dest_parent search_query" .br .\"------------------------------------- .BR doveadm " [" \-Dv "] " import " [" \-S -.IR socket_path "] " +.IR socket_path ] +.RB [ \-s ] .BI \-u " user source_location dest_parent search_query" .\"------------------------------------------------------------------------ .SH DESCRIPTION @@ -50,6 +54,16 @@ .\"------------------------------------- @INCLUDE:option-S-socket@ .\"------------------------------------- +.TP +.B \-s +When the +.B \-s +option is present, +.I dest_parent +and all new mailboxes under it will be listed in the +.I subscriptions +file. +.\"------------------------------------- @INCLUDE:option-u-user@ .\"------------------------------------------------------------------------ .SH ARGUMENTS From dovecot at dovecot.org Sun Nov 24 22:07:49 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 Nov 2013 22:07:49 +0200 Subject: dovecot-2.2: man: doveadm-log.1: Added description for option -s. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/89dd80dfb86e changeset: 17025:89dd80dfb86e user: Pascal Volk date: Sun Nov 24 19:58:45 2013 +0000 description: man: doveadm-log.1: Added description for option -s. diffstat: doc/man/doveadm-log.1.in | 18 ++++++++++++++---- 1 files changed, 14 insertions(+), 4 deletions(-) diffs (41 lines): diff -r be86adfff0ca -r 89dd80dfb86e doc/man/doveadm-log.1.in --- a/doc/man/doveadm-log.1.in Sun Nov 24 18:29:55 2013 +0000 +++ b/doc/man/doveadm-log.1.in Sun Nov 24 19:58:45 2013 +0000 @@ -1,10 +1,11 @@ -.\" Copyright (c) 2010-2012 Dovecot authors, see the included COPYING file -.TH DOVEADM\-LOG 1 "2012-02-22" "Dovecot v2.2" "Dovecot" +.\" Copyright (c) 2010-2013 Dovecot authors, see the included COPYING file +.TH DOVEADM\-LOG 1 "2013-11-24" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-log \- Locate, test or reopen Dovecot\(aqs log files .\"------------------------------------------------------------------------ .SH SYNOPSIS -.BR doveadm " [" \-Dv "] " "log errors" +.BR doveadm " [" \-Dv "] " "log errors" " [" \-s +.IR min_timestamp ] .\"------------------------------------- .br .BR doveadm " [" \-Dv "] " "log find" @@ -30,12 +31,21 @@ .\"------------------------------------------------------------------------ .SH COMMANDS .SS log errors -.B doveadm log errors +.BR "doveadm log errors" " [" \-s +.IR min_timestamp ] .PP The .B log errors command is used to show the last \- up to 1,000 \- errors and warnings. If no output is generated, no errors have occurred since the last start. +.PP +.TP +.BI \-s \ min_timestamp +An integer value, representing seconds since the epoch \- also known as +Unix timestamp. +When a min_timestamp was given, +.BR doveadm (1) +will only show errors occurred since that point in time. .\"------------------------------------- .SS log find .B doveadm log find From dovecot at dovecot.org Sun Nov 24 22:20:03 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 Nov 2013 22:20:03 +0200 Subject: dovecot-2.2: Fixed sscanf() error handling. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a9d4cdb09297 changeset: 17026:a9d4cdb09297 user: Timo Sirainen date: Sun Nov 24 21:41:06 2013 +0200 description: Fixed sscanf() error handling. Patch by Jann Horn. diffstat: src/lib-dns/dns-lookup.c | 2 +- src/lib-settings/settings.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 89dd80dfb86e -r a9d4cdb09297 src/lib-dns/dns-lookup.c --- a/src/lib-dns/dns-lookup.c Sun Nov 24 19:58:45 2013 +0000 +++ b/src/lib-dns/dns-lookup.c Sun Nov 24 21:41:06 2013 +0200 @@ -102,7 +102,7 @@ } /* first line: */ if (sscanf(line, "%d %u", &result->ret, - &result->ips_count) == 0) + &result->ips_count) != 2) return -1; if (result->ret != 0) { result->error = net_gethosterror(result->ret); diff -r 89dd80dfb86e -r a9d4cdb09297 src/lib-settings/settings.c --- a/src/lib-settings/settings.c Sun Nov 24 19:58:45 2013 +0000 +++ b/src/lib-settings/settings.c Sun Nov 24 21:41:06 2013 +0200 @@ -44,7 +44,7 @@ { int num; - if (!sscanf(value, "%i", &num) || num < 0) + if (sscanf(value, "%i", &num) != 1 || num < 0) return t_strconcat("Invalid number: ", value, NULL); *result = num; return NULL; From dovecot at dovecot.org Sun Nov 24 22:23:13 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 Nov 2013 22:23:13 +0200 Subject: dovecot-2.2: man: doveadm-mailbox.1: Added -g's description for ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ace36e525e1f changeset: 17027:ace36e525e1f user: Pascal Volk date: Sun Nov 24 20:19:48 2013 +0000 description: man: doveadm-mailbox.1: Added -g's description for mailbox create. diffstat: doc/man/doveadm-mailbox.1.in | 12 +++++++++--- 1 files changed, 9 insertions(+), 3 deletions(-) diffs (34 lines): diff -r a9d4cdb09297 -r ace36e525e1f doc/man/doveadm-mailbox.1.in --- a/doc/man/doveadm-mailbox.1.in Sun Nov 24 21:41:06 2013 +0200 +++ b/doc/man/doveadm-mailbox.1.in Sun Nov 24 20:19:48 2013 +0000 @@ -1,5 +1,5 @@ -.\" Copyright (c) 2010 Dovecot authors, see the included COPYING file -.TH DOVEADM\-MAILBOX 1 "2010-11-25" "Dovecot v2.2" "Dovecot" +.\" Copyright (c) 2010-2013 Dovecot authors, see the included COPYING file +.TH DOVEADM\-MAILBOX 1 "2013-11-24" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-mailbox \- Commands related to handling mailboxes .\"------------------------------------------------------------------------ @@ -41,6 +41,8 @@ .B doveadm mailbox create [\fB\-A\fP|\fB\-u\fP \fIuser\fP] [\fB\-S\fP \fIsocket_path\fP] +.RB [ \-g +.IR guid ] .RB [ \-s ] .IR mailbox\ ... .PP @@ -51,8 +53,12 @@ .I mail field, returned by the userdb. .PP -When the +.TP +.BI \-g \ guid +Create the mailbox with the specified GUID. +.TP .B \-s +When this option was given, the created mailboxes will be also added to the user\(aqs subscriptions. .\"------------------------------------------------------------------------ From dovecot at dovecot.org Sun Nov 24 23:02:51 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 24 Nov 2013 23:02:51 +0200 Subject: dovecot-2.2: lib-compression: Added support for liblzma (xz) Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/81e486aacbc7 changeset: 17028:81e486aacbc7 user: Timo Sirainen date: Sun Nov 24 23:02:13 2013 +0200 description: lib-compression: Added support for liblzma (xz) Annoyingly this is mainly copy&pasted [io]stream-bzlib, but I'm not sure if it's worth the effort to try to create common functions for them. diffstat: configure.ac | 24 ++ src/lib-compression/Makefile.am | 2 + src/lib-compression/compression.c | 16 + src/lib-compression/istream-lzma.c | 342 +++++++++++++++++++++++++++++++++++++ src/lib-compression/istream-zlib.h | 1 + src/lib-compression/ostream-lzma.c | 229 ++++++++++++++++++++++++ src/lib-compression/ostream-zlib.h | 1 + 7 files changed, 615 insertions(+), 0 deletions(-) diffs (truncated from 693 to 300 lines): diff -r ace36e525e1f -r 81e486aacbc7 configure.ac --- a/configure.ac Sun Nov 24 20:19:48 2013 +0000 +++ b/configure.ac Sun Nov 24 23:02:13 2013 +0200 @@ -179,6 +179,11 @@ TEST_WITH(bzlib, $withval), want_bzlib=auto) +AC_ARG_WITH(lzma, +AS_HELP_STRING([--with-lzma], [Build with LZMA compression support]), + TEST_WITH(lzma, $withval), + want_lzma=auto) + AC_ARG_WITH(libcap, AS_HELP_STRING([--with-libcap], [Build with libcap support (Dropping capabilities).]), TEST_WITH(libcap, $withval), @@ -2657,6 +2662,25 @@ fi ]) fi + +if test "$want_lzma" != "no"; then + AC_CHECK_HEADER(lzma.h, [ + AC_CHECK_LIB(lzma, lzma_stream_decoder, [ + have_lzma=yes + have_compress_lib=yes + AC_DEFINE(HAVE_LZMA,, Define if you have lzma library) + COMPRESS_LIBS="$COMPRESS_LIBS -llzma" + ], [ + if test "$want_lzma" = "yes"; then + AC_ERROR([Can't build with lzma support: liblzma not found]) + fi + ]) + ], [ + if test "$want_lzma" = "yes"; then + AC_ERROR([Can't build with lzma support: lzma.h not found]) + fi + ]) +fi AC_SUBST(COMPRESS_LIBS) AM_CONDITIONAL(BUILD_ZLIB_PLUGIN, test "$have_compress_lib" = "yes") diff -r ace36e525e1f -r 81e486aacbc7 src/lib-compression/Makefile.am --- a/src/lib-compression/Makefile.am Sun Nov 24 20:19:48 2013 +0000 +++ b/src/lib-compression/Makefile.am Sun Nov 24 23:02:13 2013 +0200 @@ -6,8 +6,10 @@ libcompression_la_SOURCES = \ compression.c \ + istream-lzma.c \ istream-zlib.c \ istream-bzlib.c \ + ostream-lzma.c \ ostream-zlib.c \ ostream-bzlib.c libcompression_la_LIBADD = \ diff -r ace36e525e1f -r 81e486aacbc7 src/lib-compression/compression.c --- a/src/lib-compression/compression.c Sun Nov 24 20:19:48 2013 +0000 +++ b/src/lib-compression/compression.c Sun Nov 24 23:02:13 2013 +0200 @@ -16,6 +16,10 @@ # define i_stream_create_bz2 NULL # define o_stream_create_bz2 NULL #endif +#ifndef HAVE_LZMA +# define i_stream_create_lzma NULL +# define o_stream_create_lzma NULL +#endif static bool is_compressed_zlib(struct istream *input) { @@ -49,6 +53,16 @@ return memcmp(data + 4, "\x31\x41\x59\x26\x53\x59", 6) == 0; } +static bool is_compressed_xz(struct istream *input) +{ + const unsigned char *data; + size_t size; + + if (i_stream_read_data(input, &data, &size, 6 - 1) <= 0) + return FALSE; + return memcmp(data, "\xfd\x37\x7a\x58\x5a", 6) == 0; +} + const struct compression_handler *compression_lookup_handler(const char *name) { unsigned int i; @@ -97,5 +111,7 @@ i_stream_create_bz2, o_stream_create_bz2 }, { "deflate", NULL, NULL, i_stream_create_deflate, o_stream_create_deflate }, + { "xz", ".xz", is_compressed_xz, + i_stream_create_lzma, o_stream_create_lzma }, { NULL, NULL, NULL, NULL, NULL } }; diff -r ace36e525e1f -r 81e486aacbc7 src/lib-compression/istream-lzma.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-compression/istream-lzma.c Sun Nov 24 23:02:13 2013 +0200 @@ -0,0 +1,342 @@ +/* Copyright (c) 2010-2013 Dovecot authors, see the included COPYING file */ + +#include "lib.h" + +#ifdef HAVE_LZMA + +#include "istream-private.h" +#include "istream-zlib.h" +#include + +#define CHUNK_SIZE (1024*64) + +#define LZMA_MEMORY_LIMIT (1024*1024*80) + +struct lzma_istream { + struct istream_private istream; + + lzma_stream strm; + uoff_t eof_offset, stream_size; + size_t high_pos; + struct stat last_parent_statbuf; + + unsigned int log_errors:1; + unsigned int marked:1; + unsigned int strm_closed:1; +}; + +static void i_stream_lzma_close(struct iostream_private *stream, + bool close_parent) +{ + struct lzma_istream *zstream = (struct lzma_istream *)stream; + + if (!zstream->strm_closed) { + lzma_end(&zstream->strm); + zstream->strm_closed = TRUE; + } + if (close_parent) + i_stream_close(zstream->istream.parent); +} + +static void lzma_read_error(struct lzma_istream *zstream, const char *error) +{ + io_stream_set_error(&zstream->istream.iostream, + "lzma.read(%s): %s at %"PRIuUOFF_T, + i_stream_get_name(&zstream->istream.istream), error, + zstream->istream.abs_start_offset + + zstream->istream.istream.v_offset); + if (zstream->log_errors) + i_error("%s", zstream->istream.iostream.error); +} + +static ssize_t i_stream_lzma_read(struct istream_private *stream) +{ + struct lzma_istream *zstream = (struct lzma_istream *)stream; + const unsigned char *data; + uoff_t high_offset; + size_t size, out_size; + lzma_ret ret; + + high_offset = stream->istream.v_offset + (stream->pos - stream->skip); + if (zstream->eof_offset == high_offset) { + i_assert(zstream->high_pos == 0 || + zstream->high_pos == stream->pos); + stream->istream.eof = TRUE; + return -1; + } + + if (stream->pos < zstream->high_pos) { + /* we're here because we seeked back within the read buffer. */ + ret = zstream->high_pos - stream->pos; + stream->pos = zstream->high_pos; + zstream->high_pos = 0; + + if (zstream->eof_offset != (uoff_t)-1) { + high_offset = stream->istream.v_offset + + (stream->pos - stream->skip); + i_assert(zstream->eof_offset == high_offset); + stream->istream.eof = TRUE; + } + return ret; + } + zstream->high_pos = 0; + + if (stream->pos + CHUNK_SIZE > stream->buffer_size) { + /* try to keep at least CHUNK_SIZE available */ + if (!zstream->marked && stream->skip > 0) { + /* don't try to keep anything cached if we don't + have a seek mark. */ + i_stream_compress(stream); + } + if (stream->max_buffer_size == 0 || + stream->buffer_size < stream->max_buffer_size) + i_stream_grow_buffer(stream, CHUNK_SIZE); + + if (stream->pos == stream->buffer_size) { + if (stream->skip > 0) { + /* lose our buffer cache */ + i_stream_compress(stream); + } + + if (stream->pos == stream->buffer_size) + return -2; /* buffer full */ + } + } + + if (i_stream_read_data(stream->parent, &data, &size, 0) < 0) { + if (stream->parent->stream_errno != 0) { + stream->istream.stream_errno = + stream->parent->stream_errno; + } else { + i_assert(stream->parent->eof); + lzma_read_error(zstream, "unexpected EOF"); + stream->istream.stream_errno = EINVAL; + } + return -1; + } + if (size == 0) { + /* no more input */ + i_assert(!stream->istream.blocking); + return 0; + } + + zstream->strm.next_in = data; + zstream->strm.avail_in = size; + + out_size = stream->buffer_size - stream->pos; + zstream->strm.next_out = stream->w_buffer + stream->pos; + zstream->strm.avail_out = out_size; + ret = lzma_code(&zstream->strm, LZMA_RUN); + + out_size -= zstream->strm.avail_out; + stream->pos += out_size; + + i_stream_skip(stream->parent, size - zstream->strm.avail_in); + + switch (ret) { + case LZMA_OK: + break; + case LZMA_DATA_ERROR: + case LZMA_BUF_ERROR: + lzma_read_error(zstream, "corrupted data"); + stream->istream.stream_errno = EINVAL; + return -1; + case LZMA_FORMAT_ERROR: + lzma_read_error(zstream, "wrong magic in header (not xz file?)"); + stream->istream.stream_errno = EINVAL; + return -1; + case LZMA_OPTIONS_ERROR: + lzma_read_error(zstream, "Unsupported xz options"); + stream->istream.stream_errno = EINVAL; + return -1; + case LZMA_MEM_ERROR: + i_fatal_status(FATAL_OUTOFMEM, "lzma.read(%s): Out of memory", + i_stream_get_name(&stream->istream)); + case LZMA_STREAM_END: + zstream->eof_offset = stream->istream.v_offset + + (stream->pos - stream->skip); + zstream->stream_size = zstream->eof_offset; + if (out_size == 0) { + stream->istream.eof = TRUE; + return -1; + } + break; + default: + lzma_read_error(zstream, t_strdup_printf( + "lzma_code() failed with %d", ret)); + stream->istream.stream_errno = EINVAL; + return -1; + } + if (out_size == 0) { + /* read more input */ + return i_stream_lzma_read(stream); + } + return out_size; +} + +static void i_stream_lzma_init(struct lzma_istream *zstream) +{ + lzma_ret ret; + + ret = lzma_stream_decoder(&zstream->strm, LZMA_MEMORY_LIMIT, + LZMA_CONCATENATED); + switch (ret) { + case LZMA_OK: + break; + case LZMA_MEM_ERROR: + i_fatal_status(FATAL_OUTOFMEM, "lzma: Out of memory"); + default: + i_fatal("lzma_stream_decoder() failed with ret=%d", ret); + } +} + +static void i_stream_lzma_reset(struct lzma_istream *zstream) +{ + struct istream_private *stream = &zstream->istream; + + i_stream_seek(stream->parent, stream->parent_start_offset); + zstream->eof_offset = (uoff_t)-1; + zstream->strm.next_in = NULL; + zstream->strm.avail_in = 0; + + stream->parent_expected_offset = stream->parent_start_offset; From dovecot at dovecot.org Mon Nov 25 00:44:37 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 25 Nov 2013 00:44:37 +0200 Subject: dovecot-2.2: Makefile: Added missing header files Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/22aef00ae211 changeset: 17029:22aef00ae211 user: Timo Sirainen date: Mon Nov 25 00:44:21 2013 +0200 description: Makefile: Added missing header files diffstat: src/plugins/mail-filter/Makefile.am | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (10 lines): diff -r 81e486aacbc7 -r 22aef00ae211 src/plugins/mail-filter/Makefile.am --- a/src/plugins/mail-filter/Makefile.am Sun Nov 24 23:02:13 2013 +0200 +++ b/src/plugins/mail-filter/Makefile.am Mon Nov 25 00:44:21 2013 +0200 @@ -17,4 +17,6 @@ ostream-ext-filter.c noinst_HEADERS = \ + istream-ext-filter.h \ + ostream-ext-filter.h mail-filter-plugin.h From dovecot at dovecot.org Mon Nov 25 00:46:49 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 25 Nov 2013 00:46:49 +0200 Subject: dovecot-2.2: Makefile: Added missing \ Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c86996b40168 changeset: 17030:c86996b40168 user: Timo Sirainen date: Mon Nov 25 00:46:40 2013 +0200 description: Makefile: Added missing \ diffstat: src/plugins/mail-filter/Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (10 lines): diff -r 22aef00ae211 -r c86996b40168 src/plugins/mail-filter/Makefile.am --- a/src/plugins/mail-filter/Makefile.am Mon Nov 25 00:44:21 2013 +0200 +++ b/src/plugins/mail-filter/Makefile.am Mon Nov 25 00:46:40 2013 +0200 @@ -18,5 +18,5 @@ noinst_HEADERS = \ istream-ext-filter.h \ - ostream-ext-filter.h + ostream-ext-filter.h \ mail-filter-plugin.h From dovecot at dovecot.org Mon Nov 25 02:29:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 25 Nov 2013 02:29:08 +0200 Subject: dovecot-2.2: Released v2.2.9. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2852a7c55fc7 changeset: 17031:2852a7c55fc7 user: Timo Sirainen date: Mon Nov 25 01:24:32 2013 +0200 description: Released v2.2.9. diffstat: NEWS | 14 ++++++++++++++ configure.ac | 2 +- 2 files changed, 15 insertions(+), 1 deletions(-) diffs (33 lines): diff -r c86996b40168 -r 2852a7c55fc7 NEWS --- a/NEWS Mon Nov 25 00:46:40 2013 +0200 +++ b/NEWS Mon Nov 25 01:24:32 2013 +0200 @@ -1,3 +1,17 @@ +v2.2.9 2013-11-25 Timo Sirainen + + + Full text search indexing can now be done automatically after + saving/copying mails by setting plugin { fts_autoindex=yes } + + replicator: Added replication_dsync_parameters setting to pass + "doveadm sync" parameters (for controlling what to replicate). + + Added mail-filter plugin + + Added liblzma/xz support (zlib_save=xz) + - v2.2.8's improved cache file handling exposed several old bugs + related to fetching mail headers. + - v2.2.7's iostream handling changes were causing some connections + to be disconnected before flushing their output (e.g. POP3 logout + message wasn't being sent) + v2.2.8 2013-11-19 Timo Sirainen + Mail cache lookups work for the mail being saved. This improves diff -r c86996b40168 -r 2852a7c55fc7 configure.ac --- a/configure.ac Mon Nov 25 00:46:40 2013 +0200 +++ b/configure.ac Mon Nov 25 01:24:32 2013 +0200 @@ -2,7 +2,7 @@ # Be sure to update ABI version also if anything changes that might require # recompiling plugins. Most importantly that means if any structs are changed. -AC_INIT([Dovecot],[2.2.8],[dovecot at dovecot.org]) +AC_INIT([Dovecot],[2.2.9],[dovecot at dovecot.org]) AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv7($PACKAGE_VERSION)", [Dovecot ABI version]) AC_CONFIG_SRCDIR([src]) From dovecot at dovecot.org Mon Nov 25 02:29:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 25 Nov 2013 02:29:08 +0200 Subject: dovecot-2.2: Added tag 2.2.9 for changeset 2852a7c55fc7 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a09e0dced4c4 changeset: 17032:a09e0dced4c4 user: Timo Sirainen date: Mon Nov 25 01:24:32 2013 +0200 description: Added tag 2.2.9 for changeset 2852a7c55fc7 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 2852a7c55fc7 -r a09e0dced4c4 .hgtags --- a/.hgtags Mon Nov 25 01:24:32 2013 +0200 +++ b/.hgtags Mon Nov 25 01:24:32 2013 +0200 @@ -112,3 +112,4 @@ 1de9a494cb2520986da97fa73cf87a3781ea2d17 2.2.6 b5b118701aa83e7b047e53f51aa5a4622ee812df 2.2.7 095a777edc2623fad2ad9d6321ed93a804e85889 2.2.8 +2852a7c55fc756e489ebff6aefcb0553f638e851 2.2.9 From dovecot at dovecot.org Mon Nov 25 02:29:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 25 Nov 2013 02:29:08 +0200 Subject: dovecot-2.2: Added signature for changeset 2852a7c55fc7 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7af50f53cc7e changeset: 17033:7af50f53cc7e user: Timo Sirainen date: Mon Nov 25 01:24:35 2013 +0200 description: Added signature for changeset 2852a7c55fc7 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r a09e0dced4c4 -r 7af50f53cc7e .hgsigs --- a/.hgsigs Mon Nov 25 01:24:32 2013 +0200 +++ b/.hgsigs Mon Nov 25 01:24:35 2013 +0200 @@ -75,3 +75,4 @@ 1de9a494cb2520986da97fa73cf87a3781ea2d17 0 iEYEABECAAYFAlJCi2cACgkQyUhSUUBVisnoWQCfecKzvsHw/MC/5DDuZXIF/CmkaBUAn2DfADK7V0fIWZ49GPyL+BPbYCXy b5b118701aa83e7b047e53f51aa5a4622ee812df 0 iEYEABECAAYFAlJ2rGIACgkQyUhSUUBVislFRwCfTgg6BYRGpKAOAvE3hhhIskLwxLUAoJjRlth0ob0ahrqIhg7du4+Z0C3j 095a777edc2623fad2ad9d6321ed93a804e85889 0 iEYEABECAAYFAlKL1YoACgkQyUhSUUBVismrpQCeP/ab1oKZp0VJ6PwmnRgDthf2hi8AoJZB+KuQIgKA5iBx3NpGcaZjmlzc +2852a7c55fc756e489ebff6aefcb0553f638e851 0 iEYEABECAAYFAlKSirAACgkQyUhSUUBVisn17ACgjsnbSktYkNMvoO2FNjA2DVBNYDwAnAtQ1Zjemwvi+Pow/px6TZ2K76J6 From dovecot at dovecot.org Tue Nov 26 22:10:32 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 26 Nov 2013 22:10:32 +0200 Subject: dovecot-2.2: replicator: Give a better error message with path w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5c170e0786f3 changeset: 17034:5c170e0786f3 user: Timo Sirainen date: Tue Nov 26 21:10:23 2013 +0100 description: replicator: Give a better error message with path when replicator.db is corrupted. diffstat: src/replication/replicator/replicator-queue.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 7af50f53cc7e -r 5c170e0786f3 src/replication/replicator/replicator-queue.c --- a/src/replication/replicator/replicator-queue.c Mon Nov 25 01:24:35 2013 +0200 +++ b/src/replication/replicator/replicator-queue.c Tue Nov 26 21:10:23 2013 +0100 @@ -366,7 +366,8 @@ ret = replicator_queue_import_line(queue, line); } T_END; if (ret < 0) { - i_error("Invalid replicator db record: %s", line); + i_error("Corrupted replicator record in %s: %s", + path, line); break; } }